diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 000000000..0c4b67d3c --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,83 @@ +# GitHub Copilot instructions for velero-io/velero + +## Changelog requirement + +Velero uses per-PR changelog fragments that are assembled into release notes. + +### File naming convention + +Every shipping PR must add exactly **one** file at: + +``` +changelogs/unreleased/- +``` + +- `` is the pull request number (e.g. `10200`). +- `` is the GitHub login of the PR author (e.g. `jdoe`). +- The file has **no extension**. +- The file content is a single line describing the change (the PR title is a + sensible default). + +**Example:** PR #10200 by `jdoe` → `changelogs/unreleased/10200-jdoe` + +The easiest way to create this file is: + +```bash +make new-changelog CHANGELOG_BODY="Brief description of the change" +``` + +`make new-changelog` reads the PR number and author from `gh pr view`; the file +is written automatically to the correct path with the correct name. + +The CI check (`hack/changelog-check.sh`) looks for +`changelogs/unreleased/-*` and fails if no file is found. + +### When a changelog is NOT required + +A changelog entry is **not** required when a PR exclusively changes non-shipping +content, i.e. the only files touched belong to one or more of these categories: + +| Category | Paths | +|---|---| +| GitHub Actions / CI workflows | `.github/**` | +| Documentation | `site/content/docs/**`, `site/**`, `docs/**`, `*.md` | +| Website (non-docs) | `site/**` (excluding `site/content/docs/**`) | + +When you open or review a PR that falls into one of the above categories (and does +**not** modify `pkg/`, `internal/`, `cmd/`, `vendor/`, `hack/`, `Makefile`, +`go.mod`, `go.sum`, or `changelogs/**`), apply the label +**`kind/changelog-not-required`** instead of requesting a changelog entry. The +`labeler.yml` auto-labeler handles this automatically for most cases; apply the +label manually if the auto-labeler did not. + +## Backport / cherry-pick workflow + +Velero uses `.github/workflows/backport.yml` to automate cherry-picks onto release +branches. + +- **Before merge:** comment `/backport release-1.17` (or `/cherrypick release-1.17`) + to add the label `backport release-1.17` to the PR. Multiple branches can be + space-delimited: `/backport release-1.17 release-1.18`. The label causes the + backport to run automatically when the PR merges. +- **After merge:** the same comment immediately creates the backport PR. +- **Shorthand:** a bare version like `/backport 1.17` is automatically expanded + to `release-1.17`; this works generically for any `X.Y` version. +- **Changelog filename:** the cherry-picked commit(s) carry over the source + PR's `changelogs/unreleased/-` file. The workflow + automatically renames it to `-` on the backport branch so + `hack/changelog-check.sh` passes and release notes cite the correct PR. +- **Changelog-not-required:** if the source PR is labeled + `kind/changelog-not-required`, that label is copied to the backport PR so + it isn't flagged as missing a changelog. +- **DCO signoff:** every commit on a backport branch is re-signed with the + bot's `Signed-off-by` trailer (`git rebase --signoff`), including + cherry-picked commits from the original author, so the DCO check always + passes on backport PRs. +- Only repository **owners, members, and collaborators** may trigger these commands. + +## General coding guidelines + +- Follow the existing code style of the file being edited. +- Add unit tests for new exported functions in `pkg/`. +- Do not commit secrets, credentials, or API tokens. +- Keep PRs focused; prefer small, reviewable changes over large omnibus PRs. diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 682c01231..a26f3eedf 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,6 +5,10 @@ updates: directory: "/" schedule: interval: "weekly" + groups: + github-actions: + patterns: + - "*" labels: - "Dependencies" - "github_actions" diff --git a/.github/labeler.yml b/.github/labeler.yml index 183f8365f..880977caf 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -31,3 +31,25 @@ has-e2e-2tests: has-unit-tests: - changed-files: - any-glob-to-any-file: pkg/**/*_test.go +# PRs that only touch non-shipping files (.github/ config, workflows, or docs) +# do not need a changelog entry; auto-apply the label so the changelog check passes. +kind/changelog-not-required: + - all: + - changed-files: + - any-glob-to-any-file: + - .github/**/* + - site/content/docs/**/* + - site/**/* + - '*.md' + - docs/**/* + - all-globs-to-all-files: + - '!pkg/**' + - '!internal/**' + - '!cmd/**' + - '!vendor/**' + - '!hack/**' + - '!Makefile' + - '!go.mod' + - '!go.sum' + - '!changelogs/**' + - '!**/*.go' diff --git a/.github/workflows/auto_assign_prs.yml b/.github/workflows/auto_assign_prs.yml index 8966b235e..bbcb3a9f4 100644 --- a/.github/workflows/auto_assign_prs.yml +++ b/.github/workflows/auto_assign_prs.yml @@ -6,6 +6,10 @@ name: "Auto Assign Author" on: pull_request_target: types: [opened, reopened, ready_for_review] + # Watch for submitted reviews so we can re-request a second CODEOWNERS + # review once only one maintainer has approved. + pull_request_review: + types: [submitted] permissions: contents: read @@ -14,10 +18,72 @@ permissions: jobs: # Automatically assigns reviewers and owner add-reviews: - if: github.repository == 'velero-io/velero' + if: github.repository == 'velero-io/velero' && github.event_name == 'pull_request_target' runs-on: ubuntu-latest steps: - name: Set the author of a PR as the assignee - uses: kentaro-m/auto-assign-action@v2.0.0 + uses: kentaro-m/auto-assign-action@v2.0.2 with: configuration-path: ".github/auto-assignees.yml" + + # `.github/CODEOWNERS` automatically requests review from the + # velero-io/maintainer team, but that request is cleared as soon as a + # single member of the team submits a review. Since we require a minimum + # of 2 reviewers (see `number_of_reviewers` in auto-assignees.yml), this + # re-requests a review from the maintainer team whenever a PR still has + # fewer than the required number of approvals, so a second CODEOWNERS + # reviewer gets pinged. + re-request-review: + if: github.repository == 'velero-io/velero' && github.event_name == 'pull_request_review' && github.event.review.state == 'approved' + runs-on: ubuntu-latest + steps: + - name: Re-request review from maintainers if more approvals are needed + uses: actions/github-script@v9 + with: + script: | + const requiredApprovals = 2; + const maintainerTeam = 'maintainer'; + const { owner, repo } = context.repo; + const pull_number = context.payload.pull_request.number; + + const { data: reviews } = await github.rest.pulls.listReviews({ + owner, + repo, + pull_number, + }); + + // Count distinct users whose most recent review is an approval. + // The Reviews API does not guarantee chronological order, so + // sort by submission time before folding into the map. + const sortedReviews = [...reviews].sort( + (a, b) => new Date(a.submitted_at) - new Date(b.submitted_at) + ); + const latestReviewByUser = new Map(); + for (const review of sortedReviews) { + latestReviewByUser.set(review.user.login, review.state); + } + const approvedReviewers = [...latestReviewByUser.entries()].filter( + ([, state]) => state === 'APPROVED' + ); + + if (approvedReviewers.length >= requiredApprovals) { + console.log( + `PR already has ${approvedReviewers.length} approvals, no need to re-request review.` + ); + return; + } + + console.log( + `PR has ${approvedReviewers.length}/${requiredApprovals} approvals, re-requesting review from @${owner}/${maintainerTeam}.` + ); + + try { + await github.rest.pulls.requestReviewers({ + owner, + repo, + pull_number, + team_reviewers: [maintainerTeam], + }); + } catch (error) { + core.warning(`Failed to re-request review from maintainers: ${error.message}`); + } diff --git a/.github/workflows/auto_label_prs.yml b/.github/workflows/auto_label_prs.yml index 21540d8cb..cc61473db 100644 --- a/.github/workflows/auto_label_prs.yml +++ b/.github/workflows/auto_label_prs.yml @@ -18,6 +18,6 @@ jobs: if: github.repository == 'velero-io/velero' runs-on: ubuntu-latest steps: - - uses: actions/labeler@v5 + - uses: actions/labeler@v7 with: configuration-path: .github/labeler.yml diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml new file mode 100644 index 000000000..7aaf37058 --- /dev/null +++ b/.github/workflows/backport.yml @@ -0,0 +1,260 @@ +name: Backport merged pull request + +# Automates cherry-picking merged PRs onto release branches. +# +# Pre-merge (open PR): +# An authorized /backport or /cherrypick comment adds one `backport ` +# label per requested branch. These labels are then picked up automatically +# when the PR is merged (see the pull_request_target: closed trigger below). +# +# Post-merge (merged PR): +# - Label a PR with e.g. `backport release-1.17` before merging; the label +# triggers the backport automatically when the PR closes as merged. +# - Comment `/backport release-1.17` or `/cherrypick release-1.17` on an +# already-merged PR to create the backport PR immediately. +# +# In both cases multiple target branches can be space-delimited in a comment: +# /backport release-1.17 release-1.18 +# +# As a shorthand, a bare release version (e.g. `1.17`) is automatically +# expanded to the corresponding `release-1.17` branch, so `/backport 1.17` +# and `/backport release-1.17` are equivalent. This works generically for +# any `X.Y` version, e.g. `/backport 1.18 1.19`. +# +# The cherry-picked commit(s) carry over the original PR's changelog file +# (changelogs/unreleased/-), which no longer matches the +# backport PR's own number. After the backport PR is created, its changelog +# file is automatically renamed to - so that +# hack/changelog-check.sh passes and release notes cite the correct PR. +# +# If the source PR is labeled `kind/changelog-not-required` (i.e. it has no +# changelog file), that label is copied to the backport PR so it isn't +# flagged as missing a changelog either. +# +# Every commit on a backport branch (the cherry-picked commit(s), even from +# the original author, plus the changelog rename commit) is re-signed with +# the bot's Signed-off-by trailer via `git rebase --signoff`, so the DCO +# check always passes regardless of whether the original commit had one. +# +# See: https://github.com/velero-io/velero/issues/9603 + +on: + pull_request_target: + types: [closed] + issue_comment: + types: [created] + +permissions: {} + +# Shared condition for authorized /backport or /cherrypick comments. +# Used by both jobs below to avoid duplicating the gate logic. +env: + AUTHORIZED_COMMENT: >- + ${{ + github.event_name == 'issue_comment' && + github.event.issue.pull_request != '' && + github.event.comment.user.id != 97796249 && + contains( + fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), + github.event.comment.author_association + ) && + ( + startsWith(github.event.comment.body, '/backport') || + startsWith(github.event.comment.body, '/cherrypick') + ) + }} + +jobs: + # ── Pre-merge: convert a /backport or /cherrypick comment into labels ─────── + # When the PR is still open the backport-action cannot run (it requires a + # merged commit). Instead, add one `backport ` label per requested + # branch so that the post-merge job picks them up automatically on close. + label-for-backport: + name: Label PR for deferred backport + # Run only when an authorized command is posted on an *open* (unmerged) PR. + if: > + github.repository == 'velero-io/velero' && + github.event_name == 'issue_comment' && + github.event.issue.pull_request != '' && + github.event.issue.state == 'open' && + github.event.comment.user.id != 97796249 && + contains( + fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), + github.event.comment.author_association + ) && + ( + startsWith(github.event.comment.body, '/backport') || + startsWith(github.event.comment.body, '/cherrypick') + ) + runs-on: ubuntu-latest + permissions: + issues: write # apply labels to the PR (PRs share the issues API) + steps: + - name: Parse branches and apply labels + env: + COMMENT_BODY: ${{ github.event.comment.body }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.issue.number }} + run: | + # Extract branch names from the first line of the comment. + # Strip the /backport or /cherrypick prefix; what remains is a + # space-delimited list of target branch names. + line=$(printf '%s' "$COMMENT_BODY" | head -n1 | tr -d '\r') + branches=$(printf '%s' "$line" | sed -E 's#^/(backport|cherrypick)[[:space:]]*##') + + if [ -z "$branches" ]; then + echo "No target branches specified in comment; nothing to label." + exit 0 + fi + + for branch in $branches; do + # Shorthand: a bare version like "1.17" expands to "release-1.17". + if [[ "$branch" =~ ^[0-9]+\.[0-9]+$ ]]; then + branch="release-${branch}" + fi + label="backport ${branch}" + echo "Applying label: '${label}'" + # Create the label if it does not exist yet (idempotent). + gh label create "${label}" --repo "${REPO}" --color "0075ca" \ + --description "Backport to ${branch}" 2>/dev/null || true + gh issue edit "${PR_NUMBER}" --repo "${REPO}" --add-label "${label}" + done + + # ── Post-merge: create backport PRs ───────────────────────────────────────── + backport: + name: Backport pull request + # Exclude comments from the backport-action bot (user id 97796249) to prevent + # recursive triggers. The bot does not post /backport commands, so startsWith + # already blocks recursion; the id check is defense in depth. + if: > + github.repository == 'velero-io/velero' && + ( + ( + github.event_name == 'pull_request_target' && + github.event.pull_request.merged && + contains(toJSON(github.event.pull_request.labels.*.name), '"backport ') + ) || ( + github.event_name == 'issue_comment' && + github.event.issue.pull_request != '' && + github.event.issue.state == 'closed' && + github.event.comment.user.id != 97796249 && + contains( + fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), + github.event.comment.author_association + ) && + ( + startsWith(github.event.comment.body, '/backport') || + startsWith(github.event.comment.body, '/cherrypick') + ) + ) + ) + runs-on: ubuntu-latest + permissions: + contents: write # push backport branches and comment + pull-requests: write # open backport PRs + steps: + - name: Parse target branches from comment + id: parse + if: github.event_name == 'issue_comment' + env: + COMMENT_BODY: ${{ github.event.comment.body }} + run: | + # First line only; strip /backport or /cherrypick prefix. + # Remaining text is a space-delimited list of target branches + # (may be empty, falls back to labels). + line=$(printf '%s' "$COMMENT_BODY" | head -n1 | tr -d '\r') + branches=$(printf '%s' "$line" | sed -E 's#^/(backport|cherrypick)[[:space:]]*##') + + normalized="" + for branch in $branches; do + # Shorthand: a bare version like "1.17" expands to "release-1.17". + if [[ "$branch" =~ ^[0-9]+\.[0-9]+$ ]]; then + branch="release-${branch}" + fi + normalized="${normalized}${normalized:+ }${branch}" + done + echo "branches=${normalized}" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Create backport pull requests + id: backport + # Pin to commit SHA: workflow has contents/pull-requests write. + uses: korthout/backport-action@2e830a1d0b8269505846ddd407a70876913ad1f8 # v4.6.0 + with: + # Labels like `backport release-1.17` select the target branch. + label_pattern: '^backport ([^ ]+)$' + # Carry over `kind/changelog-not-required` from the source PR so + # backport PRs of changelog-exempt changes aren't flagged as missing one. + copy_labels_pattern: '^kind/changelog-not-required$' + # Prefer draft PRs with conflict markers over failing the job silently. + experimental: | + { + "conflict_resolution": "draft_commit_conflicts" + } + # Empty when triggered by merge labels; set when `/backport` or `/cherrypick` includes branches. + target_branches: ${{ steps.parse.outputs.branches }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Rename changelog file(s) and ensure DCO signoff + # The cherry-picked commit(s) still carry the source PR's changelog + # filename (e.g. changelogs/unreleased/9795-kaovilai), which no + # longer matches the new backport PR's number. Rename it on each + # created backport branch so hack/changelog-check.sh passes and the + # release notes cite the correct PR. + # + # Also ensure every commit on the backport branch passes the DCO + # check by re-signing it with the bot's Signed-off-by trailer via + # `git rebase --signoff`. This covers the cherry-picked commits + # (even when the original author's commit had no trailer) as well + # as the changelog rename commit added above; it preserves any + # existing Signed-off-by trailers rather than replacing them. + if: steps.backport.outputs.created_pull_numbers != '' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + SOURCE_PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }} + CREATED_PR_NUMBERS: ${{ steps.backport.outputs.created_pull_numbers }} + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + shopt -s nullglob + for new_pr in $CREATED_PR_NUMBERS; do + if [ "$new_pr" = "$SOURCE_PR_NUMBER" ]; then + continue + fi + + branch=$(gh pr view "$new_pr" --repo "$REPO" --json headRefName -q .headRefName) + base_branch=$(gh pr view "$new_pr" --repo "$REPO" --json baseRefName -q .baseRefName) + git fetch origin "$branch" "$base_branch" + git checkout -B "$branch" "origin/${branch}" + + files=(changelogs/unreleased/"${SOURCE_PR_NUMBER}"-*) + if [ ${#files[@]} -gt 0 ]; then + for old_file in "${files[@]}"; do + suffix=$(basename "$old_file" | sed -E "s/^${SOURCE_PR_NUMBER}-//") + new_file="changelogs/unreleased/${new_pr}-${suffix}" + if [ "$old_file" != "$new_file" ]; then + git mv "$old_file" "$new_file" + fi + done + if ! git diff --cached --quiet; then + git commit -m "Rename changelog to match backport PR #${new_pr}" + fi + else + echo "No changelog file for PR ${SOURCE_PR_NUMBER} found on ${branch}; skipping rename." + fi + + # Add the bot's Signed-off-by trailer to every commit ahead of + # the target branch (cherry-picked commits + the rename commit). + if ! git rebase --signoff "origin/${base_branch}"; then + echo "::error::git rebase --signoff failed for PR #${new_pr} on branch ${branch}; aborting rebase, branch left unchanged." >&2 + git rebase --abort + exit 1 + fi + git push --force-with-lease origin "HEAD:${branch}" + done diff --git a/.github/workflows/e2e-test-kind.yaml b/.github/workflows/e2e-test-kind.yaml index fc77cb4d3..fcf0d37c2 100644 --- a/.github/workflows/e2e-test-kind.yaml +++ b/.github/workflows/e2e-test-kind.yaml @@ -1,6 +1,14 @@ name: "Run the E2E test on kind" +permissions: + contents: read env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + # Reviewed commit pins for third-party sources this workflow clones and executes. + # Bump them deliberately after reviewing the upstream changes. + # bitnami/containers: [bitnami/minio] Release 2026.7.17-debian-12-r0 + BITNAMI_CONTAINERS_COMMIT: 19fb570e551f15ab0c8264aafa93774266761b8d + # vmware-tanzu-experiments/distributed-data-generator: main as of 2025-07-15 + KIBISHII_COMMIT: bce0469e5f9dd33f31432fab22ff90ad6f2b45ca on: push: pull_request: @@ -19,28 +27,26 @@ jobs: build: runs-on: ubuntu-latest needs: get-go-version - outputs: - minio-dockerfile-sha: ${{ steps.minio-version.outputs.dockerfile_sha }} steps: - name: Check out the code - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Set up Go version - uses: actions/setup-go@v6 + uses: actions/setup-go@v7 with: go-version: ${{ needs.get-go-version.outputs.version }} # Look for a CLI that's made for this PR - name: Fetch built CLI id: cli-cache - uses: actions/cache@v4 + uses: actions/cache/restore@v6 with: path: ./_output/bin/linux/amd64/velero # The cache key a combination of the current PR number and the commit SHA key: velero-cli-${{ github.event.pull_request.number }}-${{ github.sha }} - name: Fetch built image id: image-cache - uses: actions/cache@v4 + uses: actions/cache/restore@v6 with: path: ./velero.tar # The cache key a combination of the current PR number and the commit SHA @@ -56,28 +62,48 @@ jobs: run: | IMAGE=velero VERSION=pr-test BUILD_OUTPUT_TYPE=docker make container docker save velero:pr-test-linux-amd64 -o ./velero.tar - # Check and build MinIO image once for all e2e tests - - name: Check Bitnami MinIO Dockerfile version - id: minio-version - env: - GH_TOKEN: ${{ github.token }} - run: | - DOCKERFILE_SHA=$(curl -s -H "Authorization: Bearer $GH_TOKEN" https://api.github.com/repos/bitnami/containers/commits?path=bitnami/minio/2026/debian-12/Dockerfile\&per_page=1 | jq -r '.[0].sha') - echo "dockerfile_sha=${DOCKERFILE_SHA}" >> $GITHUB_OUTPUT + # Build the MinIO image once for all e2e tests, from the reviewed bitnami/containers commit. - name: Cache MinIO Image - uses: actions/cache@v4 + uses: actions/cache/restore@v6 id: minio-cache with: path: ./minio-image.tar - key: minio-bitnami-${{ steps.minio-version.outputs.dockerfile_sha }} + key: minio-bitnami-${{ env.BITNAMI_CONTAINERS_COMMIT }} - name: Build MinIO Image from Bitnami Dockerfile if: steps.minio-cache.outputs.cache-hit != 'true' run: | - echo "Building MinIO image from Bitnami Dockerfile..." - git clone --depth 1 https://github.com/bitnami/containers.git /tmp/bitnami-containers + set -euo pipefail + echo "Building MinIO image from Bitnami Dockerfile at ${BITNAMI_CONTAINERS_COMMIT}..." + git init -q /tmp/bitnami-containers + git -C /tmp/bitnami-containers remote add origin https://github.com/bitnami/containers.git + git -C /tmp/bitnami-containers fetch --depth 1 origin "${BITNAMI_CONTAINERS_COMMIT}" + git -C /tmp/bitnami-containers checkout -q "${BITNAMI_CONTAINERS_COMMIT}" cd /tmp/bitnami-containers/bitnami/minio/2026/debian-12 docker build -t bitnami/minio:local . docker save bitnami/minio:local > ${{ github.workspace }}/minio-image.tar + # Save the freshly built artifacts to the cache explicitly, before this + # job reports completion. actions/cache saves in a post-job hook that + # runs *after* the job finishes, so the dependent run-e2e-test jobs (which + # start as soon as build completes) would race the save and miss the cache + # on a force push. See #9927. + - name: Save built CLI to cache + if: steps.cli-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + with: + path: ./_output/bin/linux/amd64/velero + key: velero-cli-${{ github.event.pull_request.number }}-${{ github.sha }} + - name: Save built image to cache + if: steps.image-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + with: + path: ./velero.tar + key: velero-image-${{ github.event.pull_request.number }}-${{ github.sha }} + - name: Save MinIO image to cache + if: steps.minio-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + with: + path: ./minio-image.tar + key: minio-bitnami-${{ steps.minio-version.outputs.dockerfile_sha }} # Create json of k8s versions to test # from guide: https://stackoverflow.com/a/65094398/4590470 setup-test-matrix: @@ -91,12 +117,41 @@ jobs: id: set-matrix # everything excluding older tags. limits needs to be high enough to cover all latest versions # and test labels - # grep -E "v[1-9]\.(2[5-9]|[3-9][0-9])" filters for v1.25 to v9.99 + # grep -E "^v[1-9]\.(2[5-9]|[3-9][0-9])\.[0-9]+$" filters for well-formed v1.25.x to v9.99.x + # GA releases only, so a pre-release tag like v1.37.0-rc.1 can't reach the + # awk step below and be misparsed as a patch release (e.g. "1.37.1") # and removes older patches of the same minor version # awk -F. '{if(!a[$1"."$2]++)print $1"."$2"."$NF}' run: | + set -euo pipefail + candidates=$(wget -q -O - "https://hub.docker.com/v2/namespaces/kindest/repositories/node/tags?page_size=50" | grep -o '"name": *"[^"]*' | grep -o '[^"]*$' | grep -E "^v[1-9]\.(2[5-9]|[3-9][0-9])\.[0-9]+$" | awk -F. '{if(!a[$1"."$2]++)print $1"."$2"."$NF}' | sort -r | sed s/v//g) + + # Docker Hub's tag listing can include tags whose manifest was never + # published or has since been removed (e.g. kindest/node:v1.37.1). If such + # a tag reaches the matrix, its job is guaranteed to fail with + # "manifest unknown" once helm/kind-action tries to pull it, which clogs + # up the e2e queue on every PR with a red job unrelated to the change + # under test. Test-pull each candidate's manifest here and drop any tag + # that isn't actually available before building the matrix. + valid=() + while IFS= read -r v; do + [ -z "$v" ] && continue + echo "Verifying kindest/node:v${v} image is available..." + if docker manifest inspect "kindest/node:v${v}" > /dev/null 2>&1; then + valid+=("$v") + else + echo "::warning::kindest/node:v${v} manifest not found on Docker Hub; excluding from e2e test matrix" + fi + done <<< "$candidates" + + if [ ${#valid[@]} -eq 0 ]; then + echo "::warning::No kindest/node tags passed the manifest availability check; the e2e test matrix will have no Kubernetes versions to test" + fi + + k8s_json=$(printf '%s\n' "${valid[@]+"${valid[@]}"}" | jq -R -c -s 'split("\n") | map(select(length > 0))') + echo "matrix={\ - \"k8s\":$(wget -q -O - "https://hub.docker.com/v2/namespaces/kindest/repositories/node/tags?page_size=50" | grep -o '"name": *"[^"]*' | grep -o '[^"]*$' | grep -v -E "alpha|beta" | grep -E "v[1-9]\.(2[5-9]|[3-9][0-9])" | awk -F. '{if(!a[$1"."$2]++)print $1"."$2"."$NF}' | sort -r | sed s/v//g | jq -R -c -s 'split("\n")[:-1]'),\ + \"k8s\":${k8s_json},\ \"labels\":[\ \"Basic && (ClusterResource || NodePort || StorageClass)\", \ \"ResourceFiltering && !FSBackup\", \ @@ -116,20 +171,20 @@ jobs: fail-fast: false steps: - name: Check out the code - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Set up Go version - uses: actions/setup-go@v6 + uses: actions/setup-go@v7 with: go-version: ${{ needs.get-go-version.outputs.version }} # Fetch the pre-built MinIO image from the build job - name: Fetch built MinIO Image - uses: actions/cache@v4 + uses: actions/cache/restore@v6 id: minio-cache with: path: ./minio-image.tar - key: minio-bitnami-${{ needs.build.outputs.minio-dockerfile-sha }} + key: minio-bitnami-${{ env.BITNAMI_CONTAINERS_COMMIT }} - name: Load MinIO Image run: | echo "Loading MinIO image..." @@ -137,20 +192,20 @@ jobs: - name: Install MinIO run: | docker run -d --rm -p 9000:9000 -e "MINIO_ROOT_USER=minio" -e "MINIO_ROOT_PASSWORD=minio123" -e "MINIO_DEFAULT_BUCKETS=bucket,additional-bucket" bitnami/minio:local - - uses: helm/kind-action@v1 + - uses: helm/kind-action@7a97ed793754775518f9db3a8151ee7461dc9c31 # v1 + fix: add curl retry flags (https://github.com/helm/kind-action/pull/165) with: cluster_name: "kind" version: "v0.32.0" node_image: "kindest/node:v${{ matrix.k8s }}" - name: Fetch built CLI id: cli-cache - uses: actions/cache@v4 + uses: actions/cache/restore@v6 with: path: ./_output/bin/linux/amd64/velero key: velero-cli-${{ github.event.pull_request.number }}-${{ github.sha }} - name: Fetch built Image id: image-cache - uses: actions/cache@v4 + uses: actions/cache/restore@v6 with: path: ./velero.tar key: velero-image-${{ github.event.pull_request.number }}-${{ github.sha }} @@ -166,10 +221,13 @@ jobs: EOF # Match kubectl version to k8s server version - curl -LO https://dl.k8s.io/release/v${{ matrix.k8s }}/bin/linux/amd64/kubectl + curl -fLO https://dl.k8s.io/release/v${{ matrix.k8s }}/bin/linux/amd64/kubectl sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl - git clone https://github.com/vmware-tanzu-experiments/distributed-data-generator.git -b main /tmp/kibishii + git init -q /tmp/kibishii + git -C /tmp/kibishii remote add origin https://github.com/vmware-tanzu-experiments/distributed-data-generator.git + git -C /tmp/kibishii fetch --depth 1 origin "${KIBISHII_COMMIT}" + git -C /tmp/kibishii checkout -q "${KIBISHII_COMMIT}" GOPATH=~/go \ CLOUD_PROVIDER=kind \ diff --git a/.github/workflows/get-go-version.yaml b/.github/workflows/get-go-version.yaml index 7a74fd845..fa4fb5e00 100644 --- a/.github/workflows/get-go-version.yaml +++ b/.github/workflows/get-go-version.yaml @@ -17,7 +17,7 @@ jobs: version: ${{ steps.pick-version.outputs.version }} steps: - name: Check out the code - uses: actions/checkout@v6 + uses: actions/checkout@v7 - id: pick-version run: | diff --git a/.github/workflows/nightly-trivy-scan.yml b/.github/workflows/nightly-trivy-scan.yml index be0aa4dcf..d3f2e6062 100644 --- a/.github/workflows/nightly-trivy-scan.yml +++ b/.github/workflows/nightly-trivy-scan.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 @@ -31,6 +31,6 @@ jobs: output: 'trivy-results.sarif' - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@v4.37.7 with: sarif_file: 'trivy-results.sarif' \ No newline at end of file diff --git a/.github/workflows/pr-changelog-check.yml b/.github/workflows/pr-changelog-check.yml index f9fb14f37..67c1a6221 100644 --- a/.github/workflows/pr-changelog-check.yml +++ b/.github/workflows/pr-changelog-check.yml @@ -13,7 +13,7 @@ jobs: steps: - name: Check out the code - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Changelog check if: ${{ !(contains(github.event.pull_request.labels.*.name, 'kind/changelog-not-required') || contains(github.event.pull_request.labels.*.name, 'Design') || contains(github.event.pull_request.labels.*.name, 'Website') || contains(github.event.pull_request.labels.*.name, 'Documentation'))}} diff --git a/.github/workflows/pr-ci-check.yml b/.github/workflows/pr-ci-check.yml index ba55e6ab0..fd5948f7b 100644 --- a/.github/workflows/pr-ci-check.yml +++ b/.github/workflows/pr-ci-check.yml @@ -14,17 +14,17 @@ jobs: fail-fast: false steps: - name: Check out the code - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Set up Go version - uses: actions/setup-go@v6 + uses: actions/setup-go@v7 with: go-version: ${{ needs.get-go-version.outputs.version }} - name: Make ci run: make ci - name: Upload test coverage - uses: codecov/codecov-action@v6 + uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} files: coverage.out diff --git a/.github/workflows/pr-codespell.yml b/.github/workflows/pr-codespell.yml index b65ae7ae5..97cdb48d4 100644 --- a/.github/workflows/pr-codespell.yml +++ b/.github/workflows/pr-codespell.yml @@ -9,13 +9,12 @@ jobs: steps: - name: Check out the code - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Codespell uses: codespell-project/actions-codespell@master with: - # ignore the config/.../crd.go file as it's generated binary data that is edited elsewhere. - skip: .git,*.png,*.jpg,*.woff,*.ttf,*.gif,*.ico,./config/crd/v1beta1/crds/crds.go,./config/crd/v1/crds/crds.go,./config/crd/v2alpha1/crds/crds.go,./go.sum,./LICENSE + skip: .git,*.png,*.jpg,*.woff,*.ttf,*.gif,*.ico,./go.sum,./LICENSE ignore_words_list: iam,aks,ist,bridget,ue,shouldnot,atleast,notin,sme,optin,sie check_filenames: true check_hidden: true diff --git a/.github/workflows/pr-containers.yml b/.github/workflows/pr-containers.yml index 910192171..b615d2b8e 100644 --- a/.github/workflows/pr-containers.yml +++ b/.github/workflows/pr-containers.yml @@ -14,7 +14,7 @@ jobs: name: Build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 name: Checkout - name: Set up QEMU @@ -25,7 +25,7 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 with: version: latest diff --git a/.github/workflows/pr-filepath-check.yml b/.github/workflows/pr-filepath-check.yml index 9b8ca593d..5ec2cb03b 100644 --- a/.github/workflows/pr-filepath-check.yml +++ b/.github/workflows/pr-filepath-check.yml @@ -9,7 +9,7 @@ jobs: steps: - name: Check out the code - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Validate file paths for Go module compatibility run: | diff --git a/.github/workflows/pr-goreleaser.yml b/.github/workflows/pr-goreleaser.yml index 802080cb5..0cbec3329 100644 --- a/.github/workflows/pr-goreleaser.yml +++ b/.github/workflows/pr-goreleaser.yml @@ -14,7 +14,7 @@ jobs: name: Build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 name: Checkout - name: Verify .goreleaser.yml and try a dryrun release. diff --git a/.github/workflows/pr-linter-check.yml b/.github/workflows/pr-linter-check.yml index 6ed7f073d..1a25569f1 100644 --- a/.github/workflows/pr-linter-check.yml +++ b/.github/workflows/pr-linter-check.yml @@ -18,10 +18,10 @@ jobs: needs: get-go-version steps: - name: Check out the code - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Set up Go version - uses: actions/setup-go@v6 + uses: actions/setup-go@v7 with: go-version: ${{ needs.get-go-version.outputs.version }} diff --git a/.github/workflows/prow-action.yml b/.github/workflows/prow-action.yml index 871f69f8f..7f8bb7f00 100644 --- a/.github/workflows/prow-action.yml +++ b/.github/workflows/prow-action.yml @@ -14,7 +14,7 @@ jobs: if: github.repository == 'velero-io/velero' runs-on: ubuntu-latest steps: - - uses: jpmcb/prow-github-actions@v1.1.3 + - uses: jpmcb/prow-github-actions@c44ac3a57d67639e39e4a4988b52049ef45b80dd # v2.0.0 with: # TODO: before allowing the /lgtm command, see if we can block merging if changelog labels are missing. prow-commands: | diff --git a/.github/workflows/push-builder.yml b/.github/workflows/push-builder.yml index 8e3e59c15..164d9104a 100644 --- a/.github/workflows/push-builder.yml +++ b/.github/workflows/push-builder.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: # The default value is "1" which fetches only a single commit. If we merge PR without squash or rebase, # there are at least two commits: the first one is the merge commit and the second one is the real commit diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index b45af38d9..f5ce8c456 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -8,6 +8,9 @@ on: tags: - '*' +permissions: + contents: read + jobs: get-go-version: uses: ./.github/workflows/get-go-version.yaml @@ -20,21 +23,21 @@ jobs: needs: get-go-version steps: - name: Check out the code - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Go version - uses: actions/setup-go@v6 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: ${{ needs.get-go-version.outputs.version }} - name: Set up QEMU id: qemu - uses: docker/setup-qemu-action@v4 + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 with: platforms: all - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 with: version: latest - name: Build @@ -45,7 +48,7 @@ jobs: - name: Test run: make test - name: Upload test coverage - uses: codecov/codecov-action@v6 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7 with: token: ${{ secrets.CODECOV_TOKEN }} files: coverage.out diff --git a/.github/workflows/rebase.yml b/.github/workflows/rebase.yml index 064bef70a..6db2503f0 100644 --- a/.github/workflows/rebase.yml +++ b/.github/workflows/rebase.yml @@ -1,18 +1,32 @@ -on: +name: Automatic Rebase + +on: issue_comment: types: [created] -name: Automatic Rebase + +permissions: {} + jobs: rebase: name: Rebase - if: github.repository == 'velero-io/velero' && github.event.issue.pull_request != '' && contains(github.event.comment.body, '/rebase') + if: >- + github.repository == 'velero-io/velero' && + github.event.issue.pull_request != null && + github.event.comment.body == '/rebase' && + contains( + fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), + github.event.comment.author_association + ) runs-on: ubuntu-latest + permissions: + # contents: write is required because updating the pull request branch + # pushes commits to the head branch. + contents: write + pull-requests: write steps: - - name: Checkout the latest code - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - name: Automatic Rebase - uses: cirrus-actions/rebase@1.8 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Rebase pull request + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.issue.number }} + run: gh pr update-branch "$PR_NUMBER" --repo "$GH_REPO" --rebase diff --git a/.github/workflows/stale-issues.yml b/.github/workflows/stale-issues.yml index 99a74872b..b66a339c8 100644 --- a/.github/workflows/stale-issues.yml +++ b/.github/workflows/stale-issues.yml @@ -8,7 +8,7 @@ jobs: if: github.repository == 'velero-io/velero' runs-on: ubuntu-latest steps: - - uses: actions/stale@v10.1.1 + - uses: actions/stale@v11.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: "This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 14 days. If a Velero team member has requested log or more information, please provide the output of the shared commands." diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md deleted file mode 100644 index fe6ec8c6f..000000000 --- a/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,148 +0,0 @@ -# Velero Code of Conduct - -Velero is a [Cloud Native Computing Foundation](https://www.cncf.io/) sandbox -project. As a CNCF project, the Velero community follows the -[**CNCF Code of Conduct**](https://github.com/cncf/foundation/blob/main/code-of-conduct.md). - -The text below is the project's adopted Code of Conduct, based on the -[Contributor Covenant](https://www.contributor-covenant.org/), and is -substantively aligned with the CNCF Code of Conduct. Where any conflict exists, -the CNCF Code of Conduct prevails. - -Instances of unacceptable behavior may be reported to the CNCF Code of -Conduct Committee at [conduct@cncf.io](mailto:conduct@cncf.io). For more -detailed instructions on how to submit a report, including how to submit a -report anonymously, please see the CNCF -[Incident Resolution Procedures](https://github.com/cncf/foundation/blob/main/code-of-conduct/coc-incident-resolution-procedures.md). -You can expect a response within three business days. - ---- - -# Contributor Covenant Code of Conduct - -## Our Pledge - -We as members, contributors, and leaders pledge to make participation in the Velero project and our -community a harassment-free experience for everyone, regardless of age, body -size, visible or invisible disability, ethnicity, sex characteristics, gender -identity and expression, level of experience, education, socioeconomic status, -nationality, personal appearance, race, religion, or sexual identity -and orientation. - -We pledge to act and interact in ways that contribute to an open, welcoming, -diverse, inclusive, and healthy community. - -## Our Standards - -Examples of behavior that contributes to a positive environment for our -community include: - -* Demonstrating empathy and kindness toward other people -* Being respectful of differing opinions, viewpoints, and experiences -* Giving and gracefully accepting constructive feedback -* Accepting responsibility and apologizing to those affected by our mistakes, - and learning from the experience -* Focusing on what is best not just for us as individuals, but for the - overall community - -Examples of unacceptable behavior include: - -* The use of sexualized language or imagery, and sexual attention or - advances of any kind -* Trolling, insulting or derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or email - address, without their explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Enforcement Responsibilities - -Community leaders are responsible for clarifying and enforcing our standards of -acceptable behavior and will take appropriate and fair corrective action in -response to any behavior that they deem inappropriate, threatening, offensive, -or harmful. - -Community leaders have the right and responsibility to remove, edit, or reject -comments, commits, code, wiki edits, issues, and other contributions that are -not aligned to this Code of Conduct, and will communicate reasons for moderation -decisions when appropriate. - -## Scope - -This Code of Conduct applies within all community spaces, and also applies when -an individual is officially representing the community in public spaces. -Examples of representing our community include using an official e-mail address, -posting via an official social media account, or acting as an appointed -representative at an online or offline event. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the CNCF Code of Conduct Committee at -[conduct@cncf.io](mailto:conduct@cncf.io). -All complaints will be reviewed and investigated promptly and fairly. - -All community leaders are obligated to respect the privacy and security of the -reporter of any incident. - -## Enforcement Guidelines - -Community leaders will follow these Community Impact Guidelines in determining -the consequences for any action they deem in violation of this Code of Conduct: - -### 1. Correction - -**Community Impact**: Use of inappropriate language or other behavior deemed -unprofessional or unwelcome in the community. - -**Consequence**: A private, written warning from community leaders, providing -clarity around the nature of the violation and an explanation of why the -behavior was inappropriate. A public apology may be requested. - -### 2. Warning - -**Community Impact**: A violation through a single incident or series -of actions. - -**Consequence**: A warning with consequences for continued behavior. No -interaction with the people involved, including unsolicited interaction with -those enforcing the Code of Conduct, for a specified period of time. This -includes avoiding interactions in community spaces as well as external channels -like social media. Violating these terms may lead to a temporary or -permanent ban. - -### 3. Temporary Ban - -**Community Impact**: A serious violation of community standards, including -sustained inappropriate behavior. - -**Consequence**: A temporary ban from any sort of interaction or public -communication with the community for a specified period of time. No public or -private interaction with the people involved, including unsolicited interaction -with those enforcing the Code of Conduct, is allowed during this period. -Violating these terms may lead to a permanent ban. - -### 4. Permanent Ban - -**Community Impact**: Demonstrating a pattern of violation of community -standards, including sustained inappropriate behavior, harassment of an -individual, or aggression toward or disparagement of classes of individuals. - -**Consequence**: A permanent ban from any sort of public interaction within -the community. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], -version 2.0, available at -https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. - -Community Impact Guidelines were inspired by [Mozilla's code of conduct -enforcement ladder](https://github.com/mozilla/diversity). - -[homepage]: https://www.contributor-covenant.org - -For answers to common questions about this code of conduct, see the FAQ at -https://www.contributor-covenant.org/faq. Translations are available at -https://www.contributor-covenant.org/translations. \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 24d7f4dbd..000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,3 +0,0 @@ -# Contributing - -Authors are expected to follow some guidelines when submitting PRs. Please see [our documentation](https://velero.io/docs/main/code-standards/) for details. diff --git a/GOVERNANCE.md b/GOVERNANCE.md deleted file mode 100644 index 687baeb00..000000000 --- a/GOVERNANCE.md +++ /dev/null @@ -1,135 +0,0 @@ -# Velero Governance - -This document defines the project governance for Velero. - -## Overview - -**Velero**, an open source project, is committed to building an open, inclusive, productive and self-governing open source community focused on building a high quality tool that enables users to safely backup and restore, perform disaster recovery, and migrate Kubernetes cluster resources and persistent volumes. The community is governed by this document with the goal of defining how community should work together to achieve this goal. - -## Code Repositories - -The following code repositories are governed by Velero community and maintained under the `vmware-tanzu\Velero` organization. - -* **[Velero](https://github.com/vmware-tanzu/velero):** Main Velero codebase -* **[Helm Chart](https://github.com/vmware-tanzu/helm-charts/tree/main/charts/velero):** The Helm chart for the Velero server component -* **[Velero CSI Plugin](https://github.com/vmware-tanzu/velero-plugin-for-csi):** This repository contains Velero plugins for snapshotting CSI backed PVCs using the CSI beta snapshot APIs -* **[Velero Plugin for vSphere](https://github.com/vmware-tanzu/velero-plugin-for-vsphere):** This repository contains the Velero Plugin for vSphere. This plugin is a volume snapshotter plugin that provides crash-consistent snapshots of vSphere block volumes and backup of volume data into S3 compatible storage. -* **[Velero Plugin for AWS](https://github.com/vmware-tanzu/velero-plugin-for-aws):** This repository contains the plugins to support running Velero on AWS, including the object store plugin and the volume snapshotter plugin -* **[Velero Plugin for GCP](https://github.com/vmware-tanzu/velero-plugin-for-gcp):** This repository contains the plugins to support running Velero on GCP, including the object store plugin and the volume snapshotter plugin -* **[Velero Plugin for Azure](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure):** This repository contains the plugins to support running Velero on Azure, including the object store plugin and the volume snapshotter plugin -* **[Velero Plugin Example](https://github.com/vmware-tanzu/velero-plugin-example):** This repository contains example plugins for Velero - - -## Community Roles - -* **Users:** Members that engage with the Velero community via any medium (Slack, GitHub, mailing lists, etc.). -* **Contributors:** Regular contributions to projects (documentation, code reviews, responding to issues, participation in proposal discussions, contributing code, etc.). -* **Maintainers**: The Velero project leaders. They are responsible for the overall health and direction of the project; final reviewers of PRs and responsible for releases. Some Maintainers are responsible for one or more components within a project, acting as technical leads for that component. Maintainers are expected to contribute code and documentation, review PRs including ensuring quality of code, triage issues, proactively fix bugs, and perform maintenance tasks for these components. - -### Maintainers - -New maintainers must be nominated by an existing maintainer and must be elected by a supermajority of existing maintainers. Likewise, maintainers can be removed by a supermajority of the existing maintainers or can resign by notifying one of the maintainers. - -### Supermajority - -A supermajority is defined as two-thirds of members in the group. -A supermajority of [Maintainers](#maintainers) is required for certain -decisions as outlined above. A supermajority vote is equivalent to the number of votes in favor being at least twice the number of votes against. For example, if you have 5 maintainers, a supermajority vote is 4 votes. Voting on decisions can happen on the mailing list, GitHub, Slack, email, or via a voting service, when appropriate. Maintainers can either vote "agree, yes, +1", "disagree, no, -1", or "abstain". A vote passes when supermajority is met. An abstain vote equals not voting at all. - -### Decision Making - -Ideally, all project decisions are resolved by consensus. If impossible, any -maintainer may call a vote. Unless otherwise specified in this document, any -vote will be decided by a supermajority of maintainers. - -Votes by maintainers belonging to the same company -will count as one vote; e.g., 4 maintainers employed by fictional company **Valerium** will -only have **one** combined vote. If voting members from a given company do not -agree, the company's vote is determined by a supermajority of voters from that -company. If no supermajority is achieved, the company is considered to have -abstained. - -## Proposal Process - -One of the most important aspects in any open source community is the concept -of proposals. Large changes to the codebase and / or new features should be -preceded by a proposal in our community repo. This process allows for all -members of the community to weigh in on the concept (including the technical -details), share their comments and ideas, and offer to help. It also ensures -that members are not duplicating work or inadvertently stepping on toes by -making large conflicting changes. - -The project roadmap is defined by accepted proposals. - -Proposals should cover the high-level objectives, use cases, and technical -recommendations on how to implement. In general, the community member(s) -interested in implementing the proposal should be either deeply engaged in the -proposal process or be an author of the proposal. - -The proposal should be documented as a separated markdown file pushed to the root of the -`design` folder in the [Velero](https://github.com/vmware-tanzu/velero/tree/main/design) -repository via PR. The name of the file should follow the name pattern `_design.md`, e.g: -`restore-hooks-design.md`. - -Use the [Proposal Template](https://github.com/vmware-tanzu/velero/blob/main/design/_template.md) as a starting point. - -### Proposal Lifecycle - -The proposal PR can follow the GitHub lifecycle of the PR to indicate its status: - -* **Open**: Proposal is created and under review and discussion. -* **Merged**: Proposal has been reviewed and is accepted (either by consensus or through a vote). -* **Closed**: Proposal has been reviewed and was rejected (either by consensus or through a vote). - -## Lazy Consensus - -To maintain velocity in a project as busy as Velero, the concept of [Lazy -Consensus](http://en.osswiki.info/concepts/lazy_consensus) is practiced. Ideas -and / or proposals should be shared by maintainers via -GitHub with the appropriate maintainer groups (e.g., -`@vmware-tanzu/velero-maintainers`) tagged. Out of respect for other contributors, -major changes should also be accompanied by a ping on Slack or a note on the -Velero mailing list as appropriate. Author(s) of proposal, Pull Requests, -issues, etc. will give a time period of no less than five (5) working days for -comment and remain cognizant of popular observed world holidays. - -Other maintainers may chime in and request additional time for review, but -should remain cognizant of blocking progress and abstain from delaying -progress unless absolutely needed. The expectation is that blocking progress -is accompanied by a guarantee to review and respond to the relevant action(s) -(proposals, PRs, issues, etc.) in short order. - -Lazy Consensus is practiced for all projects in the `Velero` org, including -the main project repository and the additional repositories. - -Lazy consensus does _not_ apply to the process of: - -* Removal of maintainers from Velero - -## Deprecation Policy - -### Deprecation Process - -Any contributor may introduce a request to deprecate a feature or an option of a feature by opening a feature request issue in the vmware-tanzu/velero GitHub project. The issue should describe why the feature is no longer needed or has become detrimental to Velero, as well as whether and how it has been superseded. The submitter should give as much detail as possible. - -Once the issue is filed, a one-month discussion period begins. Discussions take place within the issue itself as well as in the community meetings. The person who opens the issue, or a maintainer, should add the date and time marking the end of the discussion period in a comment on the issue as soon as possible after it is opened. A decision on the issue needs to be made within this one-month period. - -The feature will be deprecated by a supermajority vote of 50% plus one of the project maintainers at the time of the vote tallying, which is 72 hours after the end of the community meeting that is the end of the comment period. (Maintainers are permitted to vote in advance of the deadline, but should hold their votes until as close as possible to hear all possible discussion.) Votes will be tallied in comments on the issue. - -Non-maintainers may add non-binding votes in comments to the issue as well; these are opinions to be taken into consideration by maintainers, but they do not count as votes. - -If the vote passes, the deprecation window takes effect in the subsequent release, and the removal follows the schedule. - -### Schedule -If depreciation proposal passes by supermajority votes, the feature is deprecated in the next minor release and the feature can be removed completely after two minor version or equivalent major version e.g., if feature gets deprecated in Nth minor version, then feature can be removed after N+2 minor version or its equivalent if the major version number changes. - -### Deprecation Window - -The deprecation window is the period from the release in which the deprecation takes effect through the release in which the feature is removed. During this period, only critical security vulnerabilities and catastrophic bugs should be fixed. - -**Note:** If a backup relies on a deprecated feature, then backups made with the last Velero release before this feature is removed must still be restorable in version `n+2`. For instance, something like restic feature support, that might mean that restic is removed from the list of supported uploader types in version `n` but the underlying implementation required to restore from a restic backup won't be removed until release `n+2`. - -## Updating Governance - -All substantive changes in Governance require a supermajority agreement by all maintainers. diff --git a/Makefile b/Makefile index 515abf88d..31981217e 100644 --- a/Makefile +++ b/Makefile @@ -155,20 +155,40 @@ GOARCH = $(word 2, $(platform_temp)) GOPROXY ?= https://proxy.golang.org GOBIN=$$(pwd)/.go/bin +# Keep these build-image tool versions in sync with go.mod so the CLI/library +# pair doesn't drift (see https://github.com/velero-io/velero/issues/10023). +PROTOC_GEN_GO_VERSION := $(shell go list -m -f '{{.Version}}' google.golang.org/protobuf) +GOIMPORTS_VERSION := $(shell go list -m -f '{{.Version}}' golang.org/x/tools) + +# ============================================================================== +# ================================ COMMANDS ==================================== +# ============================================================================== + +# ================================== +# Help +# ================================== +# To document a new target, add "## " at the end of the target line. +# Example: new-target: ## Description of the new target + +.PHONY: help +help: ## Display this help message + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n\nTargets:\n"} /^[a-zA-Z0-9_%-]+:.*?##/ { printf " \033[36m%-20s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST) + + # If you want to build all binaries, see the 'all-build' rule. # If you want to build all containers, see the 'all-containers' rule. -all: +all: ## Build the default velero binary @$(MAKE) build -build-%: +build-%: ## Build specific binary @$(MAKE) --no-print-directory ARCH=$* build -all-build: $(addprefix build-, $(CLI_PLATFORMS)) +all-build: $(addprefix build-, $(CLI_PLATFORMS)) ## Build for all CLI platforms -all-containers: +all-containers: ## Build all containers @$(MAKE) --no-print-directory container -local: build-dirs +local: build-dirs ## Build locally # Add DEBUG=1 to enable debug locally GOOS=$(GOOS) \ GOARCH=$(GOARCH) \ @@ -182,7 +202,7 @@ local: build-dirs OUTPUT_DIR=$$(pwd)/_output/bin/$(GOOS)/$(GOARCH) \ ./hack/build.sh -build: _output/bin/$(GOOS)/$(GOARCH)/$(BIN) +build: _output/bin/$(GOOS)/$(GOARCH)/$(BIN) ## Build the velero binary (use build-- for specific targets) _output/bin/$(GOOS)/$(GOARCH)/$(BIN): build-dirs @echo "building: $@" @@ -202,7 +222,7 @@ _output/bin/$(GOOS)/$(GOARCH)/$(BIN): build-dirs TTY := $(shell tty -s && echo "-t") # Example: make shell CMD="date > datefile" -shell: build-dirs build-env +shell: build-dirs build-env ## Run a shell in the build container @# bind-mount the Velero root dir in at /github.com/vmware-tanzu/velero @# because the Kubernetes code-generator tools require the project to @# exist in a directory hierarchy ending like this (but *NOT* necessarily @@ -225,7 +245,7 @@ shell: build-dirs build-env $(BUILDER_IMAGE) \ /bin/sh $(CMD) -container: +container: ## Build the docker container (use container-- for specific targets) ifneq ($(CONTAINER_TOOL),docker) $(error $(DOCKER_ONLY_ERROR)) endif @@ -307,7 +327,7 @@ endif @echo "built container: $(IMAGE):$(VERSION)-windows-$(BUILDX_OSVERSION)-$(BUILDX_ARCH)" -push-manifest: +push-manifest: ## Push multi-arch manifest ifneq ($(CONTAINER_TOOL),docker) $(error $(DOCKER_ONLY_ERROR)) endif @@ -330,36 +350,36 @@ endif @docker manifest inspect --insecure=$(INSECURE_REGISTRY) $(IMAGE_TAG) SKIP_TESTS ?= -test: build-dirs +test: build-dirs ## Run unit tests ifneq ($(SKIP_TESTS), 1) @$(MAKE) shell CMD="-c 'hack/test.sh $(WHAT)'" endif -test-local: build-dirs +test-local: build-dirs ## Run unit tests locally ifneq ($(SKIP_TESTS), 1) hack/test.sh $(WHAT) endif -verify: +verify: ## Run all verify scripts ifneq ($(SKIP_TESTS), 1) @$(MAKE) shell CMD="-c 'hack/verify-all.sh'" endif -lint: +lint: ## Run linter ifneq ($(SKIP_TESTS), 1) @$(MAKE) shell CMD="-c 'hack/lint.sh'" endif -local-lint: +local-lint: ## Run linter locally ifneq ($(SKIP_TESTS), 1) @hack/lint.sh endif -update: +update: ## Run all update scripts @$(MAKE) shell CMD="-c 'hack/update-all.sh'" # update-crd is for development purpose only, it is faster than update, so is a shortcut when you want to generate CRD changes only -update-crd: +update-crd: ## Update generated CRD code @$(MAKE) shell CMD="-c 'hack/update-3generated-crd-code.sh'" build-dirs: @@ -387,7 +407,7 @@ else $(CONTAINER_TOOL) pull -q $(BUILDER_IMAGE) || $(MAKE) build-image endif -build-image: +build-image: ## Build the builder image @# When we build a new image we just untag the old one. @# This makes sure we don't leave the orphaned image behind. $(eval old_id=$(shell $(CONTAINER_TOOL) image inspect --format '{{ .ID }}' ${BUILDER_IMAGE} 2>/dev/null)) @@ -395,16 +415,16 @@ ifeq ($(BUILDX_ENABLED), true) ifneq ($(CONTAINER_TOOL),docker) $(error $(DOCKER_ONLY_ERROR)) endif - @cd hack/build-image && $(CONTAINER_TOOL) buildx build --build-arg=GOPROXY=$(GOPROXY) --output=type=docker --pull -t $(BUILDER_IMAGE) -f $(BUILDER_IMAGE_DOCKERFILE_REALPATH) . + @cd hack/build-image && $(CONTAINER_TOOL) buildx build --build-arg=GOPROXY=$(GOPROXY) --build-arg=PROTOC_GEN_GO_VERSION=$(PROTOC_GEN_GO_VERSION) --build-arg=GOIMPORTS_VERSION=$(GOIMPORTS_VERSION) --output=type=docker --pull -t $(BUILDER_IMAGE) -f $(BUILDER_IMAGE_DOCKERFILE_REALPATH) . else - @cd hack/build-image && $(CONTAINER_TOOL) build --build-arg=GOPROXY=$(GOPROXY) --pull -t $(BUILDER_IMAGE) -f $(BUILDER_IMAGE_DOCKERFILE_REALPATH) . + @cd hack/build-image && $(CONTAINER_TOOL) build --build-arg=GOPROXY=$(GOPROXY) --build-arg=PROTOC_GEN_GO_VERSION=$(PROTOC_GEN_GO_VERSION) --build-arg=GOIMPORTS_VERSION=$(GOIMPORTS_VERSION) --pull -t $(BUILDER_IMAGE) -f $(BUILDER_IMAGE_DOCKERFILE_REALPATH) . endif $(eval new_id=$(shell $(CONTAINER_TOOL) image inspect --format '{{ .ID }}' ${BUILDER_IMAGE} 2>/dev/null)) @if [ "$(old_id)" != "" ] && [ "$(old_id)" != "$(new_id)" ]; then \ $(CONTAINER_TOOL) rmi -f $$id || true; \ fi -push-build-image: +push-build-image: ## Push the builder image ifneq ($(CONTAINER_TOOL),docker) $(error $(DOCKER_ONLY_ERROR)) endif @@ -418,10 +438,10 @@ else docker push $(BUILDER_IMAGE) endif -build-image-hugo: +build-image-hugo: ## Build the hugo image for docs cd site && $(CONTAINER_TOOL) build --pull -t $(HUGO_IMAGE) . -clean: +clean: ## Clean up build artifacts and modcache # if we have a cached image then use it to run go clean --modcache # this test checks if we there is an image id in the BUILDER_IMAGE_CACHED variable. ifneq ($(strip $(BUILDER_IMAGE_CACHED)),) @@ -433,21 +453,21 @@ endif .PHONY: modules -modules: +modules: ## Tidy go modules go mod tidy .PHONY: verify-modules -verify-modules: modules +verify-modules: modules ## Verify go modules are up to date @if !(git diff --quiet HEAD -- go.sum go.mod); then \ echo "go module files are out of date, please commit the changes to go.mod and go.sum"; exit 1; \ fi -ci: verify-modules verify all test +ci: verify-modules verify all test ## Run CI checks -changelog: +changelog: ## Generate changelog hack/release-tools/changelog.sh # release builds a GitHub release using goreleaser within the build container. @@ -465,7 +485,7 @@ changelog: # RELEASE_NOTES_FILE=changelogs/CHANGELOG-1.2.md \ # PUBLISH=true \ # make release -release: +release: ## Build a GitHub release using goreleaser $(MAKE) shell CMD="-c '\ GITHUB_TOKEN=$(GITHUB_TOKEN) \ RELEASE_NOTES_FILE=$(RELEASE_NOTES_FILE) \ @@ -473,7 +493,7 @@ release: REGISTRY=$(REGISTRY) \ ./hack/release-tools/goreleaser.sh'" -serve-docs: build-image-hugo +serve-docs: build-image-hugo ## Serve the documentation site locally $(CONTAINER_TOOL) run \ --rm \ -v "$$(pwd)/site:/project" \ @@ -482,18 +502,18 @@ serve-docs: build-image-hugo server --bind=0.0.0.0 --enableGitInfo=false # gen-docs generates a new versioned docs directory under site/content/docs. # Please read the documentation in the script for instructions on how to use it. -gen-docs: +gen-docs: ## Generate a new versioned docs directory @hack/release-tools/gen-docs.sh .PHONY: test-e2e -test-e2e: local +test-e2e: local ## Run end-to-end tests $(MAKE) -e VERSION=$(VERSION) -C test/ run-e2e .PHONY: test-perf -test-perf: local +test-perf: local ## Run performance tests $(MAKE) -e VERSION=$(VERSION) -C test/ run-perf -go-generate: +go-generate: ## Run go generate go generate ./pkg/... # requires an authenticated gh cli @@ -505,11 +525,11 @@ go-generate: new-changelog: GH_LOGIN ?= $(shell gh pr view --json author --jq .author.login 2> /dev/null) new-changelog: GH_PR_NUMBER ?= $(shell gh pr view --json number --jq .number 2> /dev/null) new-changelog: CHANGELOG_BODY ?= '$(shell gh pr view --json title --jq .title)' -new-changelog: +new-changelog: ## Create a new changelog file for a PR @if [ "$(GH_LOGIN)" = "" ]; then \ echo "branch does not have PR or cli not logged in, try 'gh auth login' or 'gh pr create'"; \ exit 1; \ fi @mkdir -p ./changelogs/unreleased/ && \ echo $(CHANGELOG_BODY) > ./changelogs/unreleased/$(GH_PR_NUMBER)-$(GH_LOGIN) && \ - echo \"$(CHANGELOG_BODY)\" added to "./changelogs/unreleased/$(GH_PR_NUMBER)-$(GH_LOGIN)" \ No newline at end of file + echo \"$(CHANGELOG_BODY)\" added to "./changelogs/unreleased/$(GH_PR_NUMBER)-$(GH_LOGIN)" diff --git a/README.md b/README.md index 5457d145b..2357be2bb 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ ![100] [![Build Status][1]][2] [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/3811/badge)](https://bestpractices.coreinfrastructure.org/projects/3811) -![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/velero-io/velero) +[![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/velero-io/velero)](https://github.com/velero-io/velero/releases) +[![GitHub stars](https://img.shields.io/github/stars/velero-io/velero)](https://github.com/velero-io/velero/stargazers) +[![Docker Pulls](https://img.shields.io/docker/pulls/velero/velero.svg)](https://hub.docker.com/r/velero/velero) ## Overview diff --git a/SECURITY.md b/SECURITY.md deleted file mode 100644 index 84e6f45dc..000000000 --- a/SECURITY.md +++ /dev/null @@ -1,128 +0,0 @@ -# Security Release Process - -Velero is an open source tool with a growing community devoted to safe backup and restore, disaster recovery, and data migration of Kubernetes resources and persistent volumes. The community has adopted this security disclosure and response policy to ensure we responsibly handle critical issues. - - -## Supported Versions - -The Velero project maintains the following [governance document](https://github.com/vmware-tanzu/velero/blob/main/GOVERNANCE.md), [release document](https://github.com/vmware-tanzu/velero/blob/f42c63af1b9af445e38f78a7256b1c48ef79c10e/site/docs/main/release-instructions.md), and [support document](https://velero.io/docs/main/support-process/). Please refer to these for release and related details. Only the most recent version of Velero is supported. Each [release](https://github.com/vmware-tanzu/velero/releases) includes information about upgrading to the latest version. - - -## Reporting a Vulnerability - Private Disclosure Process - -Security is of the highest importance and all security vulnerabilities or suspected security vulnerabilities should be reported to Velero privately, to minimize attacks against current users of Velero before they are fixed. Vulnerabilities will be investigated and patched on the next patch (or minor) release as soon as possible. This information could be kept entirely internal to the project. - -If you know of a publicly disclosed security vulnerability for Velero, please **IMMEDIATELY** contact the Security Team (velero-security.pdl@broadcom.com). - - - -**IMPORTANT: Do not file public issues on GitHub for security vulnerabilities** - -To report a vulnerability or a security-related issue, please contact the email address with the details of the vulnerability. The email will be fielded by the Security Team and then shared with the Velero maintainers who have committer and release permissions. Emails will be addressed within 3 business days, including a detailed plan to investigate the issue and any potential workarounds to perform in the meantime. Do not report non-security-impacting bugs through this channel. Use [GitHub issues](https://github.com/vmware-tanzu/velero/issues/new/choose) instead. - - -## Proposed Email Content - -Provide a descriptive subject line and in the body of the email include the following information: - - - -* Basic identity information, such as your name and your affiliation or company. -* Detailed steps to reproduce the vulnerability (POC scripts, screenshots, and logs are all helpful to us). -* Description of the effects of the vulnerability on Velero and the related hardware and software configurations, so that the Security Team can reproduce it. -* How the vulnerability affects Velero usage and an estimation of the attack surface, if there is one. -* List other projects or dependencies that were used in conjunction with Velero to produce the vulnerability. - - - - -## When to report a vulnerability - - - -* When you think Velero has a potential security vulnerability. -* When you suspect a potential vulnerability but you are unsure that it impacts Velero. -* When you know of or suspect a potential vulnerability on another project that is used by Velero. - - - - -## Patch, Release, and Disclosure - -The Security Team will respond to vulnerability reports as follows: - - - - - -1. The Security Team will investigate the vulnerability and determine its effects and criticality. -2. If the issue is not deemed to be a vulnerability, the Security Team will follow up with a detailed reason for rejection. -3. The Security Team will initiate a conversation with the reporter within 3 business days. -4. If a vulnerability is acknowledged and the timeline for a fix is determined, the Security Team will work on a plan to communicate with the appropriate community, including identifying mitigating steps that affected users can take to protect themselves until the fix is rolled out. -5. The Security Team will also create a [CVSS](https://www.first.org/cvss/specification-document) using the [CVSS Calculator](https://www.first.org/cvss/calculator/3.0). The Security Team makes the final call on the calculated CVSS; it is better to move quickly than making the CVSS perfect. Issues may also be reported to [Mitre](https://cve.mitre.org/) using this [scoring calculator](https://nvd.nist.gov/vuln-metrics/cvss/v3-calculator). The CVE will initially be set to private. -6. The Security Team will work on fixing the vulnerability and perform internal testing before preparing to roll out the fix. -7. The Security Team will provide early disclosure of the vulnerability by emailing the [Velero Distributors](https://groups.google.com/u/1/g/projectvelero-distributors) mailing list. Distributors can initially plan for the vulnerability patch ahead of the fix, and later can test the fix and provide feedback to the Velero team. See the section **Early Disclosure to Velero Distributors List** for details about how to join this mailing list. -8. A public disclosure date is negotiated by the SecurityTeam, the bug submitter, and the distributors list. We prefer to fully disclose the bug as soon as possible once a user mitigation or patch is available. It is reasonable to delay disclosure when the bug or the fix is not yet fully understood, the solution is not well-tested, or for distributor coordination. The timeframe for disclosure is from immediate (especially if it’s already publicly known) to a few weeks. For a critical vulnerability with a straightforward mitigation, we expect the report date for the public disclosure date to be on the order of 14 business days. The Security Team holds the final say when setting a public disclosure date. -9. Once the fix is confirmed, the Security Team will patch the vulnerability in the next patch or minor release, and backport a patch release into all earlier supported releases. Upon release of the patched version of Velero, we will follow the **Public Disclosure Process**. - - -## Public Disclosure Process - -The Security Team publishes a [public advisory](https://github.com/vmware-tanzu/velero/security/advisories) to the Velero community via GitHub. In most cases, additional communication via Slack, Twitter, mailing lists, blog and other channels will assist in educating Velero users and rolling out the patched release to affected users. - -The Security Team will also publish any mitigating steps users can take until the fix can be applied to their Velero instances. Velero distributors will handle creating and publishing their own security advisories. - - - - -## Mailing lists - - - -* Use velero-security.pdl@broadcom.com to report security concerns to the Security Team, who uses the list to privately discuss security issues and fixes prior to disclosure. -* Join the [Velero Distributors](https://groups.google.com/u/1/g/projectvelero-distributors) mailing list for early private information and vulnerability disclosure. Early disclosure may include mitigating steps and additional information on security patch releases. See below for information on how Velero distributors or vendors can apply to join this list. - - -## Early Disclosure to Velero Distributors List - -The private list is intended to be used primarily to provide actionable information to multiple distributor projects at once. This list is not intended to inform individuals about security issues. - - -## Membership Criteria - -To be eligible to join the [Velero Distributors](https://groups.google.com/u/1/g/projectvelero-distributors) mailing list, you should: - - - -1. Be an active distributor of Velero. -2. Have a user base that is not limited to your own organization. -3. Have a publicly verifiable track record up to the present day of fixing security issues. -4. Not be a downstream or rebuild of another distributor. -5. Be a participant and active contributor in the Velero community. -6. Accept the Embargo Policy that is outlined below. -7. Have someone who is already on the list vouch for the person requesting membership on behalf of your distribution. - -**The terms and conditions of the Embargo Policy apply to all members of this mailing list. A request for membership represents your acceptance to the terms and conditions of the Embargo Policy.** - - -## Embargo Policy - -The information that members receive on the Velero Distributors mailing list must not be made public, shared, or even hinted at anywhere beyond those who need to know within your specific team, unless you receive explicit approval to do so from the Security Team. This remains true until the public disclosure date/time agreed upon by the list. Members of the list and others cannot use the information for any reason other than to get the issue fixed for your respective distribution's users. - -Before you share any information from the list with members of your team who are required to fix the issue, these team members must agree to the same terms, and only be provided with information on a need-to-know basis. - -In the unfortunate event that you share information beyond what is permitted by this policy, you must urgently inform the Security Team (velero-security.pdl@broadcom.com) of exactly what information was leaked and to whom. If you continue to leak information and break the policy outlined here, you will be permanently removed from the list. - - - - -## Requesting to Join - -Send new membership requests to projectvelero-distributors@googlegroups.com. In the body of your request please specify how you qualify for membership and fulfill each criterion listed in the Membership Criteria section above. - - -## Confidentiality, integrity and availability - -We consider vulnerabilities leading to the compromise of data confidentiality, elevation of privilege, or integrity to be our highest priority concerns. Availability, in particular in areas relating to DoS and resource exhaustion, is also a serious security concern. The Security Team takes all vulnerabilities, potential vulnerabilities, and suspected vulnerabilities seriously and will investigate them in an urgent and expeditious manner. - -Note that we do not currently consider the default settings for Velero to be secure-by-default. It is necessary for operators to explicitly configure settings, role based access control, and other resource related features in Velero to provide a hardened Velero environment. We will not act on any security disclosure that relates to a lack of safe defaults. Over time, we will work towards improved safe-by-default configuration, taking into account backwards compatibility. diff --git a/SUPPORT.md b/SUPPORT.md deleted file mode 100644 index 62c461036..000000000 --- a/SUPPORT.md +++ /dev/null @@ -1,7 +0,0 @@ -# Velero Support - -Thanks for trying out Velero! We welcome all feedback, find all the ways to connect with us on our Community page: - -- [Velero Community](https://velero.io/community/) - -You can find details on the Velero maintainers' support process [here](https://velero.io/docs/main/support-process/). diff --git a/changelogs/unreleased/10000-shubham-pampattiwar b/changelogs/unreleased/10000-shubham-pampattiwar new file mode 100644 index 000000000..4134b77e2 --- /dev/null +++ b/changelogs/unreleased/10000-shubham-pampattiwar @@ -0,0 +1 @@ +Fix stale backupLastSuccessfulTimestamp metric after schedule deletion diff --git a/changelogs/unreleased/10016-adam-jian-zhang b/changelogs/unreleased/10016-adam-jian-zhang new file mode 100644 index 000000000..1fab47983 --- /dev/null +++ b/changelogs/unreleased/10016-adam-jian-zhang @@ -0,0 +1 @@ +Fix issue #9938, add use guide for restore fine-grained filters via resource policy diff --git a/changelogs/unreleased/10035-HajimohammadiNet b/changelogs/unreleased/10035-HajimohammadiNet new file mode 100644 index 000000000..2905938ab --- /dev/null +++ b/changelogs/unreleased/10035-HajimohammadiNet @@ -0,0 +1 @@ +Trim whitespace around plugin image entries during install. diff --git a/changelogs/unreleased/10045-lubronzhan b/changelogs/unreleased/10045-lubronzhan new file mode 100644 index 000000000..d8974e9a3 --- /dev/null +++ b/changelogs/unreleased/10045-lubronzhan @@ -0,0 +1 @@ +Fix issue #5836, respect schedule.spec.template.metadata.annotations to override annotations copied from the Schedule to Backup objects, matching the existing behavior for labels diff --git a/changelogs/unreleased/10047-kaovilai b/changelogs/unreleased/10047-kaovilai new file mode 100644 index 000000000..6d96bdede --- /dev/null +++ b/changelogs/unreleased/10047-kaovilai @@ -0,0 +1 @@ +Fix restore-wait init container ignoring pod-level securityContext, falling back to hardcoded runAsUser 1000 instead of the workload's own uid/gid, causing fs-backup restores to deadlock at Init:0/1 on owner-restricted volumes diff --git a/changelogs/unreleased/10056-adam-jian-zhang b/changelogs/unreleased/10056-adam-jian-zhang new file mode 100644 index 000000000..18bd93cc6 --- /dev/null +++ b/changelogs/unreleased/10056-adam-jian-zhang @@ -0,0 +1 @@ +RIA must include additional items design diff --git a/changelogs/unreleased/10064-adam-jian-zhang b/changelogs/unreleased/10064-adam-jian-zhang new file mode 100644 index 000000000..9d45481db --- /dev/null +++ b/changelogs/unreleased/10064-adam-jian-zhang @@ -0,0 +1 @@ +Add set based label selectors for fine-grained filters diff --git a/changelogs/unreleased/10067-blackpiglet b/changelogs/unreleased/10067-blackpiglet new file mode 100644 index 000000000..3a4b67c31 --- /dev/null +++ b/changelogs/unreleased/10067-blackpiglet @@ -0,0 +1 @@ +Backup workflow for block data mover. \ No newline at end of file diff --git a/changelogs/unreleased/10070-shubham-pampattiwar b/changelogs/unreleased/10070-shubham-pampattiwar new file mode 100644 index 000000000..02f87194a --- /dev/null +++ b/changelogs/unreleased/10070-shubham-pampattiwar @@ -0,0 +1 @@ +Add snapshotClass parameter to volume policy snapshot action diff --git a/changelogs/unreleased/10071-Lyndon-Li b/changelogs/unreleased/10071-Lyndon-Li new file mode 100644 index 000000000..dd3454a4d --- /dev/null +++ b/changelogs/unreleased/10071-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #9828, add implementation for block uploader restore \ No newline at end of file diff --git a/changelogs/unreleased/10082-adam-jian-zhang b/changelogs/unreleased/10082-adam-jian-zhang new file mode 100644 index 000000000..e704ed6a7 --- /dev/null +++ b/changelogs/unreleased/10082-adam-jian-zhang @@ -0,0 +1 @@ +Add restore.velero.io/must-include-additional-items so RestoreItemActions can opt in to bypassing global restore filters for AdditionalItems (mirrors the backup-side must-include annotation; no default behavior change for existing restores/plugins) diff --git a/changelogs/unreleased/10087-adam-jian-zhang b/changelogs/unreleased/10087-adam-jian-zhang new file mode 100644 index 000000000..7edaa117f --- /dev/null +++ b/changelogs/unreleased/10087-adam-jian-zhang @@ -0,0 +1 @@ +Stop force-including VolumeSnapshotContents via resourceMustHave on every restore; CSI VolumeSnapshot/PVC RestoreItemActions now set restore.velero.io/must-include-additional-items so bound snapshot dependencies are restored only when their parent is restored (fixes #9957) diff --git a/changelogs/unreleased/10091-Lyndon-Li b/changelogs/unreleased/10091-Lyndon-Li new file mode 100644 index 000000000..b1e5deb03 --- /dev/null +++ b/changelogs/unreleased/10091-Lyndon-Li @@ -0,0 +1 @@ +Refactor block uploader thread module for better thread safety and code reading \ No newline at end of file diff --git a/changelogs/unreleased/10093-chlins b/changelogs/unreleased/10093-chlins new file mode 100644 index 000000000..143c6b856 --- /dev/null +++ b/changelogs/unreleased/10093-chlins @@ -0,0 +1 @@ +Replace rebase action with GitHub CLI diff --git a/changelogs/unreleased/10098-shubham-pampattiwar b/changelogs/unreleased/10098-shubham-pampattiwar new file mode 100644 index 000000000..0c48c1631 --- /dev/null +++ b/changelogs/unreleased/10098-shubham-pampattiwar @@ -0,0 +1 @@ +Implement server default restore resource modifier diff --git a/changelogs/unreleased/10102-chlins b/changelogs/unreleased/10102-chlins new file mode 100644 index 000000000..70b4b5c44 --- /dev/null +++ b/changelogs/unreleased/10102-chlins @@ -0,0 +1 @@ +Verify extracted item paths stay inside the backup directory diff --git a/changelogs/unreleased/10106-blackpiglet b/changelogs/unreleased/10106-blackpiglet new file mode 100644 index 000000000..404046c1e --- /dev/null +++ b/changelogs/unreleased/10106-blackpiglet @@ -0,0 +1 @@ +Fix some issues for CBT features: logs, CRD change, GetDataMover. \ No newline at end of file diff --git a/changelogs/unreleased/10125-chlins b/changelogs/unreleased/10125-chlins new file mode 100644 index 000000000..1a1b00371 --- /dev/null +++ b/changelogs/unreleased/10125-chlins @@ -0,0 +1 @@ +Cancel hook exec stream on timeout and bound hook timeouts diff --git a/changelogs/unreleased/10126-blackpiglet b/changelogs/unreleased/10126-blackpiglet new file mode 100644 index 000000000..451b1c2a0 --- /dev/null +++ b/changelogs/unreleased/10126-blackpiglet @@ -0,0 +1 @@ +Use "" as parentSnapshot for DU when BackupType is incremental. \ No newline at end of file diff --git a/changelogs/unreleased/10127-lubronzhan b/changelogs/unreleased/10127-lubronzhan new file mode 100644 index 000000000..9ac64d9d3 --- /dev/null +++ b/changelogs/unreleased/10127-lubronzhan @@ -0,0 +1 @@ +Add `velero client config set namespace-mode=auto` to make operational commands resolve their default namespace from the current kubeconfig context on every invocation diff --git a/changelogs/unreleased/10138-Jay2006sawant b/changelogs/unreleased/10138-Jay2006sawant new file mode 100644 index 000000000..cc5339217 --- /dev/null +++ b/changelogs/unreleased/10138-Jay2006sawant @@ -0,0 +1 @@ +Fix block uploader restore validation and BatchForget error handling diff --git a/changelogs/unreleased/10150-chlins b/changelogs/unreleased/10150-chlins new file mode 100644 index 000000000..158a3f43c --- /dev/null +++ b/changelogs/unreleased/10150-chlins @@ -0,0 +1 @@ +Drop node-agent host path mounts from data mover pods diff --git a/changelogs/unreleased/10155-chlins b/changelogs/unreleased/10155-chlins new file mode 100644 index 000000000..b17917098 --- /dev/null +++ b/changelogs/unreleased/10155-chlins @@ -0,0 +1 @@ +Verify downloaded build tools against architecture-specific SHA-256 checksums before installation. diff --git a/changelogs/unreleased/10158-Pragati5-DEBUG b/changelogs/unreleased/10158-Pragati5-DEBUG new file mode 100644 index 000000000..060fa5c38 --- /dev/null +++ b/changelogs/unreleased/10158-Pragati5-DEBUG @@ -0,0 +1 @@ +Add GitHub Action to automate backport/cherry-pick onto release branches diff --git a/changelogs/unreleased/10159-Pragati5-DEBUG b/changelogs/unreleased/10159-Pragati5-DEBUG new file mode 100644 index 000000000..e8f33122d --- /dev/null +++ b/changelogs/unreleased/10159-Pragati5-DEBUG @@ -0,0 +1 @@ +Fix excluded namespace objects leaking into backups when using cross-namespace listing diff --git a/changelogs/unreleased/10161-AftAb-25 b/changelogs/unreleased/10161-AftAb-25 new file mode 100644 index 000000000..f960a13df --- /dev/null +++ b/changelogs/unreleased/10161-AftAb-25 @@ -0,0 +1 @@ +Fixed a bug in the backup sync controller where transient API errors could cause backups to incorrectly lose their schedule owner references. diff --git a/changelogs/unreleased/10174-samay43 b/changelogs/unreleased/10174-samay43 new file mode 100644 index 000000000..b88a04b6c --- /dev/null +++ b/changelogs/unreleased/10174-samay43 @@ -0,0 +1 @@ +Add curl --fail flag to kubectl download in e2e kind workflow diff --git a/changelogs/unreleased/10176-blackpiglet b/changelogs/unreleased/10176-blackpiglet new file mode 100644 index 000000000..301e7c8e1 --- /dev/null +++ b/changelogs/unreleased/10176-blackpiglet @@ -0,0 +1 @@ +Support to set data mover for the uploader from volume policy. \ No newline at end of file diff --git a/changelogs/unreleased/10185-Lyndon-Li b/changelogs/unreleased/10185-Lyndon-Li new file mode 100644 index 000000000..7f8cfa422 --- /dev/null +++ b/changelogs/unreleased/10185-Lyndon-Li @@ -0,0 +1 @@ +Support full backup for file system data mover and pod volume backup \ No newline at end of file diff --git a/changelogs/unreleased/10200-Ralthos b/changelogs/unreleased/10200-Ralthos new file mode 100644 index 000000000..b54e0c7f8 --- /dev/null +++ b/changelogs/unreleased/10200-Ralthos @@ -0,0 +1 @@ +Add printer columns for Backup and Restore CRDs so kubectl shows status, errors, warnings and timing diff --git a/changelogs/unreleased/10201-Ralthos b/changelogs/unreleased/10201-Ralthos new file mode 100644 index 000000000..f41b37a2d --- /dev/null +++ b/changelogs/unreleased/10201-Ralthos @@ -0,0 +1 @@ +Show n/a instead of for unset timestamps in velero backup get and velero restore get diff --git a/changelogs/unreleased/10211-Ralthos b/changelogs/unreleased/10211-Ralthos new file mode 100644 index 000000000..ab77622f2 --- /dev/null +++ b/changelogs/unreleased/10211-Ralthos @@ -0,0 +1 @@ +Add printer columns for VolumeSnapshotLocation so kubectl shows provider and phase diff --git a/changelogs/unreleased/10218-opbot-xd b/changelogs/unreleased/10218-opbot-xd new file mode 100644 index 000000000..d7b291f3c --- /dev/null +++ b/changelogs/unreleased/10218-opbot-xd @@ -0,0 +1 @@ +Add missing test assertions for PVCBackupSummary in podvolume backupper diff --git a/changelogs/unreleased/10225-Lyndon-Li b/changelogs/unreleased/10225-Lyndon-Li new file mode 100644 index 000000000..435da13d1 --- /dev/null +++ b/changelogs/unreleased/10225-Lyndon-Li @@ -0,0 +1 @@ +Add prefetch mechanism to object reader so as to improve the restore throughput of block data mover \ No newline at end of file diff --git a/changelogs/unreleased/10227-ywk253100 b/changelogs/unreleased/10227-ywk253100 new file mode 100644 index 000000000..dcbbd6ac5 --- /dev/null +++ b/changelogs/unreleased/10227-ywk253100 @@ -0,0 +1 @@ +Add "SnapshotClass" to DataUploadResult \ No newline at end of file diff --git a/changelogs/unreleased/10229-Ralthos b/changelogs/unreleased/10229-Ralthos new file mode 100644 index 000000000..7963342e0 --- /dev/null +++ b/changelogs/unreleased/10229-Ralthos @@ -0,0 +1 @@ +Add printer columns for DownloadRequest and ServerStatusRequest so kubectl shows their target, status and server version diff --git a/changelogs/unreleased/10231-Ralthos b/changelogs/unreleased/10231-Ralthos new file mode 100644 index 000000000..415cc3d27 --- /dev/null +++ b/changelogs/unreleased/10231-Ralthos @@ -0,0 +1 @@ +Add a troubleshooting entry for artifact downloads failing when the BackupStorageLocation s3Url is only resolvable inside the cluster diff --git a/changelogs/unreleased/10234-Ralthos b/changelogs/unreleased/10234-Ralthos new file mode 100644 index 000000000..70ac3000a --- /dev/null +++ b/changelogs/unreleased/10234-Ralthos @@ -0,0 +1 @@ +Make velero restore logs return errors instead of calling os.Exit directly, matching velero backup logs, and enable the two previously skipped tests diff --git a/changelogs/unreleased/10242-beep-boopp b/changelogs/unreleased/10242-beep-boopp new file mode 100644 index 000000000..42312c448 --- /dev/null +++ b/changelogs/unreleased/10242-beep-boopp @@ -0,0 +1 @@ +Fix schedule reconciler aliasing &c.skipImmediately into Schedule specs, corrupting the server-wide --schedule-skip-immediately default after the first reconcile diff --git a/changelogs/unreleased/10243-reasonerjt b/changelogs/unreleased/10243-reasonerjt new file mode 100644 index 000000000..d650a70bd --- /dev/null +++ b/changelogs/unreleased/10243-reasonerjt @@ -0,0 +1 @@ +Mark the existed resource as skipped during restore \ No newline at end of file diff --git a/changelogs/unreleased/10245-Ralthos b/changelogs/unreleased/10245-Ralthos new file mode 100644 index 000000000..50e603dcd --- /dev/null +++ b/changelogs/unreleased/10245-Ralthos @@ -0,0 +1 @@ +Document that the DownloadRequest Processed phase means a URL has been signed, and that it does not imply the target object exists diff --git a/changelogs/unreleased/10247-opbot-xd b/changelogs/unreleased/10247-opbot-xd new file mode 100644 index 000000000..d1caa9255 --- /dev/null +++ b/changelogs/unreleased/10247-opbot-xd @@ -0,0 +1 @@ +Refactor: Replace context.TODO() with properly plumbed contexts in CSI backup actions and utility functions to enable proper cancellation of in-flight API requests. diff --git a/changelogs/unreleased/10250-Lyndon-Li b/changelogs/unreleased/10250-Lyndon-Li new file mode 100644 index 000000000..29ca747a0 --- /dev/null +++ b/changelogs/unreleased/10250-Lyndon-Li @@ -0,0 +1 @@ +Fix a potential deadlock when resultsLock is held by the informer but blocked on resChan because the early quit of RestorePodVolumes \ No newline at end of file diff --git a/changelogs/unreleased/10251-Lyndon-Li b/changelogs/unreleased/10251-Lyndon-Li new file mode 100644 index 000000000..6aead3879 --- /dev/null +++ b/changelogs/unreleased/10251-Lyndon-Li @@ -0,0 +1 @@ +Fix wrong node-agent check result when PVR restorer run concurrently \ No newline at end of file diff --git a/changelogs/unreleased/10252-Ralthos b/changelogs/unreleased/10252-Ralthos new file mode 100644 index 000000000..8766a8d42 --- /dev/null +++ b/changelogs/unreleased/10252-Ralthos @@ -0,0 +1 @@ +Skip signing a download URL when no artifacts can exist yet, and set a Failed phase with the reason so callers stop waiting diff --git a/changelogs/unreleased/10254-Lyndon-Li b/changelogs/unreleased/10254-Lyndon-Li new file mode 100644 index 000000000..e3ad3b7a8 --- /dev/null +++ b/changelogs/unreleased/10254-Lyndon-Li @@ -0,0 +1 @@ +Ignore credentialFile filled into BSL by users to avoid unexpected credentials used by Velero \ No newline at end of file diff --git a/changelogs/unreleased/10255-Lyndon-Li b/changelogs/unreleased/10255-Lyndon-Li new file mode 100644 index 000000000..f6ddc1e76 --- /dev/null +++ b/changelogs/unreleased/10255-Lyndon-Li @@ -0,0 +1 @@ +Use thread safe map for cancel recorder \ No newline at end of file diff --git a/changelogs/unreleased/10256-Lyndon-Li b/changelogs/unreleased/10256-Lyndon-Li new file mode 100644 index 000000000..c9fdb5779 --- /dev/null +++ b/changelogs/unreleased/10256-Lyndon-Li @@ -0,0 +1 @@ +Clarify the security context to Velero server \ No newline at end of file diff --git a/changelogs/unreleased/10258-Lyndon-Li b/changelogs/unreleased/10258-Lyndon-Li new file mode 100644 index 000000000..d73f8f4c6 --- /dev/null +++ b/changelogs/unreleased/10258-Lyndon-Li @@ -0,0 +1 @@ +Cap the unzip of metadata download to avoid OOM kill \ No newline at end of file diff --git a/changelogs/unreleased/10259-Jay2006sawant b/changelogs/unreleased/10259-Jay2006sawant new file mode 100644 index 000000000..ebe0ded02 --- /dev/null +++ b/changelogs/unreleased/10259-Jay2006sawant @@ -0,0 +1 @@ +Trim spaces around resource names in --ordered-resources so comma-separated lists with spaces still match diff --git a/changelogs/unreleased/10260-Lyndon-Li b/changelogs/unreleased/10260-Lyndon-Li new file mode 100644 index 000000000..418371f47 --- /dev/null +++ b/changelogs/unreleased/10260-Lyndon-Li @@ -0,0 +1 @@ +Add cap for backup data extraction \ No newline at end of file diff --git a/changelogs/unreleased/10269-blackpiglet b/changelogs/unreleased/10269-blackpiglet new file mode 100644 index 000000000..b23d58520 --- /dev/null +++ b/changelogs/unreleased/10269-blackpiglet @@ -0,0 +1 @@ +Remove PVC and PV inclusion check during creating PVR. \ No newline at end of file diff --git a/changelogs/unreleased/10270-Lyndon-Li b/changelogs/unreleased/10270-Lyndon-Li new file mode 100644 index 000000000..5bdc7e773 --- /dev/null +++ b/changelogs/unreleased/10270-Lyndon-Li @@ -0,0 +1 @@ +Cap the metadata decompression in object store \ No newline at end of file diff --git a/changelogs/unreleased/10279-harshitsaini17 b/changelogs/unreleased/10279-harshitsaini17 new file mode 100644 index 000000000..d524acdfa --- /dev/null +++ b/changelogs/unreleased/10279-harshitsaini17 @@ -0,0 +1 @@ +Use the well-known label constants exported by k8s.io/api/core/v1 instead of hardcoded label strings for kubernetes.io/hostname, kubernetes.io/os, kubernetes.io/arch, topology.kubernetes.io/zone and failure-domain.beta.kubernetes.io/zone diff --git a/changelogs/unreleased/10280-nitishmalang b/changelogs/unreleased/10280-nitishmalang new file mode 100644 index 000000000..f0a5e49b9 --- /dev/null +++ b/changelogs/unreleased/10280-nitishmalang @@ -0,0 +1 @@ +Bound WaitRestoreExecHook polling with resourceTimeout to avoid an infinite wait when restore exec hooks never complete. diff --git a/changelogs/unreleased/10283-opbot-xd b/changelogs/unreleased/10283-opbot-xd new file mode 100644 index 000000000..5f7c3aa48 --- /dev/null +++ b/changelogs/unreleased/10283-opbot-xd @@ -0,0 +1 @@ +test: add verification for skippedPVTracker in backup tests diff --git a/changelogs/unreleased/10292-samay43 b/changelogs/unreleased/10292-samay43 new file mode 100644 index 000000000..9b688ea86 --- /dev/null +++ b/changelogs/unreleased/10292-samay43 @@ -0,0 +1 @@ +Fix nil pointer dereference in EnsureDeleteVS and EnsureDeleteVSC when the API call times out before the object is retrieved diff --git a/changelogs/unreleased/10293-samay43 b/changelogs/unreleased/10293-samay43 new file mode 100644 index 000000000..eb5166687 --- /dev/null +++ b/changelogs/unreleased/10293-samay43 @@ -0,0 +1 @@ +Fix nil pointer dereference in EnsureDeletePVC, EnsureDeletePV and EnsureDeletePod when the API call times out before the object is retrieved diff --git a/changelogs/unreleased/10308-kaovilai b/changelogs/unreleased/10308-kaovilai new file mode 100644 index 000000000..ef832d521 --- /dev/null +++ b/changelogs/unreleased/10308-kaovilai @@ -0,0 +1 @@ +Fix block data mover cancellation being reported as a backup failure diff --git a/changelogs/unreleased/10309-kaovilai b/changelogs/unreleased/10309-kaovilai new file mode 100644 index 000000000..b0683330e --- /dev/null +++ b/changelogs/unreleased/10309-kaovilai @@ -0,0 +1 @@ +Report a measured zero-byte incremental instead of erasing it from status diff --git a/changelogs/unreleased/10312-samay43 b/changelogs/unreleased/10312-samay43 new file mode 100644 index 000000000..6b6cb3704 --- /dev/null +++ b/changelogs/unreleased/10312-samay43 @@ -0,0 +1 @@ +Assert expected errors from the test case rather than the returned error in pkg/util/csi tests diff --git a/changelogs/unreleased/10315-opbot-xd b/changelogs/unreleased/10315-opbot-xd new file mode 100644 index 000000000..9a0abe63d --- /dev/null +++ b/changelogs/unreleased/10315-opbot-xd @@ -0,0 +1 @@ +Testing: Implement missing unit tests for pkg/backup/snapshots.go diff --git a/changelogs/unreleased/10317-opbot-xd b/changelogs/unreleased/10317-opbot-xd new file mode 100644 index 000000000..0a18a2ec3 --- /dev/null +++ b/changelogs/unreleased/10317-opbot-xd @@ -0,0 +1 @@ +Cleanup: Remove deprecated --wait flag from velero uninstall diff --git a/changelogs/unreleased/10322-Lyndon-Li b/changelogs/unreleased/10322-Lyndon-Li new file mode 100644 index 000000000..e7f6baf9a --- /dev/null +++ b/changelogs/unreleased/10322-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #10321, when data mover pod is evicted get the message from the data mover pod instead of the terminal message \ No newline at end of file diff --git a/changelogs/unreleased/10329-lubronzhan b/changelogs/unreleased/10329-lubronzhan new file mode 100644 index 000000000..c24f6e0a2 --- /dev/null +++ b/changelogs/unreleased/10329-lubronzhan @@ -0,0 +1 @@ +Replace generated `config/crd/{v1,v2alpha1}/crds/crds.go` with `go:embed` of the CRD YAML bases, removing the codegen step and its CI drift check diff --git a/changelogs/unreleased/10333-reasonerjt b/changelogs/unreleased/10333-reasonerjt new file mode 100644 index 000000000..7cce9187b --- /dev/null +++ b/changelogs/unreleased/10333-reasonerjt @@ -0,0 +1 @@ +Enforce namespace of the "musthave" resources in restore \ No newline at end of file diff --git a/changelogs/unreleased/10338-blackpiglet b/changelogs/unreleased/10338-blackpiglet new file mode 100644 index 000000000..bf45dad9e --- /dev/null +++ b/changelogs/unreleased/10338-blackpiglet @@ -0,0 +1 @@ +Avoid duplicated InitContainer names generated in velero install CLI. \ No newline at end of file diff --git a/changelogs/unreleased/10339-shubham-pampattiwar b/changelogs/unreleased/10339-shubham-pampattiwar new file mode 100644 index 000000000..9f23a6668 --- /dev/null +++ b/changelogs/unreleased/10339-shubham-pampattiwar @@ -0,0 +1 @@ +Add readWriteOncePod backupPVC config to enable mount-level SELinux labeling diff --git a/changelogs/unreleased/10342-shubham-pampattiwar b/changelogs/unreleased/10342-shubham-pampattiwar new file mode 100644 index 000000000..adc848948 --- /dev/null +++ b/changelogs/unreleased/10342-shubham-pampattiwar @@ -0,0 +1 @@ +Fix issue #10341, avoid mutating the cached node-agent LoadAffinity so the OS node selector term is not appended repeatedly to data mover pods diff --git a/changelogs/unreleased/10343-chlins b/changelogs/unreleased/10343-chlins new file mode 100644 index 000000000..0c87d3e52 --- /dev/null +++ b/changelogs/unreleased/10343-chlins @@ -0,0 +1 @@ +Only sync finished backups from object storage diff --git a/changelogs/unreleased/10344-Lyndon-Li b/changelogs/unreleased/10344-Lyndon-Li new file mode 100644 index 000000000..ddd6aeeef --- /dev/null +++ b/changelogs/unreleased/10344-Lyndon-Li @@ -0,0 +1 @@ +Fix repo connection contest of the two repositories with the same storage type \ No newline at end of file diff --git a/changelogs/unreleased/10346-reasonerjt b/changelogs/unreleased/10346-reasonerjt new file mode 100644 index 000000000..eab333072 --- /dev/null +++ b/changelogs/unreleased/10346-reasonerjt @@ -0,0 +1 @@ +Double check the label for backup when deleting VSC- #10346 diff --git a/changelogs/unreleased/10352-samay43 b/changelogs/unreleased/10352-samay43 new file mode 100644 index 000000000..53ac1a9db --- /dev/null +++ b/changelogs/unreleased/10352-samay43 @@ -0,0 +1 @@ +Fix nil pointer dereference in WaitUntilVSCHandleIsReady when a VolumeSnapshotContent error has no message diff --git a/changelogs/unreleased/10357-samay43 b/changelogs/unreleased/10357-samay43 new file mode 100644 index 000000000..1c25341db --- /dev/null +++ b/changelogs/unreleased/10357-samay43 @@ -0,0 +1 @@ +translate parent snapshot "auto" to an empty parent snapshot in both data mover micro services diff --git a/changelogs/unreleased/10359-lubronzhan b/changelogs/unreleased/10359-lubronzhan new file mode 100644 index 000000000..b0696bb21 --- /dev/null +++ b/changelogs/unreleased/10359-lubronzhan @@ -0,0 +1 @@ +Fix e2e kind test matrix generation misparsing kindest/node pre-release tags (e.g. v1.37.0-rc.1) as bogus patch versions diff --git a/changelogs/unreleased/10363-samay43 b/changelogs/unreleased/10363-samay43 new file mode 100644 index 000000000..b562a4cdf --- /dev/null +++ b/changelogs/unreleased/10363-samay43 @@ -0,0 +1 @@ +stop routing credential selection on AZURE_USERNAME after username/password removal diff --git a/changelogs/unreleased/10370-samay43 b/changelogs/unreleased/10370-samay43 new file mode 100644 index 000000000..3ffb47ca0 --- /dev/null +++ b/changelogs/unreleased/10370-samay43 @@ -0,0 +1 @@ +fix log format string mismatches that produce wrong or mangled output diff --git a/changelogs/unreleased/10371-samay43 b/changelogs/unreleased/10371-samay43 new file mode 100644 index 000000000..d1ab7571e --- /dev/null +++ b/changelogs/unreleased/10371-samay43 @@ -0,0 +1 @@ +prevent panic when the restore hook init container command annotation is empty diff --git a/changelogs/unreleased/9697-Joeavaikath b/changelogs/unreleased/9697-Joeavaikath new file mode 100644 index 000000000..ad8e5eb2e --- /dev/null +++ b/changelogs/unreleased/9697-Joeavaikath @@ -0,0 +1 @@ +Fail backup validation when built-in data mover is requested but no node-agent pods are running diff --git a/changelogs/unreleased/9720-Joeavaikath b/changelogs/unreleased/9720-Joeavaikath new file mode 100644 index 000000000..cde7a017f --- /dev/null +++ b/changelogs/unreleased/9720-Joeavaikath @@ -0,0 +1 @@ +Add dynamic resource autocompletion to Velero CLI diff --git a/changelogs/unreleased/9773-emirot b/changelogs/unreleased/9773-emirot new file mode 100644 index 000000000..4c6f9f452 --- /dev/null +++ b/changelogs/unreleased/9773-emirot @@ -0,0 +1 @@ +docs(aws-plugin): update version diff --git a/changelogs/unreleased/9795-kaovilai b/changelogs/unreleased/9795-kaovilai new file mode 100644 index 000000000..6394ff4d3 --- /dev/null +++ b/changelogs/unreleased/9795-kaovilai @@ -0,0 +1 @@ +Skip DeleteSnapshot when ProviderSnapshotID is empty diff --git a/changelogs/unreleased/9920-shubham-pampattiwar b/changelogs/unreleased/9920-shubham-pampattiwar new file mode 100644 index 000000000..b9bda73f9 --- /dev/null +++ b/changelogs/unreleased/9920-shubham-pampattiwar @@ -0,0 +1 @@ +Support copying namespace-scoped secrets and configmaps for backup and restore PVC provisioning to enable datamover backup/restore of encrypted CSI volumes diff --git a/changelogs/unreleased/9952-alliasgher b/changelogs/unreleased/9952-alliasgher new file mode 100644 index 000000000..0f96ef5da --- /dev/null +++ b/changelogs/unreleased/9952-alliasgher @@ -0,0 +1 @@ +Fix e2e-test-kind workflow cache miss on force push by saving build artifacts explicitly instead of relying on the actions/cache post-job hook diff --git a/changelogs/unreleased/9967-adam-jian-zhang b/changelogs/unreleased/9967-adam-jian-zhang new file mode 100644 index 000000000..3bed73061 --- /dev/null +++ b/changelogs/unreleased/9967-adam-jian-zhang @@ -0,0 +1 @@ +Fix issue #9820, user guide for backup fine-grained filters via resource policy diff --git a/changelogs/unreleased/9974-blackpiglet b/changelogs/unreleased/9974-blackpiglet new file mode 100644 index 000000000..5a7d47668 --- /dev/null +++ b/changelogs/unreleased/9974-blackpiglet @@ -0,0 +1 @@ +Disable fips140 enforcement because Kopia doesn't support it. \ No newline at end of file diff --git a/changelogs/unreleased/9987-Shashank1306s b/changelogs/unreleased/9987-Shashank1306s new file mode 100644 index 000000000..4a975b5da --- /dev/null +++ b/changelogs/unreleased/9987-Shashank1306s @@ -0,0 +1 @@ +Fix ResourceDeletionStatusTracker key mismatch so restore into a terminating namespace waits once per namespace instead of once per resource diff --git a/config/crd/v1/bases/velero.io_backups.yaml b/config/crd/v1/bases/velero.io_backups.yaml index 68ec68c68..c20418c90 100644 --- a/config/crd/v1/bases/velero.io_backups.yaml +++ b/config/crd/v1/bases/velero.io_backups.yaml @@ -16,7 +16,27 @@ spec: singular: backup scope: Namespaced versions: - - name: v1 + - additionalPrinterColumns: + - description: Backup status such as New/InProgress + jsonPath: .status.phase + name: Status + type: string + - description: Total number of errors logged during the backup + jsonPath: .status.errors + name: Errors + type: integer + - description: Total number of warnings logged during the backup + jsonPath: .status.warnings + name: Warnings + type: integer + - description: The time the backup was started + jsonPath: .status.startTimestamp + name: Started + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 schema: openAPIV3Schema: description: |- @@ -59,7 +79,7 @@ spec: datamover: description: |- DataMover specifies the data mover to be used by the backup. - If DataMover is "" or "velero", the built-in data mover will be used. + If DataMover is "" or "velero", the default built-in data mover will be used. type: string defaultVolumesToFsBackup: description: |- @@ -393,6 +413,11 @@ spec: x-kubernetes-map-type: atomic metadata: properties: + annotations: + additionalProperties: + type: string + nullable: true + type: object labels: additionalProperties: type: string @@ -683,3 +708,4 @@ spec: type: object served: true storage: true + subresources: {} diff --git a/config/crd/v1/bases/velero.io_downloadrequests.yaml b/config/crd/v1/bases/velero.io_downloadrequests.yaml index 9db2e9fb8..771bf8daf 100644 --- a/config/crd/v1/bases/velero.io_downloadrequests.yaml +++ b/config/crd/v1/bases/velero.io_downloadrequests.yaml @@ -16,7 +16,23 @@ spec: singular: downloadrequest scope: Namespaced versions: - - name: v1 + - additionalPrinterColumns: + - description: The type of file to download + jsonPath: .spec.target.kind + name: Target Kind + type: string + - description: The name of the resource the file is associated with + jsonPath: .spec.target.name + name: Target Name + type: string + - description: The status of the download request + jsonPath: .status.phase + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 schema: openAPIV3Schema: description: |- @@ -79,8 +95,9 @@ spec: description: DownloadRequestStatus is the current status of a DownloadRequest. properties: downloadURL: - description: DownloadURL contains the pre-signed URL for the target - file. + description: |- + DownloadURL contains the pre-signed URL for the target file. It is signed for a fixed + lifetime and expires at Expiration, so it should be used promptly and not cached. type: string expiration: description: Expiration is when this DownloadRequest expires and can @@ -88,13 +105,24 @@ spec: format: date-time nullable: true type: string + message: + description: Message explains a Failed phase. It is empty in every + other phase. + type: string phase: - description: Phase is the current state of the DownloadRequest. + description: |- + Phase is the current state of the DownloadRequest. Processed means a URL has been + signed into DownloadURL. It does not mean the target object exists in object storage, + so a request whose target never produced a file still reaches Processed and the URL + returns 404. Callers should check that the backup or restore is in a phase that + produces the target before relying on the download. enum: - New - Processed + - Failed type: string type: object type: object served: true storage: true + subresources: {} diff --git a/config/crd/v1/bases/velero.io_podvolumebackups.yaml b/config/crd/v1/bases/velero.io_podvolumebackups.yaml index 3f4c83deb..935916db9 100644 --- a/config/crd/v1/bases/velero.io_podvolumebackups.yaml +++ b/config/crd/v1/bases/velero.io_podvolumebackups.yaml @@ -96,6 +96,13 @@ spec: description: Node is the name of the node that the Pod is running on. type: string + parentSnapshot: + description: |- + ParentSnapshot specifies the parent snapshot that current backup is based on. + If its value is "" or "auto", the data mover finds the recent backup of the same volume as parent. + If its value is "none", the data mover will do a full backup + If its value is a specific snapshotID, the data mover finds the specific snapshot as parent. + type: string pod: description: Pod is a reference to the pod containing the volume to be backed up. @@ -198,8 +205,12 @@ spec: nullable: true type: string incrementalBytes: - description: IncrementalBytes holds the number of bytes new or changed - since the last backup + description: |- + IncrementalBytes holds the number of bytes new or changed since the last backup. + A nil value means the uploader did not report a figure; a pointer to 0 means it + reported zero, i.e. nothing changed and nothing was transferred. The two are + distinct: erasing a measured zero makes a perfect incremental indistinguishable + from a full transfer in every downstream report. format: int64 type: integer message: diff --git a/config/crd/v1/bases/velero.io_podvolumerestores.yaml b/config/crd/v1/bases/velero.io_podvolumerestores.yaml index 015d143fe..2eea696c2 100644 --- a/config/crd/v1/bases/velero.io_podvolumerestores.yaml +++ b/config/crd/v1/bases/velero.io_podvolumerestores.yaml @@ -132,6 +132,9 @@ spec: repoIdentifier: description: RepoIdentifier is the backup repository identifier. type: string + restoreType: + description: RestoreType indicates the type of the restore. + type: string snapshotID: description: SnapshotID is the ID of the volume snapshot to be restored. type: string @@ -167,6 +170,7 @@ spec: - backupStorageLocation - pod - repoIdentifier + - restoreType - snapshotID - sourceNamespace - volume diff --git a/config/crd/v1/bases/velero.io_restores.yaml b/config/crd/v1/bases/velero.io_restores.yaml index 89f4baff8..b58666cca 100644 --- a/config/crd/v1/bases/velero.io_restores.yaml +++ b/config/crd/v1/bases/velero.io_restores.yaml @@ -16,7 +16,27 @@ spec: singular: restore scope: Namespaced versions: - - name: v1 + - additionalPrinterColumns: + - description: The name of the backup this restore is from + jsonPath: .spec.backupName + name: Backup + type: string + - description: Restore status such as New/InProgress + jsonPath: .status.phase + name: Status + type: string + - description: Total number of errors logged during the restore + jsonPath: .status.errors + name: Errors + type: integer + - description: Total number of warnings logged during the restore + jsonPath: .status.warnings + name: Warnings + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 schema: openAPIV3Schema: description: |- @@ -69,6 +89,11 @@ spec: for the Kubernetes resource to be restored nullable: true type: string + existingVolumeDataPolicy: + description: ExistingVolumeDataPolicy specifies the restore behavior + for the volume data to be restored + nullable: true + type: string hooks: description: Hooks represent custom behaviors that should be executed during or post restore. @@ -467,10 +492,24 @@ spec: from. If specified, and BackupName is empty, Velero will restore from the most recent successful backup created from this schedule. type: string + skipDefaultResourceModifier: + description: |- + SkipDefaultResourceModifier controls whether the server-configured default + resource modifier is applied to this restore. + When true, the default modifier is skipped even if configured on the server. + Has no effect when a per-restore ResourceModifier is specified. + nullable: true + type: boolean uploaderConfig: description: UploaderConfig specifies the configuration for the restore. nullable: true properties: + deleteExtraFiles: + description: |- + DeleteExtraFiles specifies whether to delete the extra files in the target volume that do not exist in the backup. + This setting is only applicable to File System restores (PodVolumeBackup or CSI File System Data Move) and has no effect on Block Data Move restores. + nullable: true + type: boolean parallelFilesDownload: description: ParallelFilesDownload is the concurrency number setting for restore. @@ -589,3 +628,4 @@ spec: type: object served: true storage: true + subresources: {} diff --git a/config/crd/v1/bases/velero.io_schedules.yaml b/config/crd/v1/bases/velero.io_schedules.yaml index 7ec1b6025..876cdf106 100644 --- a/config/crd/v1/bases/velero.io_schedules.yaml +++ b/config/crd/v1/bases/velero.io_schedules.yaml @@ -98,7 +98,7 @@ spec: datamover: description: |- DataMover specifies the data mover to be used by the backup. - If DataMover is "" or "velero", the built-in data mover will be used. + If DataMover is "" or "velero", the default built-in data mover will be used. type: string defaultVolumesToFsBackup: description: |- @@ -434,6 +434,11 @@ spec: x-kubernetes-map-type: atomic metadata: properties: + annotations: + additionalProperties: + type: string + nullable: true + type: object labels: additionalProperties: type: string diff --git a/config/crd/v1/bases/velero.io_serverstatusrequests.yaml b/config/crd/v1/bases/velero.io_serverstatusrequests.yaml index af35815fe..f7f6de9a4 100644 --- a/config/crd/v1/bases/velero.io_serverstatusrequests.yaml +++ b/config/crd/v1/bases/velero.io_serverstatusrequests.yaml @@ -16,7 +16,23 @@ spec: singular: serverstatusrequest scope: Namespaced versions: - - name: v1 + - additionalPrinterColumns: + - description: The status of the server status request + jsonPath: .status.phase + name: Status + type: string + - description: The Velero server version + jsonPath: .status.serverVersion + name: Server Version + type: string + - description: The time the request was processed by the controller + jsonPath: .status.processedTimestamp + name: Processed + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 schema: openAPIV3Schema: description: |- @@ -82,3 +98,4 @@ spec: type: object served: true storage: true + subresources: {} diff --git a/config/crd/v1/bases/velero.io_volumesnapshotlocations.yaml b/config/crd/v1/bases/velero.io_volumesnapshotlocations.yaml index 111a19df5..4fe7338ea 100644 --- a/config/crd/v1/bases/velero.io_volumesnapshotlocations.yaml +++ b/config/crd/v1/bases/velero.io_volumesnapshotlocations.yaml @@ -16,7 +16,19 @@ spec: singular: volumesnapshotlocation scope: Namespaced versions: - - name: v1 + - additionalPrinterColumns: + - description: Provider is the provider of the volume storage + jsonPath: .spec.provider + name: Provider + type: string + - description: Volume Snapshot Location status such as Available/Unavailable + jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 schema: openAPIV3Schema: description: VolumeSnapshotLocation is a location where Velero stores volume @@ -93,3 +105,4 @@ spec: type: object served: true storage: true + subresources: {} diff --git a/config/crd/v1/crds.go b/config/crd/v1/crds.go new file mode 100644 index 000000000..111b68cdc --- /dev/null +++ b/config/crd/v1/crds.go @@ -0,0 +1,58 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package crds embeds the controller-tools generated CRD manifests from +// ./bases into the binary via go:embed. +package crds + +import ( + "embed" + + apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/client-go/kubernetes/scheme" +) + +//go:embed bases/*.yaml +var basesFS embed.FS + +var CRDs = crds() + +func crds() []*apiextv1.CustomResourceDefinition { + apiextinstall.Install(scheme.Scheme) + decode := scheme.Codecs.UniversalDeserializer().Decode + + entries, err := basesFS.ReadDir("bases") + if err != nil { + panic(err) + } + + objs := make([]*apiextv1.CustomResourceDefinition, 0, len(entries)) + for _, entry := range entries { + data, err := basesFS.ReadFile("bases/" + entry.Name()) + if err != nil { + panic(err) + } + + obj, _, err := decode(data, nil, nil) + if err != nil { + panic(err) + } + objs = append(objs, obj.(*apiextv1.CustomResourceDefinition)) + } + + return objs +} diff --git a/config/crd/v1/crds/crds.go b/config/crd/v1/crds/crds.go deleted file mode 100644 index 5ecc27bcc..000000000 --- a/config/crd/v1/crds/crds.go +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by crds_generate.go; DO NOT EDIT. - -package crds - -import ( - "bytes" - "compress/gzip" - "io" - - apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" - apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/client-go/kubernetes/scheme" -) - -var rawCRDs = [][]byte{ - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xccW͎\xdb6\x10\xbe\xfb)\x06鵲\x1b\x14-\n\xdf\x12\xa7\x01\x82&\xc1\xc2\x0e\xf6NSc\x99Y\x8aTɡS\xf7\xe7\u074b!%[\x96hk\xbd\x87\xa2\xbc\x89\x9c\xffo\xe6#U\x14\xc5L4\xea\x11\x9dW\xd6,A4\n\xff 4\xfc\xe5\xe7O\xbf\xf8\xb9\xb2\x8b\xc3\xebٓ2\xe5\x12V\xc1\x93\xad\xd7\xe8mp\x12\xdf\xe1N\x19EʚY\x8d$JAb9\x03\x10\xc6X\x12\xbc\xed\xf9\x13@ZC\xcej\x8d\xae\xa8\xd0̟\xc2\x16\xb7A\xe9\x12]4\u07b9>\xfc0\x7f\xfd\xf3\xfc\xa7\x19\x80\x115.a+\xe4Sh\x1c6\xd6+\xb2N\xa1\x9f\x1fP\xa3\xb3seg\xbeA\xc9\xd6+gC\xb3\x84\xf3A\xd2n=\xa7\xa8\xdfFC\xeb\xce\xd01\x1ei\xe5\xe9\xb7\xec\xf1G\xe5)\x8a4:8\xa1s\x81\xc4c\xbf\xb7\x8e>\x9f\x9d\x15\xc0\"\xe9H\x99*h\xe1F\xba\xec\xdbK\xdb\xe0\x12\xa2f#$\x963\x80\xb6\b\xd1R\x01\xa2,cY\x85~p\xca\x10\xba\x95ա6'?_\xbd5\x0f\x82\xf6K\x98w\x85\x9fK\x87\xb1\xe6_T\x8d\x9eD\xddDٮ\x96o*l\xbf\xe9\xc8\xceKA86\xc6E\x9d\x9fc\xfdrl\xf0\xc2ʹF\xd0;K\x16=9e\xaa\xd9Y\xf8\xf0:\x95B\xee\xb1\x16\xcbV\xd66h\xde<|x\xfcqs\xb1\r\xd08۠#\xd5\x153\xad^g\xf6v\x01J\xf4ҩ\x86b\xdf\xfc]\\\x9c\x01\xb0\x83\xa4\x05%\xb7(z\xa0=v5Ʋ\x8d\t\xec\x0eh\xaf<\xc3\xe6УIM\xcb\xdb\u0080\xdd~EI\xf3\x81\xe9\r:6\xc3\xd8\a]rg\x1f\xd0\x118\x94\xb62\xeaϓm\x0fd\xa3S-\b=AD\xd1\b\r\a\xa1\x03~\x0f\u0094\x03˵8\x82C\xf6\t\xc1\xf4\xecE\x05?\x8c\xe3\x93u\b\xca\xec\xec\x12\xf6D\x8d_.\x16\x95\xa2n^\xa5\xad\xeb`\x14\x1d\x17q\xf4\xd46\x90u~Q\xe2\x01\xf5«\xaa\x10N\xee\x15\xa1\xa4\xe0p!\x1aU\xc4DL\x9c\xd9y]~\xe7\xda\t\xf7\x17nG@\xa7\x15\x87\xec\x0exx\xea@y\x10\xad\xa9\x94\xe2\x19\x05\xde\xe2ҭ\x7f\xdd|\x81.\x92\x84T\x02\xe5,:\xaaK\x87\x0fWS\x99\x1d\xba\xa4\xb7s\xb6\x8e6є\x8dU\x86\xe2\x87\xd4\n\r\x81\x0f\xdbZ\x11\xb7\xc1\xef\x01=1tC\xb3\xab\xc8i\xb0E\b\r\x8fN9\x14\xf8``%j\xd4+\xe1\xf1?ƊQ\xf1\x05\x83\xf0,\xb4\xfaL=\x14N\xe5\xed\x1dt,{\x05\xda!sn\x1a\x94\x8c,\x17\x97U\xd5N\xc94S;\xeb@\x8c\xe4/+\x95\xa7\x00^\x89D7d\x9d\xa8\xf0\xa3M6\x87BSm\xc7\xebm\xceP\x171\xd3V\xe2\x04\xcc\vf\f\xd2^P\x8f\fH(s\xe2\x94l\x927\x90\x89\xe8\bf\n#\x8c\xc4\xf7\xb1\x1f\x8d\x8a\x9f\xfd\x99\n?\xfc\xcb#з\xa0,\x18\xcb\xef\x8a\xe7VF\xf7\xa0,\x013\xb9\x15\xec\xef5lM\x8c\xc4N95\xa0-e\f(A9\xd9S^\xc1\xc2\x12\xe5\brA\x0fD\x81\xed\x93T\xa2\x05\x0f\x1b\xe8\xe3q\xfc,\x15\x10&6\xf2\x8a\xec\x8c)\xf5\xd5\xeb\xd7[f\x82\xe6e\xb2(*\xc1\xcc\xe15*\x11[WF*\xfd:\x87=\xf0ךm\x97Te;f \xb3l~MK\xb6DD\x04j\xdfe\x91\xff[\x10\x0f\xdd\xe9\xd6\x1c\xac\xd8j\xa3\x98ض>\xa0\xe6\xcc`\x8fU*'\x8c\x0e\x94C\xb1\xe1\x82\xfdɒ\xee\xf3\xfbۻ\xb6\xa02\xed\x99Ғ\xd7!\xfeXj2\xb1\x01\xe5\xdam\x94,\x10&\x88܉*\xca9g \f\xd1պ`Ɗ\xc1o\x15h\xab\x03\xf2\x18\xec5Z'\xb2\x06R\x95\xb9\x15\xe3\xe3\n+A\xaei\x01\xfc\x9ajxa^Y\xae\xe8\xa5eB\x12\xb7\xda6\xf7\xb8\xb2#o\xebC0\x9d\x03\xacu\x86嶄\xac\xa3h\xb6\x15۰̩\xd3F\xaa\xc6\xee8\x1bإP\\\xf5mq\xb5\xef\xec؎\xbeD\ab+\x86\xceA\x93\x9d|\f\xd6\xc6\"lE\xce\x02\x84\x9cT\xe5\x82<2\xb3\xeb\x01%\xa4\x94Z\xb35\a\xafw\x84\x89\x8cW\xb9\x15\xc9\x0f\x15\xe7h\xccV\"SPXs\xc1\x8fYM\b\x88\xaa\xe8\x0fv\x89\xad#?\xb7`\xf5\xbe\x0e0ЖL\xb3[AK\xbd\x93\xe6\x8e\x15 +3A\xa0\x9e\x12\xdar}\xbb:\x82Ң\x9ee$\x1a\xf3JCn\xad\xd9#e\x06\x99y}\xbb\"\xf7H\xd7\xd0\x1a\xady\xa5\x89\xa9\x94\xb0\xea\x13\xe9\xeb3\xd0\xfcp'\x7f\xd1@\xf2\n\xb5>S\x80\x02\xb2 k\xd8X\x15Q`\xdb\xdbO\xa0\x94\x15\x1a\x8d\x03\x90U\xcf\n\xdbr\xb7\x03+t\xb4\xe2\xc6\x1b\x10\xa6ɛ?\x93\x82\x89\xca\xf4tp\x94\x9aV:\n\xb9\au\n\x11\xdfQC\x7f\xb6\x8d\x8fh\x87\"\x87P-\xf1֞\x8e\xeb\x03~\x8c\xa9\x81+\xabM\v\"\xd3\xe4\xe2\x82HE.\x9c\xd3r\xb1p\xad+\xc6͒\x89v\x1f\x8f\x8c\xf3\xd0\xcb<\xe4\x1d\r\x1dC\xf5\x9d\xfc\xa0\x9d2\x9dD\x8b\x01X-\xd2<\xee\xc0\xec@\x91R֮\xc0\x86q \xfa\xa0\r\x14\x9e0az\xf5\xf8DzB\xa3¹\a\xa1-]=\"}\xe4E\xc59]s\xb8\"FU0@\x9b\xb5\x94\x1c\xa8\x98 \xcegІe\xe7 \x8d\x83\x14!\x8c\xf2\x1f:\x14@o\x82>\x00\xa1\x11Оf\xd6m\xe1\xbcE\xd8.U\xa2c*\x15dv:\xbb\xf2\xd3$\x03\x8eS\xb3\x90\x84K\xb1\x05\xe5z\xb7V/\b\x98\x02+p9\xb13\x90\x02n\xa7Y\xb2\xa9\xec\xe4tI\xacv\x0f\xca\x00\x13\xda\x00\x8d\b\xe7\x13\xf8\x03_\xacu\x86\xfc\xday\xa4\xb7ֱ\xce\xc3B\xa37\x9d\xa4\xf0\xe9\xfd(D\xef\xb6p\x96\xa1w\xec\x1d\xe1%:\xf411m\xbc\x17;5\xe1jò\xd2\x0f\xbbqKF\xed\x81\x06c\x1b]\xfc\xe9b\x81\x1c\xee\xf6\xda\xedC\x13\xaa\xa0&K\xb2݄\xa24\x87~mf\xa0\x88PqԞ$\xf2\x93*E\x0f\x03ܬ\x17Fg\xe4\xe7\x10\xcc#\x8e\x8aP\xed\x85yz\xdc\xef?3W\xcf\xc3G\x8d\x01\x02ʄ\xe5\x9f]\xabwا\xdd\xc2֒MH\x13\x81\xe7\xfc:\xc8q\xcd:\u00ad߉Xg\x91\xf9!!\xafe\xcb\v\xef?$\xa5vR>LQ\xe7\a[\xa7Y-\x92\f\x03Qd\r;\xbagRyԛ\xa9\x16\xbe@V\x99\xa8\xd6SCr\xb6ـ\xb2p\xca\x1dՠ]\xfc`\x98 \xc3\xeb\x1a\xd22#яGx4\x8c\xb4lḂ\x86n\xfd\x88\xe3Y2\x14;P\xeb^\xe3d\x9c\xb3=\xcb+\xcaq^\xa6\"s\xf8\xd0z\\1+3\xc2\xe4ޘ\xa3\x92\xe9\x8as\b\x02R\x96I\x9d%\xa4\x14`}\xde®\t\xfaU\x871_S\xeb\xab\xc8!\xec\t2KU\x1c\xb4\xef*G7\xb2\xb1\x19\x8b\x86)\x18\xa1!\x9c\xae\x81\x13\r\x1c2#U\x9c\"S|v%\xc5\b\x0e\x102b\xf9\xba+\x8d\x06\x81\x11\x90\x04\x97p;\x96휫g\x85\b\xe1\x90\\\x82u\xf8\f\xa1e\xc9#\xd3ESF\x99\xef;\x19\xd3\xf5\xa6Lh\xfd1\xbc\x98\xfe7%\xc1f6%J\xdaF\xbf\xba\x94\xad\xc5!\xbe\xa6m\xca?'a\x83\xe5?AhG\xb4\x9f`\xb8,Y\xa6\a\xe5\xd6R\x95\x81\xbe\xb4\xee\x14z:\v\xc2L\xf8uJ\x13:>W/\x8a\xd8!\xc2\xd7͛\xf9B\x9fȚ\x14\x9dx&\xc6\xd4]\xfc\x03\xf2\x05\xa7\x8c[?c$\xf3\xe4\xa7v\xab\x05a\x9b\x9a\xe8\xf9\x82l\x187\xa0\x8e\xa8\x7f\x92\xa9\x0f\x9c9\a1Rf=\x82\xfb\x1a&۽\xffb]0\xddl\xee%\xd2帱sd\x83\xb7ߝ\x9e'\xe0\x12\x8c\xef3\x17m\u0557\xb8bj\xff\x82\xae\xd5ۏ\xef\xe2\xeb\xabvI\x90\xbc\x1e\"\x13J\xe7\xca\xdb#\x8c\xda\xe3\xf3.|\xf8\x82>P\xbd\x00r\xb1\xea\x05\xa1\xe4\x01\x0e\xceu\xa1\x82X\xfe\xd0P9\xa1{\x05\xb8Y\x85r\xf6\x00\a\x04\x13\xdf}\xea\x97Tip\xe5\x01\x0e)Վhh\xc7Ĵ\xdfU\xb3t\xb2? !p\xd3!U\f\\\xf1\xaa\x10\xd9뉗D[\x12J\xa0\xfd\th&\x89J\xbb\x8f\xf6\xf6-J\xc0w\xda\xf1\xd2j̎\x95hV1\xe2 7\xc9\fu\xe5\x9er\x96\xd7\x1d9\x1dY\x89\x05\xf9(\x8d\xfd\xe7\xfd\x17\xa6\xfd\x0e\xef;\t\xfa\xa34\xf8˳P\xd4\r\xfc9\xe9\x19v|,B\xce\xca[\x82\xb5\xf7(ݜf\xa5\xad\xa6=\xd3d%\xecrő$\xb1+\u070evݹ\x8e\x8aJ\xe3\xf6\xa2\x90b\xe9\xc26\xb1\x9e<\xbd\xa5\xea\x90\xfbɝ\xfa\x0e\xef\xecdᾸMqN3\xc8\xc3v\r\xee\xd6R\x03[\x96%\xf6W\x80\xda\x02)\xad\tO\x93\x88D\xc3걙'>i\xb3w\xbb|Y>\xd4\xc9\x0fK;\xe5,=\x04#\x8b\x04\x1ax\u06ddO㳴:\x9bP+H\xc2dՁ\xcd\xdc\xe1\xaa)Dy\x029p\x16G\x17g\x92\xbb4\xcf15\x88\xf2\x9b\x193\xca\fY\x98k\x1aZcwSpAq\xab\xe5\x7f\xecL\x8b\xda\xf4\x7f\xa4\xa4L\xe9K\xf2\x16s}8t\xbe\xf9\xa0Y\vLB\x97\x98\xabc\xe5gO\xb9\x9d\xfb\xad\x01\x17\x04\xb8\xf3\x04\xe4\xa6\xe7\x17-\xc8\xe3Nj7mכ8\x17\x0fpp;\x86\x93]\xb6\x8d\xcc\xc5J\\8\x1f\xa2g0j\x87C\n~ \x17\xf8\xed\xe2)\xaeT\xa2\xa4&V\xeb\x88hA\xcb4\t\xc5\\\xabTG\xdd.X\x83\x13b\x1b\xd69D\xd6\xc9\x1e\xc36IDK\xa9#\x1b\xf9\x03C\x99\x10\xde\x1b\xa9\x8d\x8b\x97u|\xe6h@M\x86 \x1a\xa1\x1b\x97\xd8%U\xc8±Fy*\xf4\xdb.w;\xd0\xe0\xf7+|`\xce\x01\xb5+\xbb\x8bF\xbf\x9d\xb5\xbfp\xfb%\xd8\t\xcd\xd0c\xc1\xb6\xa5\x92\x19\xe8\xe8^vS\x12\xe6\x8bH\x96H\x1b\xf7:\xe6H\xdd*\xc9媌\x87@CIwy-!f\xae\x17\xde\x7fi\x05D\xad\xeeۿ\xa7dl\xee\xb8\bfY\x16\x05=\xce\xdfJ\x1a\xe2\xb5k\x19\xb4\xc1\x03r\x8b\x0f\xb5\xad\xd0\x12\xa4\xce\xe5\xb5\x00~\r\x8eB\xc1\xc4\n; o\x9e\xc1\xb1\xf064\x96l\x12+\xa7\xb9\xb2ס\x93\x86;\xf5\x0fN\x95K\x89[\x05\n:\xcc\xebG\xd5\xd1\x0f\x15Ҵ\x02\x123\xdc\xcdR\xe6\xdfi\xb2aJ\x9b\xf6\x10\xf4@\x9aJ\x14\xcc̅\x97x\xaf\xd4I\xeb\xaeO\xae\xe5Q\x02\x99\xcf[s\x84I\xc4\x1c\xf7\x97\x80\xb0\ra\x86\x80\xc8d%0\x80c\xf5\x18\xbbp\xc4u\x16\x96\xa5*I\x9a\xf6\x93\xc1\x1c\xb4XY\xa2\xa401\x1a\xe9iW\xff@Y?Q-Vf\xb2\xcd\fe\xb1\xc5\xcai:\x11R\xdcڙ\x8a\x05\xfd\u008a\xaa \xb4\xb0<\xc2ɜ\x15\xd0ez\x93\xf8f[\xe04a\xa4\u0558\x92\x83\x01\x9f\xbc\x968\x86L\n\xcdr\xa8'W/\bR\x10J6\x94\xf1J%Z\xc0Y䝳\x14\xf1\x96\xe0|k\x8c\xb4ΗH\x8a\x84hn\xa2\xaf8n\x8dK\x95\xee\xf1M\xb9Y\n\xe6{Y\xa5b\x12\xd3\x02\xcf\xech\xf9DJ*\x0e\xdf<\xadԡ~\xf3\xb4\xc6\xca7Ok\xa2|\xf3\xb4\xbeyZ)5\xbfyZ\xdf<\xadv\xf9\x97\xf0\xb4\xa6F\xe4\x0e:\x0e|\x9c\x1cE\xc2V\xf5\xd8\x10G\xe0\xfb\xe4\n\x9f\x03\xfe\xa4\\\xccU\x1cT$\xf1\x7f \xad;f\xb4\x9aɣNδZ\x13dޝ\xbb\x9ap%\x9f\x90u\x1f:=_\xd6\xfdj\x14♲\xee\xfd\xb0\xa7}\xec\x93r\xee\x03Q\xe6eg/|\xa2F\x014\x84\xd5\xdd6|\f\xaf!\t\x99\xe8\xff\x85\x13s{Ycg\x94\x8fg\xcf\xe2O\x96\x91(K/\xfet\xf1\xf5\x91\xff<\x04\x1f$q\x9fv\xfe\xe0w\x04\xaa]\x81\xb6\xd3ºYx_\xa7\x18\x9fEnS3\xf1k\"F`uE\xf2\x88\x8a_\xab-0P|*\xfd\x8c\U001044ea\xab\b\x9c\xa4\xb3\xaaT\x1fD\xb6SR\xc8J\xfb\xa8\x84\x85\xf56s'\xfd\x03Ș\xb0F5\xfc?\xc8NV\x91L\xf0\x11\xf2Md\x04N#\xdfI\x0e\xf4\x9b\xd0`\xe8\xfe\xcde\xf7\x8b\x91>Up\xe8l\xf3\xe3\x0e\x04\uec0bm\xfb\x00@\xb8\xa8\xc1\xdfXp,`\x11@R\x11\xc1\xb8\x93\xbc\xfa\x9a\x87\xb6ܑO\xa5\x8b=\xcd\xf6;\xc6c*iɄ'\xa7\x10vS\x04\a\xfcҹ\xbb\xddg92\xf1\xbb\xa4\x06\xceO\bL\x89\x88M$\xff\x9d\x90\xf2\x97\x98[\xfc\xe4\xed\xf9\x94\xa4\xbe9+\xe6gK\xe0;\x7f\xda^\x12}\xa6S\xf4\xe6P\xe7\xd9\xd3\xf1^0\t\xefeR\xef\x12\x13\xeeΗ9\x9f\x16\x8f=)sl:t0\x9c47\x99*7\x19Z\x98Bl6J\x93)ps\x12\xdf&\xb9\x93\xa6f/\x96\xda\xf6b\tm/\x9b\xc66*E\xa3\x1f\xe7$\xaa\xc5\xef\xeb!\x93\x93-\x7f)a;\x95\fRu\xdcד\xd6W\x9f\x8e`X\xc6\a\xd7\xee\x85|\xe4\xa2↕\x1c7R\xf7,\x8f\x06\x1b\xcc\x0e\x0e\xf5\x05\x1a\xbfJ\xf53!Y\x13\x9f\xe7\x9d\xee9y\xcbB\xaa\x1c\xd4\xe8\xb6O\xaa\x14\x8e\xca_\xcaڦ;\x90\xa3\xfd\x8ep럭\xd5\xf1\x97qz\xf07\xb0\xe2]\xbbCۗV\xd2Z\xdeFg/\xaaq\x7f\xbaΤ\xbf\x80\xd7mWi(\xa9\xc2K\x9d\xd7\a\x97\xce\x12\x9d\x9a\xdf\xd3lw\x04}G5\xd9HUPC.\xea\r\xc0\xd7\x0e\xb8\xfd\xfb⒐\x0f\xb2Ήh\xdfˣYQ\xf2\x83]\xa1\x90\x8bv\x83\xd3$ *m\xa1\xb7\x1b\xc9Y\x16\xf1ݢw3\xb9ʽ\xcb2\xf0ƨ\xac\x9d2Pڊq\xd7\rݼ\xee\x15\x98\x1bɹ|\x9c\xb9\xf6\xa7%\xfb\v^v\xfe\x84\xe8\xd0ۛ\x15\xc2\b⁷\xa7\xd7\xc9Y56k\xb0\xd3r\x83\xe7\x90\xee\xaf6\x1d\x88\xdd<\xc7\xf6\xad\xc1\x90\xbb\v\xa2\x83[\xe0Mg&\xadu\xb9Y\xb9q\f\xf5be\x86\x8a\x03\x91\x98QcvL\xe5˒*sp\x89\x1a\x8b\xce\x18\xc2\\:\x16\xdd\x19\x9c=\xfa\x97^G\xc9\x1b\xee\xba\xc6\x1d\xcaC\xd9\xdd\xf4=\xa6\xdd)\xe3\x18>\xbd8yn\xf1\x8c\xe3\x18vK\x96H\xa9\xc8\xcf\xd1̯\xb3Eʹ\xbf\x99\xf8g\xb9\x87w\xd1\xe8Y\x87<\xb7G\xd5#\xe9Y\x01\xa2\xbbtw0Ku\rx!o\xff\xd3\x13\xf2\xadB\xd7\xfeN\xd5S\x02e\xb7]\x10\x11\xfc\xc2\r\xb3\xa1\xb3\x98}\u009b\xf1\x0f\xe4\xe6\x1e\xd7h\xb5i\xf3*\xea\xd7h!T\x166\x83#p|\x83\xefϟ\x9a\xa6\x8dTt\v?Iw\xf9\xf8\x14ۻ\xb5;\x97\xd2{\xaf'\xe4\x8f\x06\xa5\x89]\xc0\xeb\xafA?\x02\xd6\xe4|\xf7.5\xb6\xa3\x9cyM\xb31\xfc\x14\xbe\xdf\xdd\xfd\xe4\xb02\xac\x80\xcbw\x95Kw\xb06Q\x83%q\xc0\xd6AZ\xdb\xff\xee\xe4#^\xfe\x1b\x8fc\x86\xc7$\x1ad\x14`\xb29\xa6 \xceB\xa9*\xb9\xa49\xa8k)6l;\x81\xdd/\x9d\xcaG\xd3l\x86?z\xe4\xea9*\xc0?s\x0e\x82\xf5y8\a\xfe\x81q\xd0nX\t\x06\xf8\xa6ߪ\xb6\xc7U\xb1v>\xdc\xc6~\xac;\x18\x98\xe3\x1cZ\x18\x8a.AY/\xca\x05\xad+\x1ddu\x18\xf1\x86#L\x18\xd8B\x7f\x158b\x81ݭ\xd28}\x06s\x82k\x99\x1fc\xf1\xad\x0e\xf2\xf7\xc3-\x8f8\xd9\ny\xc5n\xdcsN\xc8\xcd\xfd\xb5&\x95\xc81\\|\xff\x97\xdbYR\xb7\xef\xdc\\\x1f\xb4uʨ\xde\xc7[\xb5\x9c㖽pޱ\xdcD\x10\x18\x82\xd3z \xe5\x91\x19\x7fq\xd7yoZ\x1dZ\xf2\f=\xfd\x80W\xfaO?\xfe\xe0n\xfe\xf7O\xc6xu\xac\x14^\x93\xea_\x05\xc0kE\x9f\xf0\xfeC'\xf9K\xbf5\x06\x8a\xd2\xc4|\x8dis\xf8\xfd\x18\xc0\xdaO\x93\x86\xf2\x96V\xd2P!\xe6i\xeb\x83\xc8\xc6\x12˼5\x1a\xe1\xe6\x98>\xc6\bp\xed\xcfC\x9c\x8d\x005\xc0!\x02\xe8*\xcb@\xebM\xc5\xf9\xa1>\x8e\xf1\x95P\xe3\x03e\xfc|\xa4p\xd0\x06\x05\xc1\xa27\ni\x12a\x9f\xee\r\"\x0f\x9a\x1e\x8e*\xcd#\x85\xe7\x82φԆ\x16'=\xd8p\xdd\a\x83o\x19\xa9\xbc\x95TI\xeb\xb1Sݰ?6\xb94\xe0\\K\\dYh\x90\x13\u0603 vvv$\x0e\xcfẗ́\xe2O\xb8\xba\x19.\xccw!\x14\x12}\xb1\x89\xf8h\x87Ɨ\x81\xbe\xd35L\xcc\x15\xc5\xf7L\xfaD\xe8;\xbf.Zqe\xbd\x7fXZ\x10\xa7y\xadC\xaf\xb9t照\x19\xb9\xeb\xdb\xd5\x10\xb8SL\\\xff\xb9\x97'\xaaq\x1f\xdd'\x99\xb4>\xba\xb3\fZ\x04b-\xe3\xe7\xc7\x1dU\xfd\xb4Kݱ\xa5s8\xb2p\x86\x8er\xee\x0f:\x16\xa05݆\xdb\xdc\x1f\xed\xd2c\v\x02\\x\xcem\x9eD\x806\xa7\xe2\xbaw\x99;\x95\xa1\x99\xa9\xa8\xef $\xf8\xb6j}\xa7\t\x971\xa8\xf8\xa0\v\vO\xa8\x855\xd9LB})\x99JYý\xaf+Zڠ'\x8c\xdci\x1e\xbd\x03ζ\xf8\xa4\x93\xe5ܖ\xaa5\xdd\xc22\x93\x9c\x03Z\xeb\xfe\xb8\x9eS\xd7\xfd\xd9\xc3\xcf@\xf5$j\x1f\xdau\xfd\x0e\xa0\xe3\xb6\xdb\xf8\xa6.\xdd\x1d\x9f53LA\xf3\xc2`o@\x12;\x9e\xe5(;*D\x9f\xdf돴]7h\x9d7\xcb>\xce\xeb_\xdf[4/jE\xc6Y\xd0_\xa5Z\x90\x82\t\xfb\x0f\x15\xb9\xdb\xc0\v\x8dg\x8d\x7f'\xe5\xc3mĉ\xed\r\xfe\x87\xbab\xb3\xd5\xc1\x84\x1b6\x1e\x18]\xcb\xca\xef\xbe\xd7\x0em|[\x05o\xe6?\xf3r\x13a\x8e\xcc\a=t\x06#\xba?t MN\x05\xae\xe7\x01X\xb7\xe1\x897\xce\x0f\x8bc\xc8G\xcfI6\xb0[/\x17x7\xa0\xb9\x8f`\xa0\xa3\xb0#\x15\x05R_|\xd16觬z=\x99\x87\x9c\xc9\x1e\x8d\x7fhj\x0f\xd1\xd1\r\xb3\xe5\xee\r \xd8q\x02ϻ`\xc7g*&\x84\xff\xc6֩\xef.h-\xdcB\x96\xd8`\x94n襻\x8f\xd0߮X\x92\xbfVPEh\xb0\f\x0f\xc3\xdd\x1a\xaa\xfa!_w\f\x1er\xcc\xe8@m\x8cTY\x89\x1b%\xb7\nt_X\x97\xe4o\x94\x19&\xb6\x1f\xa4\xba\xe1Ֆ\x89O\xc3G~\xc6*\xdfPe\x98\x15v7\x9e\xd8@\x99\xa0\x9c\xfd=f\xd7\xda\x1f\xa7\x01]\x0f.\xb0\x96$a\x18C\x1fށ\xf5q\a\xe3\x02Q\x13Zz\xba\x9e\xe2\xaf\x04\x9eL\xd9\xd4ڗh|\x91\xd0\xed%\xf9(\xa3\x86\xc1\xa7C\xb1.L뒁6K\xd8l\xa42n\xb7z\xb9$l\x13\x82\x0f\xd6\xe6`\xdc\xcc=\xe2IXl\x9b\xb9N4i\xa6/\fz+\x9c\x85\xf1*\xfb\x82\x1e\xdc\xce\x14Ͳ\xcazX\xaf\xb5\xa1<\xe2\xe0<\xc9\xf0c\x94\xe7{|\xb0\xf2\x97'\xed\xe4\xadڀ\xfaAG\xecǑ\x14/\xd3p^\x1f\xb7(\x82 \x8f\x8a\x19c}*9\x92J\xe0Ie\xaco\xc59і\xd4'E\x1f\x893\xa3\xabᔜ4\x94\xefj(C\xe6\xd9c\x8d/3֯\x82\xfa\xec#_˲9\xdbQ\xb1\x1d\xbc\xa1`\xa7d\xb5\xdd\x05I\x1ep\xa6I^\x01\x06kѤ\xe8\xf0ⲩ\x94h\xa5\x12\x8c\x1c\xfb&A\x18p\xb84{\xc0\xf7K\u074b\xc6\xfe)\xeb\xd7\xfe\r\x94\xe5F\xc9b\xe9\xfb\xc5X\xea\xc2\xef\xe4+&\xad\xe7bvQ\xaa\x13\xe7\xb5\xfbg\x06P\x12\xca\x12\x04\xa1\xda\xf7\x9cpS\xd4\xc9\xd3\xd4ovj\xb8\x91\x9a%x\xfbQ\x8e\xff\xb5\r 0\xbc\f\x7fw\x99\xe1W0\xd8g\f\x8fO\xfe\b>\xec\xa90n9QO\x91\x17n\x12\xbb\x98\xb5\x90\xd1vb{R\x90\xe6\xb6\x03a\">\x83\xdd\xc5Yt\xeb\xd35\xdcE`\xd7\xfe\xf9\xd5\x1a\xf0\x82h&\u008b\xe0.\xf5\xc3I\x7ft'P\xe0C\x95Rų1\xc7\x03.]\x84^6ֲ\xaf=\x89\xf7'/\xc5\xef\x8f`\x1c\x1d\xea\xc6wI\xeb*a\xf9\xfc\a\x16\xdb\x0f\xc04\xde̢\xf2\xc7\xdf\xfd\xb0\xf6>i\xa9\x17\xa7\xc8\xd8\xca\x0f\x17u\xc3K\xb8\xee;\xa47\x1c\xac\xb6i\x80\xee\xa2r\x96\xce\xed\xcf\x18M;g(-\xbc}\x7f\x9eX\xd2\xfe\x8cA\xb4g\x8b\xa0\x9d\x17\xe5G\x8a\x0fD\x9f\xa4\xb5\x7f\xf3m#!4\x0f\xf6\xdcA\xb4V\f-\f\xfcE\xa3h\xd19\xb7\xf7#\xda\xe9\xbce-|O\xfe\x97\xff\x0f\x00\x00\xff\xff9i\xfd\xfe\xeb\x83\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xccZK\x93۸\x11\xbe\xebWt\xed\x1e\xf6\xb2\x94줒J\xe96\x96\x93*W\xc6\xf1\xd4hvr]\bhJ\xb0@\x80\x01@\xc9\xca㿧\x1a\x0f\x89\xe2C\x0f;q\u008b-\x12h\xf4\xf3\xeb\x0f\xc0\x14E1a\xb5|E\xeb\xa4\xd1s`\xb5\xc4/\x1e5\xfdr\xd3\xed\x1f\xdcT\x9a\xd9\xee\xedd+\xb5\x98âq\xdeT\xcf\xe8Lc9\xbe\xc7Rj\xe9\xa5ѓ\n=\x13̳\xf9\x04\x80im<\xa3\u05ce~\x02p\xa3\xbd5J\xa1-֨\xa7\xdbf\x85\xabF*\x816\b\xcfK\xef\xdeL\xdf\xfe~\xfa\xbb\t\x80f\x15\xcea\xc5\xf8\xb6\xa9\x9d7\x96\xadQ\x19\x1eENw\xa8К\xa94\x13W#\xa7\x15\xd6\xd64\xf5\x1cN\x1f\xa2\x84\xb4z\xd4\xfc]\x10\xb6\x8c\xc2\x1e\x93\xb0\xf0]I\xe7\xff<>\xe6Q:\x1f\xc6ժ\xb1L\x8d\xa9\x15\x86\xb8\x8d\xb1\xfe/\xa7\xa5\vX9\x15\xbfH\xbdn\x14\xb3#\xd3'\x00\x8e\x9b\x1a\xe7\x10f\u05cc\xa3\x98\x00$\xd7\x04i\x050!\x82\xb3\x99z\xb2R{\xb4\v\xa3\x9aJ\x1f\xd7\x12踕\xb5\x0fΌ\xb6@2\x06\xb25\xe0<\xf3\x8d\x03\xd7\xf0\r0\a\x0f;&\x15[)\x9c\xfd\xa2Y\xfe\x7f\x90\a\xf0\xd9\x19\xfd\xc4\xfcf\x0e\xd38kZo\x98\xcb_c\x8c\x9eZo\xfc\x81\fp\xdeJ\xbd\x1eR\xe9\x919\xffʔ\x14A\x93\x17Y!H\a~\x83\xa0\x98\xf3\xe0\xe9\x05\xfd\x8a\x1e\x02r\x11B\xf6\x10\xec\x99K\xeb\x00좔\xe0\xa3aMUo\xad3\xb5I\x15x\xedH\x89\xfaӛ\xa4}Kl\xce\xef)\xb7x\x14\xe9<\xab\xea3\xb9\x0fk\x1c\x13v\xe6\x8a\xf7X\xb2F\xf9\xb6\xa9\x14%\xd5\xce\xcbs\xb3j\xe4S\x11g\x9d\xad\xf8\xfe\xec]\\ue\x8cB\x16\xa5\xc4Q\xbb\xb71\v\xf9\x06+6O\x83M\x8d\xfa\xe1\xe9\xc3\xebo\x97g\xafa(\x91:EA\x81c\xad\xd8l\xd0\"\xbc\x86\xfa\x8bqsɴ\xa3L\x00\xb3\xfa\x8cܟ\x82X[S\xa3\xf52\x17K|ZX\xd4z\xdb\xd1\xe9\x9f\xc5\xd97\x002#\xce\x02A\xa0\x841\xafR\xfd\xa0H\x96\x83)\xc1o\xa4\x03\x8b\xb5E\x87:\xc2\x14\xbdf:)8\xed\x88^\xa2%1Tۍ\x12\x84e;\xb4\x1e,r\xb3\xd6\xf2\xefG\xd9\x0e\xbcI\xc9\xec\xd1y\b\x15\xaa\x99\xa2dm\xf0g`Zt$W\xec\x00\x16iMhtK^\x98\xe0\xbaz|\xa4j\x90\xba4s\xd8x_\xbb\xf9l\xb6\x96>#47U\xd5h\xe9\x0f\xb3\x00\xb6r\xd5xc\xddL\xe0\x0e\xd5\xcc\xc9u\xc1,\xdfH\x8f\xdc7\x16g\xac\x96E0DGH\xadď6a\xba;[\xb6W\xd2\xf1\t\x90zGx\b^c\xcaDQ\xd1\xc4S\x14\xe8\x15\xb9\xee\xf9\x8f\xcb\x17Ț\xc4HŠ\x9c\x86\xf6\xfc\x92\xe3Cޔ\xbaD\x1b\xe7\x95\xd6TA&jQ\x1b\xa9}\xf8\xc1\x95D\xed\xc15\xabJzJ\x83\xbf5\xe8<\x85\xae+v\x11\xba\x18\xac\x10\x9a:\x80Dw\xc0\a\r\vV\xa1Z0\x87\xdf9V\x14\x15WP\x10n\x8aV\xbb7w\aG\xf7\xb6>\xe4\x9e:\x12\xdaA4X\xd6\xc8\xcf\xeaN\xa0\x93\x96*\xc33\x8f\xa1\xba:\x0eJP1ޔ\xf33\f\x12\xf40\xceѹ\x8fF`\xf7KG\xe5\x87\xe3\xc03\x1dk\xb4\x95t\xa1\xbdBil\xb7\xf3\xb0#\x92\xb7\x9f\x8cx݀\x03\xa0n\xaa\xbe\"\x05<#\x13\x9f\xb4:\x8c|\xfa\xab\x95\xbe\xbf\xd0H \xe9\x89*.\x0f\x9a?\xa1\x95F\\1\xfe]g\xf8\xd1\x05\x1b\xb3\x872\xe4\xbf\xf6\xea@\xd8\xe5\x0e\x9a\xf7Q;?\x0fO\x1f2\x82\xc7\xdaJ\x85\x99|5\x85\x87TԦ\x847 \xa4#\"\xe1\x82о\xb3t\xa3\x02ј\x83\xb7\xcd]\xe6s\xa3K\xb9\xee\x1b\xdd\xe6Fc\x19sEt\xc7s\x8b\xb0\x12\xa1\x16eGm\xcdN\n\xb4\x05Շ,%O\x9a46v\x90R\xa2\x12=l\x1a\xad\xb2`\x8aEAE\xcdԕ\x18.\x8e\x03\x03\x93fR\xc7\f>\t\bXc\xabԚ\xb5G-\xb0\xdbm\x826&\x00\x9aC\x01{\xe97\x11)\xd5P\xdd\xc1\xc5ڣg\x8b\x87\xa1\xd7\x1d\xdd_6H#c\xe3Ep\xc8-\xfa\x90m\xa8(}(\x95\xa6\x00\x1f\x1b\x17\xb0\xb6\x8b\x13\xf9\t\x84/\xcf\xde\xe2\xa1\xefh\xb8\x16\xdcD\x85FT\x0e$j\x0e?\xfcpݤ^w\xcb\x0fQ\xf7l\xa8\xc5\x12-\xea\x1e\x9b\xc8\xcfK\xe8Q\x944\x94aX\x96Ƚܡ:\x84\x9eD\xe0\xf93\xac\x1a\x0f\xa2\xc1\x105Ʒ{f\x85\x03n\xaa\x9ay\xb9\x92J\xfa\x03H7\"\x9f)e\xf6(Rı\xaa\xfda\n\x1f\xb4\xf3LstG\x1eD\x1e\x8b\xa9\xc0t\x1c\x95\xaa8\x10:f\x8700\x8a\xaf\x8c\xf3\xc0\xd1R:\xaa\x03\xec\xad\xd1\xeb1c\a\xda!\xed\x01\xadF\x8f\xa1#\n\xc3\x1d5C\x8e\xb5w3\xb3C\xbb\x93\xb8\x9f\xed\x8d\xddJ\xbd.H\xc1\"\x81\xcf,\xec\xecf?\x86\x7f\xbe&\vL\x1dq\xe2\x86\xe4]\x86Z?\x10\xbd\xf5\x1b\x8c-b\x19s\xd0X \x02A\xa9]\xa5܍\xc8:TvC\xbc\xbc\xfd\xe4\x90\x0f\xf5\x8f-\xf6[\xc7\x05P\x01\xf8R\x9c|[T\xac.\xe2h\xe6M%\xf9\xa4km\xcc\xfb\xcb\xf8\x937+R\vɉܞ\xe3F\xdeĉ\xb3=̀\x1b\xba\xbb\x9c1\xb4\x1cvS47q\x85+\x1a\x7fj\x8f=m}#t\xa7\xfe\xef\xd0\x13\xeft\xa0\x91\xf8\x01\xb3}?\a\xc0\xe4FkB*o\x80\x1d\xdb\xc0O\xae\xdb\xff\xeeD\xcfU÷8\xe0\xf8\x9e)\xef\xc2\xc0\xec\xe38\x8dti\x1c\x86\xc6tM\r\xb8^\x11\x9c-\xd0ޢ\xcb(\xf2-\x1eH\u0091[0X<\xc0\xaa\xd1BaVu\xbfAM\xdb1Y\x1e\x88\xec\xbf<.\xb3c\x03\x01K[\xa7\xec\xde1 yO\xbb\x00JA1\x87_\x1c\xa6u\x9f\xb1\x04\xa9\x9dG\xd6#\xe9\xf1\x89\xbdq\x0e\xab\xc3\x00\u05fa\xd9A\xcfX~\xbb\x8f\x82\xae\xe4\xa1\xd4 8\xc6\xc4J\xb0\x92\xfa{\xde\x0f-\x1e\x02\xc4\x12\xdf \"}\xe6ґe\xeetth\xd0i\xf1\fdR\xc7\x02a\xd5\xd8\":\x1fR\x01\x8by9H䇃q\xb9.\xe0\x12\xb3\xe89\xfb>v1*\x13\x80\xdd\xc80\xe0z\xb2\xc0E\xa6\x017\xb0\x8d\x9e\x99\xa39\x05w\xb2\x0e\xf8\x0e\xcc\x03\xfe\xfb\xec\x03\xeef \xf0\xddY\bܖ)\x97\xd9\b|\x13#\xb9\xe0\x8bK\\\x05\xae\xf2\x15\xb8\xc8Y`\x94\xb7\xc05\xee\x02w\xf2\x17\bx\x82\xa5\xfcr\x032?\x85\x81\xb9\x93\xd6\xcco\xa8kH\x81\xc0\x06\xfaj<\xa1\x18q\xd0q\xd3\xfb)\x85\xef+\xfa\xee%\xd2\x17չ\x87\xf7e@\xbfB\x8c\x9eҰ\xa3\x17\xf2\xef\x04 \xe7\a c\x04mТ\xdd\xf1\xb4\xfdO\xf1X\x81\x0f\xa0\xf8\x992\xaf\xfd\x19\x17\x8e'\xf2\x99\xff\x10K\xa3Ͱ\xb1\x16]m\xb4\xa0\xb6w\xdb\xe1\xc4I\xe5\xff\xdc\x11\xc5pX\x8bs\xfa\xda\xf9\x96\xa3p\xd3\xf9\\\xb8߸\xfb\x84.\xde\xfa\xb4Ͽ\xccʡݵ\x0e\xe9:6~\x97\xb3\xb9\xc1\xce\xd6:\xb0#\xaa\xa4\xa1\xd1\xe1\xc8\"4\xad\xe9d`F\x9b\x17\xfa\xd0<\xa4\x03m\xf64\xb9%-v=\x13\xe9M8\xb4dZ\xa4\xe3b\xfa4 y/\x95\xa2\x1ef\xb12\xe4,\xd4^Zj\x96,\xb4\xb1\xddo\xa6o\xfewg\x81\x8a9\xbf\xc6Qq\xff\x99\xa6\x00[\x99\xc6\x0f\xf4\xfeV\xc2\x0f\xd6t\xb8e\xbfG\xc7\xf0\xb7\x03\xd7\xe8\t\x8d\xc9\x11፵\xe1\xb2._\"ݱ\xd1\x1cC\xe0\x87Ο8\xb4\xbf\xf5\xff\x00\xe2\x06\xbb\x06\xbbt\xefe촭\xb8&'\xb7\xdf4\xab\xe3\x15\xec\x1c\xfe\xf1\xafɿ\x03\x00\x00\xff\xff%\xff\\)\x99#\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcVK\x8f\xdb6\x10\xbe\xfbW\f\xd0k%wQ\xb4(tk69,\xda\x06\xc6n\x90;M\x8e-f)\x92\x9d\x19z\xeb>\xfe{AR\xf2C\xb67\xc9%\xba\x99\xf3\xe4\xf7\xcd|t\xd34\v\x15\xedG$\xb6\xc1w\xa0\xa2ſ\x04}\xfe\xc5\xed\xf3/\xdcڰ\xdc\xdd-\x9e\xad7\x1d\xdc'\x960<\"\x87D\x1a\xdf\xe2\xc6z+6\xf8ŀ\xa2\x8c\x12\xd5-\x00\x94\xf7AT>\xe6\xfc\x13@\a/\x14\x9cCj\xb6\xe8\xdb\xe7\xb4\xc6u\xb2\xce \x95\xe4S\xe9\xdd\x0f\xed\xdd\xcf\xedO\v\x00\xaf\x06\xec\xc0\xa0C\xc1\xb5\xd2\xcf)\x12\xfe\x99\x90\x85\xdb\x1d:\xa4\xd0ڰ\xe0\x88:\xe7\xdfRH\xb1\x83\xa3\xa1Ə\xb5k\xdfoK\xaa7%\xd5cMU\xacβ\xfcv\xcb\xe3w;zE\x97H\xb9\xeb\r\x15\a\xee\x03\xc9\xfbc\xd1\x06̚\xaa\xc5\xfamr\x8a\xae\x06/\x00X\x87\x88\x1d\x94ب4\x9a\x05\xc0\bH\xc9Հ2\xa6@\xac܊\xac\x17\xa4\xfb\xe0\xd2\xe0\x8f\x95\x905\xd9(\x05\xc2\x0f=\x96\xcbC\u0600\xf4\b\xb5\x1cH\x805\x8e\x1d\x98\x12\a\xf0\x89\x83_)\xe9;h3\x92mu͍\x8c\x0e\x95\x847\xf3c\xd9\xe7\x86Y\xc8\xfa\xed\xad\x16X\x94$\x9e\x9a(um\xf0@'ȟ7P\xfc\xdb\xd8+>\xaf\xfeT\f\xb7*W\x9f\xdd]EZ\xf78\xa8n\xf4\r\x11\xfd\xaf\xab\x87\x8f?>\x9d\x1d\xc3y\xafWH\aˠ\xa6N3p\x155\b\x1e!\x10\f\x81&T\xb9=$\x8d\x14\"\x92؉\xff\xfa\x9d\xac\xd5\xc9鬅\x7f\x9b3\x1b@\xee\xbaF\x81\xc9\xfb\x85\\@\x1c\x87\x02\xcdx\xd1\n\xaee \x8c\x84\x8c\xben\\>V\x1e\xc2\xfa\x13jig\xa9\x9f\x90r\x9a<\xaeə\xbc\x96;$\x01B\x1d\xb6\xde\xfe}\xc8\xcd\xf9\u07b9\xa8SR \xc9c畃\x9dr\t\xbf\a\xe5\xcd,\xf3\xa0\xf6@\x98kB\xf2'\xf9J\x00\xcf\xfb\xf8#\x83h\xfd&tЋD\xee\x96˭\x95Ilt\x18\x86\xe4\xad\xec\x97E7\xec:I ^\x1aܡ[\xb2\xdd6\x8ato\x05\xb5$¥\x8a\xb6)\x17\xf1Ep\xda\xc1|G\xa3<\xf1Yً\xe9\xa9_ч\xaf\xa0'\xabE\x9d\x91\x9a\xaa^\xf1\xc8B>\xca\xd0=\xbe{\xfa\x00S'\x95\xa9J\xca\xd1\xf5\x02\x97\x89\x9f\x8c\xa6\xf5\x1b\xa4\x1a\xb7\xa10\x94\x9c\xe8M\f\xd6K\xf9\xa1\x9dE/\xc0i=X\xe1ib3u\xf3\xb4\xf7E\x90\xb3\x02\xa4h\x94\xa0\x99;b\xa3\x13Q\x19\xbe\x83ΫkA_\x8a\x05\x12\x05\xba8\x9d5\xf5\xae8\x95\xff\x12\xcaz\x06\xe5\xf7c H\xaf\x04^\x90\xf2\x86萲Z\xa1\x01\x93.\xf0\x1ba9}\x93\"\x05\x8d|\xb1\x8a\x00Vp\xb8\xd2\xd3+\xec\xe4\xcf'\xe7\xd4\xdaa\aB\to0\xab\x88\xd4~f+o\xdfg Xe\x9fk\x1c\x1c\xde\xfbϒP\xe0\xf6i\xb8\xac\xd4\xc0{|\xb9r\xfa\xe0W\x14\xb6\x84<\x1f\xf9l\\U\xf4p\xfe\x1e\xbc\x82\xd2ա\xbc8\xe4,\x85\xe6\x04E\x96@j{\x8a+\xa7\xf5A\xe9;\xf8\xe7\xbf\xc5\xff\x01\x00\x00\xff\xffߙ6&\xcb\n\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcW\xcdn\xe36\x10\xbe\xfb)\x06\xe8\xa5\x05Vr\x83\xa2E\xe1[\xeb\xec!\xd8t\x1b\xc4\xdb\xdc)j$\xb1\xa1H\x96\x1c:\x9b\xa2\x0f_\f)\xf9G\x96\x1d粺y8\x9c\x9fof\xbe\xa1\x8b\xa2X\b\xa7\x9e\xd0\ae\xcd\n\x84S\xf8\x95\xd0\xf0\xafP>\xff\x1aJe\x97ۛų2\xf5\n\xd61\x90\xed\x1f1\xd8\xe8%\xdeb\xa3\x8c\"e͢G\x12\xb5 \xb1Z\x00\bc,\t\x16\a\xfe\t \xad!o\xb5F_\xb4h\xca\xe7Xa\x15\x95\xae\xd1'\xe3\xa3\xeb\xed\x8f\xe5\xcd/\xe5\xcf\v\x00#z\\Am_\x8c\xb6\xa2\xf6\xf8O\xc4@\xa1ܢFoKe\x17\xc1\xa1dۭ\xb7ѭ`\x7f\x90\xef\x0e~s̷\x83\x99\xc7l&\x9dh\x15\xe8\xd3\xdc\xe9\xbd\x1a4\x9c\x8e^\xe8\xd3 \xd2a謧\xcf{G\x05\xf0y>R\xa6\x8dZ\xf8\x93\x9b\v\x80 \xad\xc3\x15\xa4\x8bNH\xac\x17\x00C\xf6\xc9P1$\xbe\xbdɦd\x87\xbd\xc8\x1e\x00\xacC\xf3\xdb\xc3\xdd\xd3O\x9b#1@\x8dAz\xe5(a\xf8_\xb1\x93\xc347P\x01\x04\f\xe1\x00\xd9]\x84 \f\bO\xaa\x11\x92\xa0\xf1\xb6\x87J\xc8\xe7\xe8\xc0V\x7f\xa3$\bd\xbdh\xf1\x03\x84(;\x10l%+\x1c\xf8Ҷ\x85Fi,w2\xe7\xadCOj\x04)\x7f\a\xbdv \xbd\x94\x05\x7f\x9cx\xbe\x0557\x1d\x06\xa0\x0eG\xf0\xb0\x1e\xb0\x02\xdb\x00u*\x80G\xe71\xa0\xc9m\xc8ba\x86lʉ\xe9\rz6\xc35\x8d\xba\xe6^ݢ'\xf0(mkԿ;ہ\x11c\xa7ZP\x02\xd3\x10z#4l\x85\x8e\xf8\x01\x84\xa9'\x96{\xf1\n\x1e\x13\x82\xd1\x1c\xd8K\x17\xc24\x8e?\xacGP\xa6\xb1+\xe8\x88\\X-\x97\xad\xa2q\x02\xa5\xed\xfbh\x14\xbd.\xd30\xa9*\x92\xf5aY\xe3\x16\xf52\xa8\xb6\x10^v\x8aPR\xf4\xb8\x14N\x15)\x11\x93\xa6\xb0\xec\xeb\xef\xfc0\xb3\xe1\xc8-\xbdrC\x06\xf2ʴ\a\aip\xdeQ\x1e\x1e\xa5\xdc]\xd9TNq_\x05\x161t\x8f\x1f7_`\x8c$Wjh\xb1\x9d\xea\t.c}\x18Me\x1a\xf4\xf9^jS\xb6\x89\xa6vV\x19J?\xa4Vh\bB\xaczEa\xecu.\xdd\xd4\xec:\xb1\x14T\b\xd1Ղ\xb0\x9e*\xdc\x19X\x8b\x1e\xf5Z\x04\xfcƵ⪄\x82\x8bpU\xb5\x0e\xb9w\xaa\x9c\xe1=8\x18\x99\xf3Li'\x94\xb1q(\xb9\xb0\x8c-\xdfT\x8d\x92y\xa4\x1a\xebA\xec\x19d@\xfa\x18\xa8y\x06H\xc1\t\xdf\"M\xa5\x93X\xbe$%v\xff҉c\xc2\xfa\x1e˶d\xce\tC \x99\x8f~\x98\x16\xeaR\f0\xdb賑\x8c\xfd\xcd00\xaeL(Lv\x871\x9d\xba\xe6\x0fM\xec\xe7\x1d\x14\xf0{\x8a\xf9\u07b6\x17\xcf\xd7\xd6\x10\xcf\xc5E\xa5'\xabc\x8f\x1b#\\\xe8\xec\x1b\xbaw\x84\xfd\x9f\x0e}\xde\xd0\x17U\xc7E\xbfۊ\x17\x14\xa3>\xeb\xf7\x11y\x83\xe0\xf9L\a\x85\xab\xac\\\x11ӠyU\xa2\xeb\xcd\xdd{ <\xa3\xfe\x8e\"ݙƾ\x91\xe2^qV\xef\f\r\x8c_zC\xbc\xdd\xd3\xfc\n\x19{\x9a\xaf\xe4݉\xf0)V\xe8\r\x12\x86=S\xbf(\xeaf-\x02\xbctJv\xe9b\x1a\b^\x02!X\xa9\xe6(\xf5\x8a\xf0\x99G\x94Ǚ\xa1,ҰΈ9\xf8\x13\xf1\x19\xf6;\xe7\xa0\x18\x18\xe9*\x06%A1\xbc\x83C\x93\xfe\b\xb5\x8cާ\x15\x95\xa5\xfc2\x99^\xb8\x96DG\xe6\xf9\xeb\xf1\xfe\r&\xbd\xddk\xa6ǸP&G\xe3<\x16A\xb5\xfc\x82\xe23\xe6\xd2\xc4q\xa7`\xe4\xef\xf8\x85w\f\xd4lE\xf1\xabSy\x00\xdf\b\xf1\xe3N1\x13>\x9a\xbc\xe7\xa7o\xd8d\x10\x03?\xb7@\n3\x13c\x85P\xa3F\xc2\x1a\xaa\u05fc\xb9^\x03a\x7f\x1awc}/h\x05\xbc\xff\vR3md\xa2֢Ҹ\x02\xf2\xf1\\\x97\xcd&\xee:\x11f\xc6\xf0(\xe7\a֙k\x8c\xdd0^\xec\f8\xbb_\n\xf8\x8c/3\xd2\ao%\x86\x80\xa7ct6\x93\xd9!8\x11\x06~\xa4\xd5\a(\r\x7f\x19\x06\xc9\xff\x01\x00\x00\xff\xff\xe5\x0fY\x99e\x0e\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4:K\x93\x1b\xb7\xd1\xf7\xfd\x15]\xeb\x83\xed*\ri\xf9\xfb\xe2J\xf1&\xad\xe2\x14\x13[\xd9\x12W\xba\xb8|\xc0\f\x9a\x1cxg\x00\x04\xc0\x90b\x1c\xff\xf7T\x03\x987H\xeeR\xb6\x85\xcb.\xf1h\xf4\xfb\x85ɲ\xec\x86i\xf1\x01\x8d\x15J\xae\x80i\x81\x1f\x1dJ\xfae\x17\x8f\x7f\xb5\v\xa1\x96\xfb\x977\x8fB\xf2\x15\xdc5֩\xfa\x1dZ\u0558\x02\xdf\xe0VHᄒ75:ƙc\xab\x1b\x00&\xa5r\x8c\xa6-\xfd\x04(\x94tFU\x15\x9al\x87r\xf1\xd8\xe4\x987\xa2\xe2h<\xf0\xf6\xea\xfd7\x8b\x97\xdf-\xfer\x03 Y\x8d+Њ\xefU\xd5Ԙ\xb3\xe2\xb1\xd1v\xb1\xc7\n\x8dZ\buc5\x16\x04{gT\xa3W\xd0/\x84\xb3\xf1ހ\xf3\xbd\xe2\x1f<\x98\xd7\x1e\x8c_\xa9\x84u\xffL\xad\xfe \xac\xf3;t\xd5\x18V͑\xf0\x8b\xb6Tƽ\xed/\xca@\xef\xf3\xb0\"䮩\x98\x99\x1d\xbc\x01\xb0\x85Ҹ\x02\x7fN\xb3\x02\xf9\r@$\xde\xc3ɀq\xee\xd9ɪ{#\xa4CsG\x10dw\vG[\x18\xa1\x9dg\xd7\x04s\xb0\x8e\xb9Ƃm\x8a\x12\x98\x85\xb7xX\xae\xe5\xbdQ;\x836\xa0\r\xf0\x8bU\xf2\x9e\xb9r\x05\x8b\xb0}\xa1Kf1\xae\x06\xb6o\xfcB\x9crGB\xd9:#\xe4.\x85ă\xa8\x11xc\xbc\xb8\x89\xfa\x02\xc1\x95\xc2ΰ;0K\x18\x1a\xe7\xc9N\xe3\xe2\xd7\t\xa2u\xac\xd6S\xa4\x06G\x03V\x9c9L\xe1t\xa7j]\xa1C\x0e\xf9\xd1aK\xc9V\x99\x9a\xb9\x15\b\xe9\xbe\xfb\xff\xd3\xec\x88\xfcZ\xf8\xa3o\x94\x1c\xf3\xe65\xcd\xc2`:`B\xb2ڡI2H9V}\n\"\x8e\x00\xbc\x1e\x9c\x0f\x98\x04\xb8\xc3\xf9\x8b\xa8\xacea\xb0Fy\x1dB\xa2?=\xc7f\bz\xb8\xaa\x8dPF\xb8\xe3\n^~\xf3T4\xc9>@m\xc1\x95\bQy6N\x19\xb6C\xf8A\x15A\xd1\x0e%\x9a\xa8hy\xd4\xfeR5\x15\x87\xbc\x15\f\x80u\xca$\x95Mc\xb1\b\xa7\"\xdc\x16\xecD\xe3\xc6w\xfe\x11\x06Q\x18dI\x83h\xdd\xe9\xc2\xef\x10J\xa6\xad\xe2\xd5\x0e\x9fd\x11C\x96Jű\xe3\x1f\xce\xd0\x12\x16\xb4Q\x05Z{\xc6P\t\xc6\b\x91\xb7\xfd\xc4E\x06\x95\xe8\xf7\xb4\xf84\xbaR\x8c\xa3\x01\xa7\xa0d\x92WHd0p\x86I\xbb\x8d*2\x17`{\xec\xe1\xa8Ǩ\xbc\x8f\v\xa7\xd0\t\xbb\xf6/\x83\xbb.J\xac\xd9*\xeeU\x1a\xe5\xab\xfb\xf5\x87\xffی\xa6I\x8d\x95F\xe3D\xeb\xef\xc3\x18\x84\xcd\xc1,\x8c\xc9\xfdo6Z\x03\xa0\v\xc2)\xe0\x14?\xd1z6\xc4@\x80<\xe2\x14\xd8#,\x18\xd4\x06-\x99\x96\xd7(\xb5\x05&A\xe5\xbf`\xe1\x16\x13\xd0\x1b4\x04\xa6\xb5\x85B\xc9=\x1a\a\x06\v\xb5\x93\xe2?\x1dlK\xbc\xa6K+\xe6\xd0:o\x8cF\xb2\n\xf6\xacj\xf0\x050\xc9'\x90kv\x04\x83t'4r\x00\xcf\x1f\xb0S<~T\x06AȭZA霶\xab\xe5r'\\\x9bL\x14\xaa\xae\x1b)\xdcq\xe9\xf3\x02\x917N\x19\xbb\xe4\xb8\xc7ji\xc5.c\xa6(\x85\xc3\xc25\x06\x97L\x8b\xcc\x13\"}B\xb1\xa8\xf9\x17&\xa6\x1fvt\xedL\xd0a\xf8\x1c\xe0\x19⡬\x80\x8c\x80EP\x81\xc4^\n4E\xac{\xf7\xb7\xcd\x03\xb4\x98\x04I\x05\xa1\xf4[g|i\xe5C\xdc\x14rK:O\xe7\xb6F\xd5\x1e&J\xae\x95\x90\xce\xff(*\x81ҁm\xf2Z8R\x83\x7f7h\x1d\x89n\n\xf6\xce'\\\x90\x93-\x91\a\xe0\xd3\rk\tw\xac\xc6\xea\x8eY\xfc\x93eER\xb1\x19\t\xe1I\xd2\x1a\xa6\x91\xd3́\xbd\x83\x856\t\x13\x86a\xc6;%:W\xaaB6\xe5%\x85\xbb\v4S\x00L\t\xcbG[W2\xd7\xe2F\x9bL#圷4\x94|\x968\xb4\xe2\x17\xf0\x8a720\xb8E\x83>\x1b\t\xbe_+\x1f!\x1c\x13\xb2\xf5i\xa1^\x01\xa7\x12\x98\xe5A\x89\x90\xc3\xd46\xe0\xac}\xc0\x99@\x99\xc4\xf8\xd5\xfd\xba\r\x86-\x13#\xee\xb3xw\x91?4\xb6\x02+\xee3\x87\xcbw'5\x97\xc6z\x1b\x90\xf0\x11\xc1)`\xa0\x05\x168\x8a\xc6 \xa4u\xc8x\x9c$'h0\xae\xbd\b\x9e\xfe$\x92\x10\x8a\xca\x18\xb5I&\xc0(\xf2\b\x0e\xff\xd8\xfc\xeb\xed\xf2\xef*\xd0\x01\xac\xa0\xd4\xcc\xd7z>\xdf~\xd1\xd5{\x1c\xad0ȩz\xc3Eͤآu\x8b\b\r\x8d\xfd\xe9۟\xd3\xfc\x03\xf8^\x19\xc0\x8f\x8c\xaa\xa6\x17 \x02ϻ`֪\x8d\xb0\x81\xf0\x0e\"\x1c\x84+=\xa2Z\xf1H\xe0\xc1\x93\xe0\xd8#Yr \xa1A\xa8\xc4c\xc2~¸\xf5\xd9\\\x8f\xe6\xafd=\xbf\xdd\xc2W\xc1y\xdd\xd2\xcfۀF\x97\xb6\f\r\xacG'X\x99\x11\xbb\x1d\xf6y\xffLY(\xccR\x80\xfa\x1a\x94!Z\xa5\x1a\x80\xf0\x80IN!> \x9f\xa1\xf7ӷ?\xdf\xc2Wc\x1e\x9c\xb8JH\x8e\x1f\xe1[\xf2>\x9e7Z\xf1\xaf\x17\xf0\xe0\xf5\xe0(\x1d\xfbH7\x15\xa5\xb2(A\xc9\xea\x18\x12\xe0=\x82U5\xc2\x01\xab*\v\t\"\x87\x03;\x82ڞ\xb8\xa7\x15\x11\xa9&\x03͌;\x9b$F>\x9c7\x9ay\xd6Ԏ\xa7ًϢ\x9ed\xbd\x9f-\x03y\"'|\xb9\xf0\t\x9c\x18\x96^Wp\xe2\xb1\xc9\xd1Ht\xe8\x99\xc1Ua\x89\x0f\x05jg\x97j\x8ff/\xf0\xb0<(\xf3(\xe4.#ê\xd4\xed\xd2\xf7\xc1\x96_\xf8?\xd7\x12\xee\xdbT\x9fJ\xbd\a\xf2\xf9X@\xb7\xdb\xe55\x1ch\xb3\xfb\xa7Ǯ\x93|\xd8Ąs\n\x93l\xfeP\x8a\xa2lk\xbd\x81\xb7\xad\x19\x0f\xee\x98\xc9\xe3g\xb2\x1d\xe2sc\b\xa3c\x16\x9b\xb8\x19\x93\x9c\xfe\xb7\xc2:\x9a\xbf\x86\xb1\x8d\xf8$\xe7\xf2~\xfd\xe6sZT#\xae\xf1$'j\x980>f=VY\xcdt\x16v3\xa7jQLvS\x0e\xbf\xe6$\xa4\xad@s!\xfd{7\xda\xdc&\xa8\x89j\xa0\xdb\xf3\xac\xfcӱ]\"\xe1\x1bv\xb1ϥ\x85g\xf9uY\x15\x1e\xd8\xce\x023\b\fj\xa6I#\x1e\U0005814cC3A\xe9\x02e\x04]c\x10\x98\xd6\x15\xc5\xf4\x90E$ \xc6\xfc7\xb2\x87YO\xdf)\x86$E\xd9v\xa56蜐\x9f\x919\xef'\x88\xfc\xbe\x8c\xeazv\x85\x92[\xb1\x8b\xdd\xce9\xa7dSU,\xafp\x05\xce4\xa7j\xae\xb3\x8c|\xa0-\xe7\xe9\x7f?\xd8\xdaj\xf8\x85\x06c\x9a\xaaQ\xdbqN\fʦ\x9e\xa3\x92\xc1\xa3҂%\xe6\rZ7\xb3^Z\xb8\xbd}\x8e\x8d\x05\xa5\xbc\xa6\xe4\x0eep\xaa*\x8d\x8a\x1e\x13\xf8\xb62u\xaa\xaf\xf2\x92B\x7f\x86o\xa0\xea\x9eʑ1\xdeY\xba]2\xd93\xe8.\xb7SZ\xf1\xc9\xcc\xd8\rN\x16\x03}O\xea!\xf9\x86\xf63\xbaH\xe1\x91-\xf24\x04G\xd7>\xbdQ\xda}m\x1f\x89\n;\xed\x90w\x8d\xfek$\xfej\n\xc4\xf7~\r\x8fF!j\xecJ\xff\xb1\xaf\v\xc5]\x8e\xa0\rj\x96\xec\n\x81\xef\xdc[\xdf\xc2\xfc\xd2\x06`\xc2Bc\x91\xfb\x0e\xda\xec\xee\x19\x84\xf6\x9d\x893\x87\x19\x9d\xbf\xce_\xa4\x1bS\xe1\xcdo\xf8RrU\x97j\x0ef\xceB\xd6r\xcd?ᴏ\x8d)\x8e\xf5\xe0:~\x05h\xc8}\x15JE\xf2\x96\x89\n9\xb4o\xdcτ\x92\xe3\x96R\x9c\xe0\xe3\xda>ND\xeft\xfdw^\x92\t&\xcc\x13\x9e?R\x98ӧ\xc6\v\x92\\O\xb6C\xa9\xaa(/\xd9\xd49\x1a2L\xff\xe0\t\x12\x0fT\xf7\x17%\x93\xbb\xa4\x93k\x1f\xec\x10*f\x1d\xe4\xfd'\x03)\xe2\x87/\xa6Sʆ/\x9c\xfd\xa8\xd1Z\xb6\xbb\xe4\xce\x7f\f\xbbB\xe7.\x1e\x01\x96\xabƥ\xed\xf7K\x1b]\xd0\xf3\xba\x87ɦ\xd8\xd8\xfb1W\xb6\xcen\xdbT\x95?3\x8c\x1b\xfd\xa7\x1d\x1e\xab\x1c\xd3\x19\xff\x99\xd6\xe19\x04Kf/\xb1\xea\x9e\xf6\xa4\xfcq\x17\xec\xce:d8\x13\xd8\xdf\xe2!1\xdb\xfa\xb9\xc4\xd2}t\x9e\x89\xa5ٗ\x18\xc3\xc5\xd0\x1bOq\xae]K\xc2\xec\xbesH\xac}\xef\xbdʳ\x98\x1d\xf1\xbb\xc6mv\xbd\xf5\xde\xf2\xfcg\v3\xfb\x1b\xe7\x1fL\xf2\xa1\xd8RM\x88\xfe|\xabA\x01Rl\xa4\xc5'\x01ﺜ\x02.\xac\xaeر\xa3ŗ~d\xaa\xe9\xf7\x91ޢZ\x8f\xa9\xf1T*{\xbe\xc3\xdd}-\x92\xaek\xcf\xfb\v\xb8\xe03\xfc\xba:\xed\f\x7f\x8f\x1bΤ\xe2V2mK\xe5\xd6o.\xa8Ʀ\xdb\xd8\xdac_V\xfa\xc0\xe2\x9f\xde⦨\n\tT{\xef\xf6,g1\xfex\xe8\x1a-ތ \\\x88\xfb\xf1[\xa6Ttݐ\x17 \a\xe4\x1fv\xef\xa6_p\xbc\xe8\x82\fs\xb1A\x1e\xe2Q\xaa\xab\xa0\xa4\xaf#\x94\x99\xbf\xb2\xc3\xc5@>&\xe8ό\xe1Iu\x9aMz\xcc\xf9\x00v|\xd3\x1c\xce4y\xf7ܿ\x82_\x7f\xbb\xf9_\x00\x00\x00\xff\xff\xfc¬w\xb0(\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4Z͒\x1b\xb7\x11\xbe\xefSt\xad\x0f\xb6\xab4d\xa4$\xae\x14o\xd2*Nmbo\xb6DI\x17\x97\x0f\xe0\xa09\x03s\x06\x80\x01\f\xb9\xb4\xe3wO5\x80\x19\xce\x0fH.\xa9\x925\x17i\xf1\xd3\xf8\xf0u\xa3\xbb\xd1`\x96e7L\x8b\x8fh\xacPr\x01L\v|r(\xe9/;\xdb\xfc\xc3΄\x9ao_\xdel\x84\xe4\v\xb8k\xacS\xf5;\xb4\xaa19\xbeŵ\x90\xc2\t%ojt\x8c3\xc7\x167\x00LJ\xe5\x185[\xfa\x13 W\xd2\x19UUh\xb2\x02\xe5lӬpՈ\x8a\xa3\xf1\xc2ۥ\xb7\x7f\x99\xbd\xfcn\xf6\xf7\x1b\x00\xc9j\\\x80V|\xab\xaa\xa6F\x83\xd6)\x83v\xb6\xc5\n\x8d\x9a\tuc5\xe6$\xbc0\xaa\xd1\v8t\x84\xc9q\xe1\x00\xfaQ\xf1\x8f^λ \xc7wUº\xff$\xbb\x7f\x10\xd6\xf9!\xbaj\f\xab\x128|\xaf-\x95q\x0f\x87\xb52\xd0[\x13z\x84,\x9a\x8a\x99\xe9\xcc\x1b\x00\x9b+\x8d\v\xf0\x135ˑ\xdf\x00D\x06\xbc\xa0\f\x18\xe7\x9eSV=\x1a!\x1d\x9a;\x12!\xbbe8\xda\xdc\b\xed\xd3%\xb3\x18{\x03\xf9K\xdf\x11\x9bܞ0[g\x84,R(ދ\x1a\x817\xc6+\x9d\xf6\x9f#\xb8R\xd8)\xbc\x1d\xb3\x04\xd18\xbf\xf14\x18\xdfO\"\xadc\xb5\x1e\xa3\xeaM\r\xb08s\x98\x02u\xa7j]\xa1C\x0e\xab\xbd\xc3v+kej\xe6\x16 \xa4\xfb\xeeo\xc7\xf9\x88\x84\xcd\xfcԷJ\x0e\xc9yC\xad\xd0k\x0eHH[\x05\x9a$Cʱ\xeaS\x808\x12\xf0\xa67? \tr\xfb\xedg\xa1\x90\xe9\x81Z\x83+\x11ް|\xd3hX:eX\x81\xf0\x83ʃ\nw%\x1a\xf4#Va\x04\x9dm\x10\xa4;e\x92\xaaӘ\xcf\xc2\xd8(\xac\x955\xd2\xdfp\xa1\xcfb_\xb9A\x96\xb4\xaf\xd6I\xcd\xfc\b\xa1d\xda\xc8^\x17\xf8,\x03\xeb\x13)\x15\xc7\x1ek\x13\\\u00826*GkO\x18>\t\x19 y84\x9c\xa5\xa8D?\xa6\x05\xd4\xe8J1\x8e\x06\x9c\x82\x92I^aС3L\xdau\xb4\x8c\xa9\n\xdbi\xef\xf7z\b\xe5C+\xaf\xd73\xc1\x14\x86n_\x067\x98\x97X\xb3E\x1c\xab4\xca\u05cf\xf7\x1f\xff\xba\x1c4\x03Ѣ\xd18\xd1\xfa\xd1\xf0\xf5BR\xaf\x15\x86{\xfe_6\xe8\x03\xa0\x05\xc2,\xe0\x14\x9b\xd0z.\xa2\x7fE\x1e1\x05\x8e\x84\x05\x83ڠE\x19\xa2\x1553\tj\xf5\v\xe6n6\x12\xbdDCb\xc8\xed7\x15\xa7\x90\xb6E\xe3\xc0`\xae\n)~\xebd[\"\x9c\x16\xad\x98C\xeb\xfcA4\x92U\xb0eU\x83/\x80I>\x92\\\xb3=\x18\xa45\xa1\x91=y~\x82\x1d\xe3\xf8\xd1[\x93\\\xab\x05\x94\xcei\xbb\x98\xcf\v\xe1\xda@\x9d\xab\xban\xa4p\xfb\xb9\x8f\xb9b\xd58e\xec\x9c\xe3\x16\xab\xb9\x15E\xc6L^\n\x87\xb9k\fΙ\x16\x99߈\xf4\xc1zV\xf3\xafL\f\xedv\xb0\xecD\xd1\xe1\xf3\xe1\xf5\x02\xf5P\xbc\xa5\x93\xc0\xa2\xa8\xb0Ń\x16\xa8\x89\xa8{\xf7\xcf\xe5{h\x91\x04M\x05\xa5\x1c\x86Nxi\xf5Cl\n\xb9&çyk\xa3j/\x13%\xd7JH\xe7\xff\xc8+\x81ҁmV\xb5pd\x06\xbf6h\x1d\xa9n,\xf6\xce'3\xb0\xa2\x03E~\x80\x8f\a\xdcK\xb8c5Vw\xcc⟬+Ҋ\xcdH\t\xcf\xd2V?E\x1b\x0f\x0e\xf4\xf6:\xda\xfc\xea\x88j\xc7\xfem\xa91'\xcd\x12\xb94U\xacE\x8c$ke\x80M\xc6\x0f\x99J\xbb\x00\xfa\x92\x11e<\xe8\x9c\xd9\xd1\xf7&%\xa8E,{\x8e<\xc6;\x1b\x03U5\fT\xfdo\x12#\rje\x85Sf\x7f\x88\x94c\x938\xaa\x1d\xfar&s\xac\xae\xd9ޝ\x9f\tBr\xe2\x1d;\x93&g\x14\xa4z\xa0J\x16\x8a\x0e\xd9D\x1dp\xefh\x1cٹE\x97ެ<\x1aل\x84C\x8e\t\xfd\\r\xbc\xed\x95R\x15\xb21\x9bZ\xf13\x9b~T\xd1q\x18\\\xa3A\x1f\xff\x83\x9b\xd5\xca;cDŽl\xddGH\xb9\xc1\xa9\xc4>V\xe4n\x8e\xa9\xe6\xb8\x1d\u0089\x90\x94\x04\xfc\xfa\xf1\xbe\r;\xadeE\xe8\x93\xc8\xd2\xe7'i\x16\xf4\xad\x05V\xdc\a\xea\xf3k'-\x84\xbe\xfbu\x00\xe1}\xafS\xc0@\v\xccq\x10\xf7@H\xeb\x90\xf1\xd8H\xee\xc6`\xec{\x11|\xeaQ\x90\x10\xaeE1>\x92J\x80\x91\x8f\x17\x1c\xfe\xbd\xfc\xef\xc3\xfc_*\xec\x03XN\x99\x90\xbf\xab`\x8dҽ\xe8\xee+\x1c\xad0\xc8\xe9\xf6\x81\xb3\x9aI\xb1F\xebfQ\x1a\x1a\xfbӫ\x9f\xd3\xfc\x01|\xaf\f\xe0\x13\xa3\xa4\xff\x05\x88\xc0y\x176Z\xab\x116l\xbc\x93\b;\xe1J\x0fT+\x1e7\xb8\xf3[plC'&l\xa1A\xa8\xc4\x06\xd3\xec\x03\xdc\xfa\xe4\xe9\x00\xf3wr)\x7f\xdc\xc27\xc1I\xdcҟ\xb7\x01F\x97 \xf4\xbd\xce\x01\x8e+\x99\x03gDQ\xe0!ў\x18\v\x054\n\x05߂2\xb4W\xa9z\"\xbc`\xd2Sp\xc4\xc8'\xf0~z\xf5\xf3-|3\xe4\xe0\xc8RBr|\x82Wt\xc6=7Z\xf1og\xf0\xde\xdb\xc1^:\xf6D+奲(A\xc9j\x1f\xf2\xcd-\x82U5\xc2\x0e\xab*\v\xa9\x18\x87\x1dۃZ\x1fY\xa7U\x11\x99&\x03͌;\x99\x8eE\x1eN\x1f\x9ai~\xd2~\xcf;/>_y\xd6\xe9\xfdb\xb1\xfe\x99L\xf8\xc4\xfc\x13\x98\xe8_u\xae`bӬ\xd0Ht\xe8\xc9\xe0*\xb7\xc4C\x8e\xdaٹڢ\xd9\n\xdc\xcdw\xcal\x84,22\xc6,h\xdd\xce}1g\xfe\x95\xff\xe7ڍ\xfb:˧\xee\xde\v\xf9r\x14\xd0\xeav~\r\x03m\x1e\xfd\xfc\xd8u\x94\x87e\xcc\xec\xc62\xe9\xcc\xefJ\x91\x97\xed\xad\xaa\xe7mkƃ;fr\xff\x85\xce\x0e\xf1\xdc\x18B\xb4\xcfb)2c\x92\xd3\xff\xad\xb0\x8eگ!\xb6\x11\x9f\xe4\\>ܿ\xfd\x92'\xaa\x11\xd7x\x92#\xb7\x85\xf0=e\aTY\xcdt\x16F3\xa7j\x91\x8fFS\xae|\xcfIIk\x81\xe6L\xf6\xf7n0\xb8\xcd\xda\x13Yw7梴\xdbJ\xa6m\xa9\xdc\xfd\xdb38\x96\xdd\xc0\x16\xc3A\x871\xe9leё8\x99k>\x03\xcfR\xfc\x96p[ID4\xb4\xc5T\xa9B\xe4\xac\x02\xeb\xdbd,VF\x98\xad\xec)\xa0T=r\f\xb7_U\xec\xe1\xf5\xbe\xe0ḧ\x1dB\x1e\x8enQ+#\n!Yu\xf0\xd8\xfe\xea(Y\xcd\xfc_\t[\xad\x99\xd6B\x16\x17q\xdbַ\x96蜐E\"\xd1\xef\x97\xdfO]\aN\x9e\x93\xf3.\xe0\xc3\b\b0\x83\xc0hO\xa4\xaa\r\uecd0uj&(e\xa4\xac0\xa6\xd6+\x04\xa6uEy]\xc8$S\xbe\xa9\xad\xd6\xe5J\xaeE\x11+\xa7S\xa6dSUlU\xe1\x02\x9ci\x8e]ڒǽ_(<\xa3\xf1\x0f\xbd\xa1\xad\xbaϔ*ӻ\x1a\x140\xa7\x9bA\xd9\xd4S(\x19l\x94\x16,\xd1N\x87s☨\xe3\xf6\xf6\x12\x93\n'\xff\f\a\xe1Μ*8D\xc7\x11\xaf!\xf1\x8a\x1d\xdcG:\x9a_\xeaP\f\xfe\xdaНj\x880K\xd7VFc\xb4\xe27c\xd2\xfa\xbex\xd4y\xf0\xa4\xe3\x8e\xe1\xa1\x1f\xf5\x06\n\x9eU\x96\xf2\x85\xf2K\nS\xe19,\xf2\x1e\xd2\x00\xd7>\x92\xd1\x05\xe3\xea\xd2\x14\xdda\xb5C\u07bd!\\S\xb7y=\x16\xe2\vʆ\xc7C\"j\xec\x8a\x1c\xd1N̡\xec\x12B\x8c6\xa8Y\xd2\"\xc0?\nX_\x18\xfd\xda\x06i\xc2Bc\x91{\xdf:Y\xfchL\xe0\xccaF\xf3\xafs \xe9bWx\x9e\xeb\xbf\xc2\\U\xf9\x9a\x8a\x99r\xc8:\xda\xfc\xfbP\xfb0\x98\xa2\xec \xaf#,\x88C\xee\xafܠ$\xac\x99\xa8\x90C\xf7,}1\xf3\t\xd0\xd3d\xecs\x92_\xa3\xb5\xac8\xe7\xb4~\f\xa3B\xe5-N\x01\xb6R\x8d;b\x95_\xdbx\xb4.\x8a\xc9R\xf1sH\x1e\x14\xf70\xe4\xf1'\xb7)\x9a\x84Z\xfa\xcfp\x17a\xf4E\xcdsEJ\x1a\x93r5\x1d\xe4Ӿ\x06Nİ\a\xdc%Z\xdb\x13\x9c\xe8z\x8cn!\xd15\xf9=@\xbf3T\x92S9Mۗ\x94\xd9=\xb6'\xfa\xbe\xf7\xc7\xe5\"\xb6#\xbek\x1cBW\x87.U\xd5\xfa\x00\xffH.\x9bz\x85\x86T\xb1Je\xc4\xc0$\xefk.UL\xe8$\xb4a8\x88\x8a\xf5\xb0X@\xf7\xa7\xdc)\xe0\xc2\xea\x8a\xed\xbb\xcd\xf8\x1b\x1c\x1d\xe9\xf4s\xc2\xe1\\\xb5\xbe\x8a\"ϑ\xbc\xedt\xa5\xba\xfb\xd1B\xfa~z:Ӈ3پ\xef\xef~\x8c\xf0yV8\x91w\x0e\x7f\x1cr\x8d\x81,\a\x12\xce\x05\x8b\xf8c\x95\xcb}\xfcp\x99?ӽ'ٛ4z\xe4\xbc';>y\xf5[\x9aU\xf7\x1e\xbc\x80\xdf\xff\xb8\xf9\x7f\x00\x00\x00\xff\xffϡa\xa5-&\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=Ks\xdc8sw\xff\n\x94sؤJ#ǕG\xa5ts\xb4v\xac|\xdf\xca*\xc9\xf1\x9e1d\xcf\x10\x9f@\x80\v\x80\x1a\xcf&\xf9\xef)4\x1e|\fHbF\x1a\xednjyQ\x89\x04\x1a@\xbf\xbb\xd1\xc0\xacV\xab7\xb4a\xdf@i&\xc5\x15\xa1\r\x83\xef\x06\x84\xfdO_>\xfe\x9b\xbed\xf2\xdd\xd3\xfb7\x8fL\x94W\xe4\xba\xd5F\xd6\xf7\xa0e\xab\n\xf8\x116L0äxS\x83\xa1%5\xf4\xea\r!T\bi\xa8}\xad\xed\xbf\x84\x14R\x18%9\a\xb5ڂ\xb8|lװn\x19/A!\xf00\xf4\xd3?^\xbe\xff\xd7\xcb\x7fyC\x88\xa05\\\x11\x05\xdaH\x05\xfa\xf2\t8(y\xc9\xe4\x1b\xdd@aan\x95l\x9b+\xd2}p}\xfcxn\xae\xf7\xae;\xbe\xe1L\x9b\xbf\xf4\xdf\xfe\x95i\x83_\x1a\xde*ʻ\xc1𥮤2\xb7\x1d\xc0\x15Q\xbe\xb9fb\xdbr\xaab\x877\x84\xe8B6pE\xb0}C\v(\xdf\x10\xe2\x17\x85\xfdW~=O\xef\x1d\x88\xa2\x82\x9a:\xc0\x84\xc8\x06ć\xbb\x9bo\xff\xf40xMH\t\xbaP\xac1\x88\x9a\xffY\xc5\xf7$,\x810M(\xf9\x86(\xb0\xb3A\x92\x10SQC\x144\n4\b\xa3\x89\xa9\x80Ц\xe1\xac@\x8a\x10\xb9\xe9A\n\xbd4\xd9(Yw\xd0ִxl\x1bb$\xa1\xc4P\xb5\x05C\xfeҮA\t0\xa0I\xc1[m@]F@\x8d\x92\r(\xc3\x02\xba\xdc\xd3\xe3\xaa\xde۹\x85\xd9\xc7\xe2\xc2\xf5\"\xa5e/pK\xf0\xf8\x84ң\x8f\xc8\r1\x15\xd3\xddR\xc3\xf2\b\x15D\xae\xff\x06\x85\xb9\x1c\x81~\x00e\xc1X궼\xb4\\\xf9\x04\xca\"\xab\x90[\xc1~\x8d\xb0\xb5]\xb8\x1d\x94S\x03\xda\x10&\f(A9y\xa2\xbc\x85\vBE9\x82\\\xd3=Q`\xc7$\xad\xe8\xc1\xc3\x0ez<\x8f\x9f\x90xb#\xafHeL\xa3\xaf\u07bd\xdb2\x13d\xad\x90u\xdd\nf\xf6\xefPlغ5R\xe9w%<\x01\x7f\xa7\xd9vEUQ1\x03\x85i\x15\xbc\xa3\r[\xe1B\x04\xca\xdbe]\xfe]$\xea`X\xb3\xb7<\xaa\x8dbb\xdb\xfb\x80\xa2r\x04y\xac\x109\xc6s\xa0\xdc\x12;*\xd8W\x16u\xf7\x1f\x1f\xbe\xf6\x99\x92iO\x94\x1eoN\xd1\xc7b\x93\x89\r(\xd7\x0fY\xd3\xc2\x04Q6\x92\t\x83\xff\x14\x9c\x810D\xb7\xeb\x9a\x19\xcb\x06\xbf\xb4\xa0-\xbf\xcb1\xd8k\xd4Gd\r\xa4mJj\xa0\x1c7\xb8\x11\xe4\x9a\xd6\xc0\xaf\xa9\x86W\xa6\x95\xa5\x8a^Y\"dQ\xab\xafeǍ\x1dz{\x1f\x82\xae\x9c \xad\xd7\"\x0f\r\x14\x03I\xb3\xdd\xd8&\xa8\x8b\x8dT\x03%c\xbb\fq\x94\x16~\xfb8-b\xd5\xe2\xf8\xcb\x12\x97\xd9\xe7\xdfco\xcbovf\xad`\xbf\xb4\x80\xcaԉ?\x1c\xea+\xd5S\xfa\xc3Dzј\xba\x93\x88\xb6\x0f|/x[B\x19\xf5\xfa\xc1\x02s\x96\xf1\xf1\x00\n\x9aCʄ\x15\"k\x97\xecZD\xf7\x15\x158U@\x844\txL8x\x84\t\xc4@\x92&\xd8\xd0@\x9d\x98\xf1\xec\x92\t\x11-\xe7t\xcd\xe1\x8a\x18\xd5\x1e\xa2\xd1\xf5\xa5J\xd1\xfd\x04\xb6\x82o\xf0,dE ^\xd5pV ɣBA|\xfdqQŴU\x94a\x95w\x92\xb3b\xbf\x80\xaf\x8f\xc9NAZ\xbd\xec\xfa\x15\x925T\xf4\x89I\x95\x12\x03\xa9\xb0iϞwjZZ-遌m\\悓Ȫ\xa4|\\b\x88϶Mg\x1dH\x81\xaef\\\x8a\xa7\xb6\xb7\xddk \xf0\x1d\x8a\xd6$\xa6IH٢i\x92\x8a4R\x9bi\xbaO\xab.\xd2w\x8eR\x1fg\x98\xe6`eIVw\x8fW\u0081\xa8\x16\a\x03\x85,\x05\xd8eԖ\xa8][%[\xd7v\x12)dM5\x94D\x8aɑ\x91]Z\x0eڏU\"gtz\xe8\xa2[?z<\x84\xd35p\xa2\x81Ca\xa4:Df\x0eJݓ\xa3X'P\x99ЦC\t\xe8\x160\x03\x92XN\xdfU\xac\xa8\x9c\x87a\xd9\x13\xe1\x90R\x82\xb6\xda\x04]\xe6\xfd\xd4\"\xc9\x12\xf9\xfd sڣ{\x16\xc4j\f/\xa5Q\xba'C\rwO\x12\xb5\x9d\xee=\xd0-\xfe\xbd\x91\xb3\xcb\xfe\xff\x89\xd8`LN`\xda\x19\xf9'\xe8~f\xf3\xf4$\xdfb\x84\a\xfa\x92\xdcl\bԍ\xd9_\x10f\xc2\xdb%I\xa0\x9c\xf7\xc6\xf8\x03\xd3\xe6x\xa6\xcf$M\x8eL\x9c\x890q\x88? ]\xd0d{\x84=\x82Igs\x0e\x9f\\np\xcf#$\\\xff\xd43\xc0\xa1\x9d\x93\x0f\x8b\x1d\x9e\xec\vD\x04\xc6\xf0\xb9l\xe0\x1e/\n\x89\xdcI\xfa\xc9\xd4%\xe1\t\xb8?a\x99Y\xac\xd2\x1f\xa3\x9f\xfaD\x0e\xf8A;ZZ\x89\xa9\x98\xcfij@\x99\xc9%\xa8{\xbeQ\xce\xca8\x90\x93\x91\x1bqAn\xa5\xb1\x7f0@\xd3\xc8(?Jз\xd2\xe0\x9b\xb3`\xd4M\xfc\x9c\xf8t#\xa0\xa0\t\xa7\xe5-\xc2\xfa9?g\xd3,\xb7E\xdc3Mn\x84\x8dW\x1cJ2\x87\xc2\xf4\xae\x1b\xce\rT\xb7\x1a\xd3uB\x8a\x15\xda\xcc\xe4H\x1e\xdfR\r\xd0\xfd\xecA\xfd\x80_\xad\xb1p_\\\x92\x99\xd3\x02\xca\x10Yb\xf6\x93\x1aز\"s\xbc\x1a\xd4\x16HcUx\x1eGd*V\xbf\x9a\xe3\xd8'\xcfz\xf7\x9f\xef\xabǘ/XY\x93\xb3\xf2\x10\x8c\xac3p\xe0uw\xb9\xbc\x9e\x95\x95ٌV\x81\x13\x16\x9bN$G\xa7\x9b\xe6 \xe5\x19\xe8@+\x8e.\xce\"uiY\xe2\xe6\x1a\xe5wGX\x94#x\xe1X\xd5Л\xbb3\xc15m\xacZ\xf8okiQ\x9a\xfe\x974\x94)}I>\xe0N\x19\x87\xc17\x9f\x87\xeb\x81\xc9\x18\xb2\xb1CY\xfey\xa2\xdc\xda~\xab\xc0\x05\x01\xee<\x01\xb99\xf0\x8b.Ȯ\x92ڙ\xed\r\x03\x8e\xfb\x15o\x1fa\xff\xf6\xc2\x0e\xbf8d_ɼ\xbd\x11o\x9d\x0fq\xa00\xa2\xc3!\x05ߓ\xb7\xf8\xed\xeds\\\xa9LN\xcdl6`њ6y\x1c*\x92\xc9\xfa\xee\x19pL?7\xdf%当=\xb7\xda,\x16m\xa46\x9f\xd3yÉ\xf9܅\x1eC\xcf8\x91c[\x8c\x18|\x1e-\xea{\xebDn\f(\x9fKt6 \xc4\x1fό\xccR\xbb2\xfd\xc9\xc6d \x8d\xf9]\x8b\xe0\x05nr\x1b79S<\xc6a\xb5x9\xd2\xdb\xff\xf8\xbd\x97ϴ\x92k\xff\xef/\xe4\xa5\x1d\xeaB\xd65\x1d\xefjfM\xf5\xda\xf5\f<\xed\x019\xea\xabm\x8b\xf2\x9ck\x91;\x1e\xc2\xfd\xcb\x1d3\x15\x13\x84\x06\xb5\x01\xca3\x14%\x8dL\xe5\xb0SOE5Y\x03\x88\x98\xa2\xff=\xb8\x125\x1378\x00y\x7f\x06\xd7#\xa2\xeb\x9c\xce\xeeu\xa4I\xa4||\xe1LV#K\xb2\xab@\xc1\x801\x0e\xf3\xee\xe8\xa9\niz)\x8b#\x1c\xd2F\x96?h\xb2aJ\x9b\xfe\x144iu.\xad\x8f$\x9f\x9d\xf7WV\x83l\xcd9\x11\xfc\xb1\x1bf\xb0\xd7\\\xd3\xef\xacnkBk\xd9:cnX\x1dwu=zw\x94\x99\xb8m\x85\xf9\x1b#-\t\x1a\x0e\x06\xc8\x1a6\xe9\xfd\xde\xd4SH\xa1Y\t*T)8\xb21i\x05sC\x19oS\xbbD\xa9\xe7\xd8\bX|T\xea\xa4\x00\xf8\x8b\xeb\xd9\xcb;Vr7DP\xe6\xdaq#\r\b\xdb\x10f\b\x88\xc2b\x1c\x94S\xc98\x84G\x06\xa2\x86\xe5\xea\xb9<\x05n\x1f\x10m\x9d\x87\x80\x15\n$\x13\xb3)\xb7~\xf3O\x94\xf1s\x90\xcdr\xde'\xa9\ue056\xa7\xe4h~\xeeu' t\xabp\xf3\xdf\xe9\x8e\x1d\xe3ys\xb6\x94#\x9c\xb6\xa2\xa8\x00\x95\x90\x18\xea\x06\a\x9e\tm\x80\xe6\xf2\x82\xf5\x8aZ!\x98\xd8\xe6\xd1.;\x11\xda=\x0e\xd5k)9\xd0\xe9]\xc8\uec78~\x05M\xf4s7\xcc35QG\x04\xb7m\x8etȦ\xa8UZ\x84\x1a\x03u\xe3DN\x12Պ\xbeu9\x83\":&\f\xf7\xb3x\xc9\xf8\x9a\t\x96A\xdb\x01]o\x043}\xe7т8\xab\xf3h\a\x88\xee\xc0)\x19\xb6\x9b\x01\x00+\xa0!\x0e\xc1\xb9G\xae9\u0091\\\x03\xa1e\t\xa5\xcb]ZWć%\xae\xf0m\xa2\xb8!\xb9\xba\xe3=\xc1,ʆg\x10tb\x1eV=\xc1\xaa\x15\x8fB\xee\xc4\n\x83q}\xb4\x0e91K\xf5\xdc\xe1\xcd\xc9\xcahY\xbf\xe4\xab\xe9%-4\xe4\xd7|\x9e\n\xfe\xd3\x19\xb4L6\xdf\x1c\x95\xf0\x98\xe3\x82%\xbd\xe6\n\xb0'>.\xcebn\xfc\x99\xce~S\xfa\xda\x15K?\xab,\xee&\r\xaa\xe7\x14\xee*0\x15\xa8P\x9a\xbd\u0092\xf4rv\x87\xb4\v^b\x9d\x9ce\xaa\xe0\"\xbb\xf2\xcfQ\xe5\x1cF7-\xe7\x17\x96\xb7i˓ᰑ(b\x87\x9c\x95U?\x96\xf6\x18r\xaa/\xb2\xf1د\xb4\x18\xd6\x17\xc6*\x88P`(\xc3ȞƩ\xf5baio\x7f\x7fXN\x81\xf9\xbf0\xfd\u07fc\xf40\xa3R\"\x1f\x8d\xb9U\x9a\x11\x89\tX\t\x06롱\xab\xaf\xf0\xed|\xa1\xef\xef\v\xa7\x06\xea/\x8d\x97\x98I\x176\x03\xad\t8\xa3z\x13\xb4\x06\xadv\xae@\xb4\x03>gh\xfb\x7f(\xdc)\x88\x00&ů_+\b\xe2\xeb\xab\xf7\x99&\xffL*\xd9&\xaa\xfafP\xb6Pݱ\xbc\xe0A\xa1\x87\xdfP\x00C\x9f\xde_\x0e\xbf\x18\xe9\xcb>0\x8b\x96\x00\x84AQ\x97\x99e\xa2dO\xacl)\x0fR\u06dd!p\f\xd4\xf1Y\x02\x9aTD0\xee\x180\xf4\x1f0\x1c\xf9Ҹm\x99\xa3Uܼ/\x9aW\x1drrMȰ\xe6c\xc2\x1a\x1e\xbb}\xf1\"U\xb0\xbfI\xad\xc7\xf1\x15\x1e9\x91\xc4B5\xc7\t5\x1c\x99\xc5b\xcf\xdeoɩ\xd28&\xe6>[E\xc6\xcb\xd7ad\xe1g\xb9\xe6\xe2\x18윽\xbe\xe2\x15\xab*^\xa7\x96\"\xb3\x82\xe2\xe5J!\xf3\xa2ϓJ\x01\x96\x03\x96\xe9*\x88\xc5ڇg\x054'-i\xb1\xa6\xe1\x98J\x86E\xea\xe4\x89٫\xd5*\xbcZ\x85\xc2\xeb\xd6%\xccr\xd1\xec\xc7c*\x0fb\x9c\xf4\x13m\x1a&\xb6\x87L\x91\xcb:\xb3l\xb3\xcc2\xb7\xa3\x89\fx\xa6\x1f\xcet\xd1\xe1D\xe8\xeb\x8eK'\"ɐ\xb6d\xc2\xc8K\xf2A\xec=\xdc\x04\x9c^\xf8(\xa498\xc8f\xa7\xb5c\x9c\xf7Ok!\xd8yP\xfe̤\xa6\xb5\x9bՔ\xb7\x9f\xa4\xabT\x03\xa7\xfc\xa4\xc0\xf1\xcb\bF?;\xfa\x9a\x9e\x7f\xddr\xc3\x1a\x0e֣{be\xf2\f\x99\xa9`\x1f\x91\xfc7\x89'\xa4\xd6{\x84\xf4\xe5>\xca\xe2\xe5(\x88\xa1\x9a\xec\x80sBS\xdcq\xb0\xfc\u009dL.\xe4\n\x8f\x04Z\xf2\x06&\xf1\xe7\x99/\x9c\x14\xe310\xa4^\x9d\x80[P\x81\xa7\x9bub!\x93\xe60G\x8b\x1e\xf8\xe5.\xba\xc0w\xbf\xb4\xa0\xf6D>a\t\x83\xf7\u07ba\xb3\n^\xddh\x1bc\x06\x05\xe8\x95\xf1Ԧ\xc2A(\xd3)(\xf2A8_b<\x1f\xecc5_\x17\xaaYun\xa3\xb0\xe4\x18\x13݅\x8c\xbd\x13ݖ\xdc\xfeܢ\xfe\xf3\x06nLJn\x8b\xbeR\xbe?\xfb\x1b\x15\xeb\x9fR\xa4\x9f\xb7\x1d\xb4X\x94\x7f\xae@n)\x94\xcb\xf6^\xf3\x8a\xee\x8f\xdbD=c\x91\xfd9\x8a\xeb31\x95SL\x7f\x1c\x9e^\xa1x\xfeU\x8b\xe6_\xabX>\xbbH>k\x1f3{\xd3*w\x9b\xf1Ī\xef\xe5]\xf7\xf9\xa2\xf7\x8cb\xf7\x8c\x9d\xb4\xe5E\x9e\xb0\xbc\x8cb\xf6\xe3\x8a\xd83h\x96+\x8a\xafX\xac\xfe\x8aE\xea\xaf]\x9c\xbe\xc0Y\v\x9f\x8f+B?y\a&l\xf5\xdf\xca\x12\xee\xa42K\xc1\xc9ݸ}b'\xb5\x17\xb0I^\x12\x11\x9a&V\x89!\x86\x0f/N[Tz\xd33\xb8\xd3?\xc9\xd2\xcemi\x8f\xe5~\xd4\xfc\xe0\xac\xf2\x06\x14\bw\xcd\xc7\x7f>|\xb9\x8d\xf0S>\xaf\xf7\x8cG\xd7K8\x0f\xa6\xf4\xc8\xf1[s\xbe\x98\xc9a\v}\x80\x17\xde\x17\xa1\r\xfb\x0f\xbc\xef\xed\x19\xe9\xa0\x0fw7\b#\xf8ix\x81\\\xac\xa2\x88;\x96k\xb0\x16+\xa2jR,n6\x03\x88Ê\xdf\xfe5JP\xba+\xb3\x82\xc5d\xa1\xc6\xcb\n\xdeݍ\x9b\xc7\xd4(\x9f\xac\xd3(\xf6D:\x8e\xac\x98*W\rUf\x8fl\xa3/\x06s\bff.\x9d3\xa9X\x0f\xaf\x01K\xa27\xdc\xfe\x85{\x91\xfbf\xb8\xdb;\xc6\xdd)\xf3\x98>\x7f\xb2x\xf2\xe4\x05\xe71m\xb1W\x88\xa9\xc4\xebd\x81ɋ\xa5\xc9\xd417\x05%e`\xe1ڠ\x9ej\xa0\xe4Z\x8a\r\xdb\xfeD\x9b`F\x1c>'\x95\x85O\xd14\x16\xb4\x05養u\xb5ihwzPi,a%t.eU~B\xc8w\x01\xb0\x06\xb7\xbd\xed\xb4R\\B\x03j\xd5\xe5ۺی\xf6\xcd\xf4l\xf5\xc5(d\xf5\xb7\xdc\fj\x17\xac\x1a4\xa0\x84\xff\x96\x9a\xab/\xb8y\xc0z\x9b\xdet\xf7q\xb2\x16\x1bv\x86\x96q\xfc\xe0x9\xd1fT\xac\x93\x00>ʧt\x18\xdcHUS\x13$\x00\x13z\xd4\xe1\xddݚ\xf6\xd0@q9$\xf9\x9f:\xf9O\x9d\xfc\xa7N~Y\x9dl\x95\xdbݷ\x93R\xe1\xf7\xb1\xf7\xbc\xefI9\x8f\xe9\xff\x04\x18\xdb\x1f\xddO-h\xa3\xab\xc45x\xcf\xf3?\xf1\x86HCM\xfb\x9cE:\x00\x83u\xb2\xa2\xeay\x90;\b>fX6J+vKjp\xe0\xfe\xa4\x15\xe3\x17\xbd\xec\xed\xeb\x94\xe9d^\xb1u\xf2\xe5Z\x0e=\x13\xea\aw$\xacj;\xc4\xd4\t\x05:\x8b\xe1v\xc6\xc1\x8f\xf9\xc4B\xe6\xd5Ly\x06\xe3\x84\xeb\x98\x10_\xb9\xb8\"\xc9[\x9a2ob\xfaM\x11=\xa3\xd5tQA\xd9r8\xf5\x1eև^\xff\xe5\x9bX\xc3h\x19w\xb1Zd\xf7\f\xb4\xf5\xb0\x86w\xbezJx\xc8}JN\x05ᘰqW>\x16\xeev\xe0\xa2\x00\xad7-\x0f\x95\xa3\x85\x02j\xa0\f͙\x8e3>\xaa\xf6\xb1m\xb8\xa4%(\xe7\x92-\xa0\xf5\xbf\x06\x8dG<[\xe0\xcbVu\xd7\xed\xce^U\xfa,\xcd\xd5PE9\a\xfe\x89q\xd0?ʝ\xb0\xf3\xca\x10ȻT\xbf\xdeY٢U֬\xef\x89h\xeb5(\xa2\xc1\x98\xe9\x04\xdeF\xaa\xf9S+\x0e\xefL\x18\xd8B*\xe7\xb9S\xcc\xc0CC\x95\x06\x9cQ\xc6\n~\x1euq\x19\xc1\r\xa7[W\x9e\\\xb2\x82\x1a\x88\x06\x18G\x98\x9a>\xf6\xd7\b\x8b\xef\xb1ZTNlDd\v\xf5\xd41\xb9I\xb1\x9e\xba\xf29a\xaa\x93\x97>;\x8b\\\xd0\xc6\xe0\xa1D\xa4#\x12\xd1x\x18x\x91\xfa\xe8\xde\xe7\x01\xd8iN\xf3GK|\x11\xb36\xb4ND\t\xcbz\xe7\xfa\x10\f^ծ\xca^-t\xff\xd2\xdbX\xf4LvT\xc7\x03.I\u07fb\x83\xed\xc0\xa0\xabnACI\xe0\t\x04\xb1\xa2H\x19\x87r\x8eS\xbf\xe2\xe6\x9ez\x02\xf5\x83\x8ep\xb0:۲\xf8\x83\xa1\xcaĩ\x1f\xfa1.\x86\xbb\"%5\xb0\xb2\xbdOs\xdd\xd2WW+ub\x89\x06\x9e6\xf6\xe2Q\x84\xa3\x90\xd6\xfa\xb93\xc25hM\xb7!1\xb8\x03\x05d\v\xc2\xe2=\xee\xf7$=\xa6p\xcc\xda\x1b\x8bAb\x80\x16\xa6\xa5~\x00\xe7\xc2Ŋ\x96pg\x10\x9cC\x1d^1\xe2\f:\x9f\xaa3\x18\xdc`D\xb4\xc5\xde)ʄ85v3\x1dv癚\xaf\x11ʔz\xf4\xeb\x1b\xfc8\x82/z\xf1\x8d,ي\x8a\x8a\xed\xe4!\xe3J\xc9v[\x05ޜr\x88H\xd9b\xe4ܠ*\xd0\xe1ǜL\xabD\xaf\x90\xc2\u05fdMi\xe98\xddi\x1f\xe5\x19\x8aZu\x87\r;U5c\U000f3cc4\x13\x10\x17m\x7f\x02\"\xd5{Q\xcc\x1e\x8b<ܣ:ʵL\"!j\xe3\x17CB\x848\x85\x84\xbe/\xd1E<\xbf\x1b\x8cL\xf9('\xa2cމ\xc1%\u0383Z^t\xdf\t\x1a\xba;ǡC\x0f\x82\xbf\x93\xd2n\x03\b\xc7D\xbe8v:\xee\xfd\xfdF\xacO\xd1\xdb\xfaxr\xec\xfam\x04ct,\xddF\xb1\xdd0!\xde\xfc{\xb6Iɋ\xfbż5\x87\x7f8\xf8\xfa\xca\xc7\xcbwT\t&\xb6'a\xe4g\xdf7\x11\xcf{\xb0\xe7\x8c\xe8\xc3\xcc_,\xa6O\x9a\xa5\x83\x97\xc8\xe0e\x0f\xcf~$\xff\xe6\xff\x02\x00\x00\xff\xffJ\xb7g~\xf1r\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=\xdbr\x1c+\x92\xef\xfa\nB\xfb0\x97P\xb7DZ\x97\xd8ЛG\xb6g;\xc6c+,\x1d=\x0f]\x95\xad\xe6\xa8\n\xea\x00%\xb9wv\xfe}#\x13\xa8[\x17]T\xab%ϙ1/\xb6\xaa!I2\x93\xbc@\x02\x8b\xc5\xe2\x8cW\xe2\x0e\xb4\x11J^2^\t\xf8fA\xe2_f\xf9\xf0\xdff)ԛǷg\x0fB\xe6\x97\xec\xaa6V\x95_\xc1\xa8Zg\xf0\x1e6B\n+\x94<+\xc1\xf2\x9c[~y\xc6\x18\x97RY\x8e\x9f\r\xfe\xc9X\xa6\xa4ժ(@/\xeeA.\x1f\xea5\xackQ\xe4\xa0\tx\xe8\xfa\xf1\x0f˷\xff\xb5\xfc\xcf3\xc6$/ᒙl\vy]\x80Y>B\x01Z-\x85:3\x15d\b\xf4^\xab\xba\xbad\xed\x0f\xae\x91\xef\xd0!{\xe3\xdbӧB\x18\xfb\xe7\xde\xe7O\xc2X\xfa\xa9*j͋N\x7f\xf4\xd5l\x95\xb6\x9f[\x98\v\xf7\xbb\xfbM\xc8\xfb\xba\xe0\xbamsƘ\xc9T\x05\x97\x8c\x9aT<\xa3\xaa~l\x04b\xc1x\x9e\x13\xb5xq\xad\x85\xb4\xa0\xafTQ\x97\xb2\xe9 \a\x93iQY\xa2ƍ\xe5\xb66Lm\x98\xddB\xb7\x1f,?\x1b%\xaf\xb9\xdd^\xb2\xa5\xa1z\xcbj\xcbM\xf8Ց\xcf\x01\xf0\x9f\xec\x0eq3V\vy?\xd6\xdb;v\xa5\x95d\xf0\xad\xd2`\x10e\x96\x13s\xe5={ڂdV1]KB\xe5\x8f<{\xa8\xab\x11D*Ȗ\x03<=&\xfd\x8fS\xb8\xdcn\x81\x15\xdcXfE\t\x8c\xfb\x0e\xd9\x137\x84\xc3Fif\xb7\xc2L\xd3\x04\x81\xf4\xb0u\xe8|\x1a~v\b\xe5܂G\xa7\x03*\b\xf62\xd3@2}+J0\x96\x97}\x98\xef\xee!\x01\x18\x91\xa8\xe2\xb5\xf1r\x14Z_w?9\x00k\xa5\n\xe0\xf2\xac\xad\xf4\xf8\xd6\xc9^\xb6\x85\x92_\xfaʪ\x02\xf9\xeezu\xf7\xef7\xbdϬO\xd1\xff[4\xdfY\xc3\r&\f\xe3\xec\x8ef\x10\xd3~J3\xbb\xe5\x96i@1\x00i\xb1F\xa5a\x11H\x9d3\xa5;\xa0*\xd0B\xe5\"\v,\xa2\xc6f\xab\xea\"gk@n-\x9bڕV\x15h+\xc2|r\xa5\xa3z:_\x0f\xa1\x8f\x05G\xecZ91\x05C\x92\xe9g\x1b\xe4\x9eHn\xf2\bӎ\x878\x88\x9f\xb9dj\xfd3dv9\x00}\x03\x1a\xc1\x84QdJ>\x82F\x8ad\xea^\x8a\xffm`\x1b\x9c\x12\x96$Ղ\xb1\x8c\xe6\xb3\xe4\x05{\xe4E\r\x17\x8c\xcb|\x00\xb9\xe4;\xa6\x01\xfbd\xb5\xec\xc0\xa3\x06f\x88\xc7_\x94\x06&\xe4F]\xb2\xad\xb5\x95\xb9|\xf3\xe6^ؠ\x903U\x96\xb5\x14v\xf7\x86t\xabX\xd7Vi\xf3&\x87G(\xde\x18q\xbf\xe0:\xdb\n\v\x99\xad5\xbc\xe1\x95X\xd0@$)\xe5e\x99\xff[\xe0\xb7\xe9u\xbb73]!u:\x83=\xa8g\x9dt9Pn\x88-\x17\xf0\x13\x92\xee뇛ۮ\xe4\t\xe3\x99\xd2\x11\xc0\x18\x7f\x90\x9aBn\xc0낍V%\xc1\x04\x99WJHK\x7fd\x85\x00i\x99\xa9ץ\xb0(\x06\xbf\xd4`,\xb2n\b\xf6\x8a\x8c\x16\nm]\xe1\xdc͇\x15V\x92]\xf1\x12\x8a+n\xe0\x95y\x85\\1\vdB\x12\xb7\xba\xa6xXّ\xb7\xf3C\xb0\xa7\x11\xd6\x06]qSA֛j\xd8NlD\xe6&\x14\xaa\xe4F\x95\fԲ+㳟~!\xdd7\xfc:\xc0\xc3)\xc8\xd0+\x184JvK\xccom#\x8a\x9c\x83ƔfR\xd9=\x98\xfb\xaa\xb5C\t\x0fe\x02\x93=ag{*5Œ\x8e\x00im\xebP\xbe\xa2\xac&\xbc\x1fD\xb5*K\xc8\x05\xb7P\xec\x8eB\xbf\x0fb\x8c̊\xfaak\xa7\xe7ŦG\xf4\xbc\x06&:\xedi2\xfe5\xd4ط\xc6\x7f%\xcbNF\xd4Ѥ\v\xac\x96-\x0f\a\xfdHx\xda'\rc\xab\r\xb3\x1au\xae\xc7\xeeI\x14\x05\xcedĸ\x82\xbc\x87Z\xbc;\xb1a\u0086Ѭ9a \xd9\xd2yQ\xcb\xd6gh\xec?\"8\xc0\x8eԾ\xeb\x1f=\x15n\x99\x84o\xb6\xad\x85Î\x8c`\xc3\v3\x18\x82WH\xb3\x86q\xc1ֵ=\x0e\x03(+\xbb\xbbpm7\xaa(\xd4\x133\xa4l\xd1\bn\xc4}\xad\xddd\xffm\x0e\x1b^\x17\xf6\xd2\xe1\xfc\xbb\x98\xb4\x8eO3\ve\x85&\xf3\x189\xbd\xf5mq\xc08[\xf2&\xfe\bnr\xf0C\x94w?F\x80(\xe7\xc5VZ=\x8aܛ\xf3=u\xc5\x0e\xaa,,\x8eQ\xb78ґ_\a\xa3\xf9cS\xb93\xb7\xb6\xea\x89=b\x00\x00\xe8-r\x1c\x14\x02\x85\x9c\xd5\xd5\xc5(Lƞ\x84ݲJ\x19#\xd6\x05xρ\t\x99\x15u\x8e\n\xe6c]\x14$\x82+\x99i(\xd1\xe1)Ƹ\xcd\x18Ⱥ\x1cG|AP\"?u\xe0\x8e\xd68\xa0\xa7\xb0dF\xdcH^\x99\xad\xb28\x8dTm\x13\x887*\nX\xaenV\x03h\x1d\xea\"\x8f)~\xa0\xc9m\x15{\xe2\u0092\xa1\xba\xbaY\xb1;\xa2{h͜\x86b\xb6\xd6\x12\x9d\x83H\x7f_\x81\xe7\xbb[\xf5\x93\x01\x96\xd7\xe4ׄ\x98\xe0\x82\xada\x83N\x80\x06\x84\x81?\x81\xd6h\x14\r!\xa1\xea=?3\x94['\xc78\xa1\xbc\x9b$\f{\xfb\aV\nY\xdbѩ:Ie\x94\xa6R=\x82~\x0eq\xdfs\xcb\xff\x82@\x064%Q%\xe8~\x96\x11}\xd7;\xfaq\x1d1_\xae\xac6\x1d\xa8°\xf3sT\xa1\xe7.\x86?\xbfp\x10jQ\u0605\x90\xdd~\x82>Ǟ\x8e#\x88\xa3\xafc\xba\xb9U\x1f\x8d\x9b\x90ϢO\x04\xe6\x88\xf1\xacT\x1e&\xfaF\x14\xc0\xcc\xceX(\x83\xaaoåN\f8,\xe4`\x15\x85\ac\x90\xde~P\xe3\x04\x91uQ\xf0u\x01\x97d\x19\x0f\xd0l\\I\x8f\x11\xed+\x18+\xb2S\x92\xccA\x1c!\x98\xf6?\xf4(C\xf1\x16\x7f\x00\xc6#\xe0==1\xb8+\x8a\x0e\xd1\xfbԊ\xe2Vi\xc8\xd0\xf1\xbf\xf4\x01\x85\x80\x82\x82\x18\xa9X\xa1\xe4=h\x87Ec\xe0\xd1\xc0\x00\nh\xce\xd0W\xd7h\x96\x85d\x9b\x1a\xdd\xf8%C-\x11\x95\x11!\x8d\x05\x1e\x11\xe6\x13\xf0\x0e\xbe\xa1E\x80\xfc\xaa\xa8\x8d\x05}\x93\xa9\n\xf2\xb0j7j\xceRy\xf8\xe1 d\x1f\xf4\x15\"\x03\xe4C\xe6*-he,&\xdam\xfc\x87\xe6\x91\x16\xf1\x90\xd5~\bm`7\xa9[\fXlx\xfe\xfb\xf3\v\x92\x80~\xef\xfd~\f\xe3\x1a\x1a2\xcd\xd2\xcd\xe4&\x8d\xb7\x10\x16\xca\bu'u\xd4\f\xbes\xad\xf9\xee\x00כ\x15\xc8\x17\xe0{\f\xf6\x80\xf32T\xfbN\xbc\x1f\xf6\xff\xaf\xc8\xfd\xd3\xf2\xdb\xd0*>\x17\x12\xf9\\\bc{l6n\xe9\x0f\xc9:\x16w{\x02I\a\x13\xd5\xe4\x14W\xffA\x88yҹ\x13\x9b,\x8dl\xfa\t\xf0OEɭR\x0f)\xd4\xfb\x1f\xac\u05ee\xfb\xb1\x8cv\x9a\xd8\x1a\xb6\xfcQ(m\x86k\xcb\xf0\r\xb2\xdaF5\v\xb7,\x17\x9b\rh\x84E{#\xcdV\xca!b\x1d\x8e\xf9XGeE+\f\xc6\xd52\x1dYJԈ\r\x85\xa2\xfa(T\xe7\xe0`hA\x0eD.\x1eE^\xf3\x82|\t.37>\xde\xe0\x17\xd3j\x13\x02\xb1\x87\x7fT\xaa]q\x0eM\x18$2\xb1\xb7T\xa8$\xa0\x8f_bl\xb4_5N\x89\xb0\xfer\xb0od\xa6\xae\v0\xbe\xbb\x9c\xdc\xe4V']\xb4\xccr\v3\x05_C\xc1\f\x14\x90Y\xa5\xe3\x14J\x91\x03WR\x95n\x84\xb8#Z\xb6\x1fm\xb5\x83\x99\x00\xcb(\xc4݊l\xeb\xdcW\x144\x82\xc5r\x05\x86\x96\x92xU\x15\x11\xd3ՖI\xe1\xf0\x9dM鍶$h\x90!ܘ.iK\xa2~n\xcb(\xd9۹٧\xfa\xf8\xe6\xc8(\xbe\xffJD\x0fV\xe7Ha\x9f\xd0$\x8c6Y\x92\xe7C\x94\xf4Hq\x01f\xd9Y\xd2\x146|Mah\xcf\x7f\xdc\xdb\x7f\xda#ʯ\x8bw\xc7M\x98\x19\xac\x9b\x9cS/˸\xa6\x9b\x7f\x12\xbe\x91ɺ\xf1\x16k\x16\xcf>u[^\xd0V\x8agH~\xc16\xa2\xb0@N\xd5\x14\xa2l\x06\xe7NI\xa0T\v\xcchg\xddf\xdb\x0f͆[B\x8b\x01\xad\x86\x00\x9c\x83\x1e\xa2\x1c\xe2A\x02Hָ\x16\xb4\xd3,\xdcJ\xb9YR$\xd9\xfdB\xae\xe0\xbb\xcf\xef\xe3\xb1g\xb7$J\xeaޠ\x12&\xad+\xef\x06\x8eQ\x17W\x1f\xaa\x84_\xc8_k\x02A\xb7\xffp\xc18{\x80\x9ds\xb1\xb8d\xc87\x1e*'\xa2\xa0\x81\xd2(HS<\xc0\x8e@\x8d\xe7E\x8c\x979\xd2\xe2\xca\x03\x8cl\x95\xc6J\x8f\xae\x88\x9f߀rt\xc3\x0fD\x98\x94\xd9Ԗ\x86\xa8~\xfa\x8cd%\xc4\xcb\f\xbd\x14J\xe0ˑ\xc3N\x16\xa7n_\xfdL\xa2\a\xd8\xfd\xc68^\xe3,\xdb\nک\xe3\xb4z\xa36\xb3\x18\xee\xca\x1d/D\xdet\xe6\xe6\xd5J^\xb0\xcf\xca\xe2?\x1f\xbe\t\x83\x1d˜\xbdW`>+K_^\x94\xcan\x10\xafA\xe3\xb0\xfb\x87\x03t\x96\x04\x89\xd8\u0378q\xb6\x14\x05\xb5\xe1\x870l%1$s$\x9a\xd1\x1d%X\xb9.]gemh\x7fZ*\xb9p\xcbbc\xbdy\x1e(\xddc\xc1I:\xf6\x9dޢ1r\xbf\xb8T\xaf\x82g\x90\x87-:\xcaA\xe2\x16\xeeE6\xa3\xcf\x12\xf4=\xb0\n\xcdB\xba\xb4\xccP\xd4~d\xf3\xc5+\xdds\xe8\x96o\x8b\x87z\rZ\x82\x05\xb3@\xb3\xb6\xf0P\xac*\x13\xe9\xe2m\xc2H\xa2\xceXY\xe0\\O\xac\x19\xa4%\xa9z$\x8d\xe9p\xf5Tb=\x93L\xe4E\x90ە$\x05\xddl\xe0y\xd6k\xa6\xdc\x1c\xa3b:cq.@\xc9ik\xedoh\xe9i6\xfe\x9dU\\h\xb3d\xef(\x1d\xba\x80\xdeo~a\xb2\x03&\xb1ۊV\xd9\x7f\xa9\xc5#/\xd0\xff@\x03!\x19\x14\xce\x1bQ\x9b=_\xed\x82=m\x95qnC\xb3iw\xfe\x00;\xb7\xa3\x9c\xd4mWa\x9d\xaf\xe4\xb9\xf3e\xf6\x14O\xe3\xf8(Y\xec\xd89\xfdv\xfe\\\xf7n\x86DϨ\xda\x13\xe5\x92W\xe9\x92L\xc9\xc6s\x02\r\fփC\x84\x8d\x9b\xac[\f\x10\xa6(\x90,ʕ2\x91d\x91\bZ\t\x82~\xad\x8cu\xeb\x90=\x7f\x7ft\xa1R\x85\xc5I\xc67\x1643V\xe9\x90NJ\x8a?e)\xbe[n\xb7`\xc0\xefC\xf9EO\a\x18\xa3\xd8\xf3V78\xabr\xee\xf6¨#\x9e\x91\xf7Dm+\xad20Ѽ\x88\xb6$ڦ\x91\xac\xa5.\x1d\x9au]\ue8bfM\x92\xd6NY\x94\x0ee\x9e#\x8f\xa4;\"2\xfa\xf0\xad\xb3D\x8d\xda\x05\xffN\x91\xd6cpdt\x00\xa6,\xf90\x87:\x19\xdd+\xd7:\xcc1\x0f̅[\xfa\xbe&\x9d3\xc7\xebhD\xf9\x1f͵)\x85\\QG\xec\xed\v\xbaC^\x8b\xc7ң\xc6\xca\xf1N\xfaU\xe8\xac\xe5^\xf3\xc1'\"*\xda\xf8\xd1\xd0c\xee\xfe\x9e\by\xd7R\xd9\xce2\xceL'\xbaR\xf9o\f\xdb\bml\x17\rs \xb1j\x14\xd4\x11\xa1\xa7\xfc\xa0\xf5ё\xe7\x17\xd7z\x90:\xe9\xb2\xcd\xe7\xc4ہ\xa4[\xfe\b>\xdd\x17d\xa6jIKa\xa8\a\xb0\x9b\x19\x10\x1dk\x9c\x15H\xb4w\x9d\xc6\xd1\f̱\xb2 I\x12rrݬ\xdb\xe4#\x17i\xebV\xec8\xb6\xdaC9\x9cc\xe5\xf8y\x14\x12<\xbbg\x10J\xfeM\x94u\xc9x\x89<$\xb7C\x94\xd0\x1cCp\xecn\xd2>\xb1\x05\x19-\xabp\x96U\x05X\xf0i\x9b3\xf0Ȕ4\"\x87\xc6\xf4{\x11P\x92q\xb6ᢨ\xf5\f\xad:\x9b\xe4s\x830\xafMN\x1fY\xa5#\xb2 \x12%\xae\xb3\xcf\xf0\x82\xa75~\xa5\xe7\xf9\xb1)\x0e\xa3\x86\xf9\xfeb\xa5\x85r'(N\xef2\xfa\xb4c.w?|\xc6\x1f>\xe3\x0f\x9fqNG?|Ɖ\xf2\xc3g\xfc\xe13\x1e.?|Ɣ\xf2\xc3g\x9c\x89\xc8\xf7\xf2\x19S0\\\xd0\x1a\xe7\x81\nIX%\xa6BL\xa1=їO\xfa\xf1g5N\x92˼\x1a\a9r\x88'r\xfc\"\xe6u\xb4ƫIn\xc6\x19\x18\xe6\x8e;z\x9a\xe00\x9f\xe0\xf4L@\xe0\xf4\xa7gV\a!\x9f\xf0\xf4\x8c\x1fBZ\x84q\xd4ٙ@\xa4\xf9\xa7'.|\x12Q\t\xaaA\xfe\x83mU\x1d9\xb51Aڄ,\xda4\x82\xf4\x92j}b\x04X\xfe\xf8v\xd9\xff\xc5*\x9fbK76D\x80\xd1%\x1e<\xa7\x9b\x1b:\az\xbc\x1e\b\xf7K\r\x852\x02Li&E\xe1$6@\xe8\xc9+\xfbR\xb9\xd5\xc1\xa3\xfd\xa6\xe95\xac\xf4Dܹ\xe9\xb7M\xb6\xe4\xb4\xfb\xfe\x8c\xa4ۓ\x1e\x8d\xfani\xb5\xc7%Ӧ\xaeP&$Φ\xa7˦\xb0Օ\xf4$\xd9\xe4\b95!v\xee\nċ&\xbf\xbeL\xcak2\xcd\xd2\xd2[\xe7R\xecURY_9\x81\xf5\xf5\xd2Vg$\xab\x9e\xfe\xd4K\xfaZ\xfa\xd1ٕi\xcb2\x87\x13N\x93\xd2L\x93\x96nR\x06|\xd4P\x93\xd2G\xe7&\x8d&q2}\xba\xbejZ\xe8\xab&\x83\xbe~\n褴MV\x98\x9b\xe49~3d(\xd3\x0e@\xf1=\x84\xf3\xb9dR\xba\xe7\x9a?+\xee\xfc2\x80\x85\xc2\x12\xdc\xd4W\x8c\x03ʺ\xb0\xa2*\xdaK\xecb\x01\xe7\x16v\xcdeE?+:\"\xefo\xea\xfa\xf2\xb5\x91\xf8\xe5 \xaa\xe1\x86=AQ0\x1e\x9b\x9b{T\xc8\xdc婙Z\x00\xdaF\x9c\xe5\xfe2&\x7f\xe3ꅛ.t\x1b\x00Y\xd82\xb6\xd4\xc7\xe5\u16fe\x0e\x1a\xb0T=\xb6登x\x83\xbe\xfdR\x83\xde1\xbaw\xac\xf1\xcd\xdaC\xa5~\xa2\x1b\fL\x83\xfa\xf1\xea\xf0О\xc9^\x80Ӫ\a\xf6N:\x8f`\x88\x13\xb5A\xbd\xd3\x06t\xa8Te\xecr>\x16&\xe8>\b\xa9\x1a\b\x91\xa6)\xce\xff\x9cS\x96/\x11ޝ\"\xc0K\xf2\x80\xe6y\xaf\xdf\xf1\xf4䱧&ӓQ\x92NI\xbeD\xb87'\xe0\x9b實\x9f\x82\x9c\xbf\xf1\xfc§\x1e_\xea\xb4\xe3\f\ua95en\x9cO\xbbW:\xcd\xf8\xea\xa7\x18_\xf3\xf4\xe2\xacS\x8b\xc9\xe9Y\xb32\x0e\xe6\xa4V=\xe3\xb8]Z.\xc1\xf4)\xc4\xc4Ӈ\x89\x99\x06i\x83?r؉\xa7\v\xe7\x9f*L\xe4\xef\x9c)\xfdʧ\a_\xf9\xd4\xe0\xf78-\x98 \x81\tU\xe6\x9f\n|\xf6\x96\x94\xd29\xe8\xc9m\xbf9R;)\xaf\xa9\xb1\\\x1f\xb1\xc1\xbeV\xb8M\x16k\xf5b\x002K\xfe\xf5\x03z\xe9\xe2\xd068Jf\xc7#\xea\xedK\xb6\xeeZ\xdf!\xf6O`\xb8\xadK\x03\x15G\x03@\x81\x1b\xa5fE]\x85\x0f<\xdb\x0ez\xd8r\xc36J\x97ܲ\xf3f\xb3\xf8\x8d\xeb\x00\xff>_2\xf6Q5\xb9:\xdd\xfbҌ(\xabb\x87\x91\x18;\xef6x\x9e\x94D\xa53\xf4|\xad\n\x91E|\xce\xd1{\xf5\\\x83\xbdˆ\xe8濬\x93-\x12\v|\xb0\xb9\b\xb7.\xf6\xafdv\x97\xe0\x1f\xb9V\xc2+\xf1'z\xa3\xea\x04\xabn\xef\xaeW\x04+\x88\x11=~\xd5$(6,_\x03\xba\f\xed\xd8\x0f\xe9\x93զ\a\xb5\x9f#\xdc}\xe1\x03r\xf7\x9cKp[\xbcj\xce\x14j\xad\xeb\x95\xc3\xe5PO(_\\\xee\x98\xf2\xefu\b\x9d/*\xae\xed\xce%\x13]\xf4\xf0\bv}j\xd5젵\xda\x7f\xae\xa6[zd\x0f/\xd5\xd0N\xf6\xae\xea'\x0f\f\xe9\xf9\x1c\x9c\x0e\x9f\xaa\x9els*2Z\xa5\xf9=|R\xeeA\xa2\x141\xe9\xb7\xe8=W\xe5=\xb7\x90\xaf\xed'aL\xd1\xfb\xb1\r\x01\xb6\xe73\xf6.\xfaGl\x8f|\xca\xc0\xda\xe292r{\xfbɍ\x94ށy\xef\x9ftA}l\x00Y\x10(࠭\xf1\xbf[\xf5D\x17\xe0\xc7טë+\x9d\x87߀\x0e\x8aP\n\xefQì\xabB\xf1\x1c\xf4\x15\xbd<\x930\xe2\x9fz\r\x06\xee@\xff\xfd\x1ao7#\xe3\t=\xbf`\x96\fztE\x01\xc5GQ\x80q\x88'\x9a\x86\xeb\xfd\x96\x8d\xa5\xa8˵\xf3T7\xf8c\xd3\xc9\x01\xcb\xec\x86J\x1b\f\x15h\xf4\x13\xddVDm\x82\xe4\x1f&\x06k\xf8(\xa4\x85{\x18\x8f\xa1'l\x82{\xa3\x81\x1c\x80\xa0\xc0(\xe2\xfbsl\xe5\xb1G\x90\xbbx\xeb\x81\f4\x8b\x9119Vޭ\xba\xbe\xbb2\xac\x969m\x00\xdc\xfd\xe9\xe6(\xf9}\xec\xbd/\x13tB\x8az\xbf\x1bo\xd9\t\x11:ډ|\xfa\xb8\x12\x8f\xc1\xe2ƨLPTA\x0f\n\xd1I\xa4\x97\xbbC\xfcP\x80x@:j\x03_\x9e$\xe8\xaf\xc1\x02\x99\x95\x8c\xbd\xdb2\xad\xfd~ڃ\x16}\xaf\xc5*\xec{\x04\xc6\x00\x00Sa\x9f˸\x97\x80\xc2\xf6\x9a0͋p\xfb\xf4\x9cP!qK7\xee\xb0-\xc6\x1f\xb0Z4\x0fm\x9d%\x90۽\x7f\xd4\a<\xfe\x0e\xa0{()㕭uЮ\xb5\xa6[\xd6\x11\b\xb8Kȏ{\t\xb0} \xee\x18\x06\xb7/\xb4\xb5\xfb\x0f\x93oȎ\xc0i\xde\xf2\x8b>\f\xe6\"j\xf7\xc6\xeb\x02\xe1\x1f\xc7\xe3\xd1\x19\x838߸\a\xdf&\x88\xf0\xa9\xad96\xe0f\x188d\xff\x84ܫ\x8e\x84.ݟ\x18\xc35\xd6iN\xb9z9\xa2\x86\xe1\xb2\xfe\x9b\x18\x13ƏB.\xd8g؏\xd8\x17\xec\x83\xc4A\xec\x13\xc0\x9dw\x84\x9c\xb6VH;\xce\x19\xe2cӊ\x0e\x9b\x8eh\xc8i\xb1\xbd\x1b\xc0\x18d\xb2ӣOM\x15w\xda\u0530ߊ1o\x94v\xcc2\x1c\xe8\xef\xf6~\x8dj\xf0\x83\xda;\xa6\xb9G\xd5\xc8\xdeGz=0\xefH\x8e\xf7һ_\xeau\xfb\xa0\x02\xfb\xdb\xdf\xcf\xfe?\x00\x00\xff\xff0\xe5e\x05\x8f|\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcV\xcfo+5\x10\xbe\xe7\xaf\x18\x89+\xbb\xa1B \x94\x1b*\x1c*\xe0\xa9j\x9ezw\xbc\x93d\xa8\xd7^f\xc6)A\xfc\xf1\xc8\xf6n\x9b\xee:\xb4\x8f\x03\xbe\xad\xed\xf9\xe6\x9bo~x\x9b\xa6Y\x99\x81\x1e\x91\x85\x82߀\x19\b\xffT\xf4\xe9Kڧ\x1f\xa4\xa5\xb0>ݬ\x9e\xc8w\x1b\xb8\x8d\xa2\xa1\x7f@\t\x91-\xfe\x84{\xf2\xa4\x14\xfc\xaaG5\x9dQ\xb3Y\x01\x18\uf0da\xb4-\xe9\x13\xc0\x06\xaf\x1c\x9cCn\x0e\xe8ۧ\xb8\xc3]$\xd7!g\xf0\xc9\xf5\xe9\x9b\xf6\xe6\xfb\xf6\xbb\x15\x807=n@\x90ә\x1a\x8d\xc2\xf8GDQiO\xe8\x90CKa%\x03ڄ\x7f\xe0\x10\x87\r\xbc\x1e\x14\xfb\xd1w\xe1\xbd\xcdP\xdb\f\xf5P\xa0\xf2\xa9#\xd1_\xae\xdd\xf8\x95\xc6[\x83\x8bl\\\x9dP\xbe \xc7\xc0\xfa\xe9\xd5i\x03\"\\N\xc8\x1f\xa23\\5^\x01\x88\r\x03n \xdb\x0e\xc6b\xb7\x02\x18\x05\xc9Xͨ\xc5\xe9\xa6\xc0\xd9#\xf6\xa68\x01\b\x03\xfa\x1f\xef\xef\x1e\xbfݾ\xd9\x06\xe8P,ӠYֿ\x9b\x97}\xa8\x85\t$``\xa4\x04\x1a\xc0X\x8b\"`#3z\x85B\x19\xc8\xef\x03\xf79\xad`v!\xea\x05\xaa\x1e\x11\x1e\xb3\xfec\x98\xed\xcb\xe1\xc0a@V\x9a\xa4)\xeb\xa2\xe2.v\xff\x8dxZ)\xd6b\x05]*=\x94\xecy\xd4\v\xbbQ\x1e\b{\xd0#\t0\x0e\x8c\x82\xbe\x14c\xda6\x1e\xc2\xeew\xb4\xdaΠ\x8b.\x922\x19]\x97*\xf6\x84\xac\xc0h\xc3\xc1\xd3_/ؒ\x04JN\x9dѬ\x9dWdo\x1c\x9c\x8c\x8b\xf85\x18\xdf͐{s\x06\xc6\xe4\x13\xa2\xbf\xc0\xcb\x062\xe7\xf1[`\xccRo\xe0\xa8:\xc8f\xbd>\x90N}hC\xdfGOz^疢]\xd4\xc0\xb2\xee\xf0\x84n-th\f\xdb#)Z\x8d\x8ck3P\x93\x03\xf1\xb9\x17۾\xfb\x8a\xc7Ε7n\xf5\x9cjP\x94\xc9\x1f.\x0er\xeb|AzR#\x95b*P%\xc4\xd7,\xa4\xad$\xdd\xc3\xcf\xdb\xcf01)\x99*Iy\xbd\xba\xd0e\xcaOR\x93\xfc\x1e\xb9\xd8\xed9\xf4\x19\x13}7\x04\xf2\x9a?\xac\xa3\\\xb8qד\xcaT\xda)us\xd8\xdb<\xab`\x87\x10\x87\xce(v\xf3\vw\x1enM\x8f\xee\xd6\b\xfeϹJY\x91&%\xe1Cٺ\x9c\xc0\xf3\xcbEދ\x83iv^ImeJl\a\xb4)\xb9I\xdfdM{\xb2\xa5\xad\xf6\x81\xc1\xd4L\xda\x0f1\xc9\x16_\xc8e\x9cH\x85\xcdlN\xa5.\x7f\x9fM},哣\x11\x9co\xce8ݧ;s\xff\x8e\xf6h\xcf\xd6a\x81(S\bߧ\x92\x16\xfa\xd8/}6\xf0\t\x9f+\xbb\xf7\x1c҄\xc6\xf9\xa8\xb9Z\x1bP\x1e\xb1\x03\xf9E\xb8\xf3\xc8ʭ\xfc0.G~\x0eh\x04\x02\x8eާ\x96\x0e~\x01Yy\x11\x16wH\xb1\xaf\xb0\xa9\xf2\xb9\xf3\xfb\x90\xff\"Lrl\xb4\xb4\x13\x8e\xc9\x1e\xfd\x14^\x15\xc0\xeb\xb9.k9\xe7>$hY\xf9y\xfeo\xc6i.\x11c\xd5w\x93YU\x0f\x92ǚ\xe2\xf5\xfe\x1aYF\xe7\xcc\xce\xe1\x06\x94\xe3Һ\xd8\x1afs\x9eW\xcdTj\x9f\xa9GQ\xd3\x0f\xef\x14\xd0\xe2UH\xeb~\x81\x92\x9a\xe7\xf9\x88\xfeZ\x8b\xc0\xb3\x91W\xe7\x15\xc8\xdd\xf9\x9a\xe9\xed\xcb\xdf\xe6\xb2\xcfJ=o \xcd\xfaF\xa9\"䇔\xaa\xa6\xb4\xd4y\xf5\xb7f\xa1\xd2\xf6\xf2\xee4H\xde\xf4\xcb\xf4W\xb3\x8c\xe1*\x85j\x05,63|w\x11\x9eh`s\x98\x02\xfe'\x00\x00\xff\xff\xef\xf8\xa6>\x10\f\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcVM\x8f\xdb6\x10\xbd\xfbW\f\x92kd7(Z\x14\xbe\x05\xdb\x1e\x82&\xc5\"N\xf7N\x93#{j\x8ad\x87C9.\xfa\xe3\v\x92Ү-\xcb\xc9nQT\x17\xc3\xe4\xf0q>\u07bca\xd34\v\x15\xe8\x019\x92wkP\x81\xf0\x8b\xa0\xcb\xff\xe2\xf2\xf0S\\\x92_\xf5o\x17\arf\rw)\x8a\xef>a\xf4\x895\xfe\x8c-9\x12\xf2nѡ(\xa3D\xad\x17\x00\xca9/*/\xc7\xfc\x17@{'\xec\xadEnv薇\xb4\xc5m\"k\x90\v\xf8xu\xff\xdd\xf2\xed\x8f\xcb\x1f\x16\x00Nu\xb8\x86\xde\xdb\xd4at*Ľ\x17\xebu\xc5\\\xf6h\x91\xfd\x92\xfc\"\x06\xd4\xf9\x8a\x1d\xfb\x14\xd6\xf0\xb4Q!\x86\xeb\xab\xeb\x0f\x05m3\xa0}\x18Њ\x81\xa5(\xbf~\xc5\xe8\x03E)\x86\xc1&V\xf6\xa6g\xc5&\xee=\xcboO\xb77\xd0G[w\xc8\xed\x92U|\xeb\xfc\x02 j\x1fp\r\xe5xP\x1a\xcd\x02`\xc8O\x81k\xc6Լ\xad\x88z\x8f\x9d\xaa\xf7\x00\xf8\x80\xee\xdd\xfd\xfb\x87\xef7\x17\xcb\x00\x06\xa3f\nR\xb2<\x1f\"P\x04\x05\xa3'p\xdc##<\x94|B\x14\xcf\x18\a\xa7\x1fA\x01F\xff\xe3\xf2q1\xb0\x0f\xc8Bc\xf0\xf5;\xe3\xd7\xd9\xeaį\xbf\x9b\x8b=\x80\x1cJ=\x05&\x13\r#\xc8\x1e\xc7t\xa0\x19\xa2\a߂\xec)\x02c`\x8c\xe8*\xf5\xf2\xb2r\xe0\xb7\x7f\xa0\x96\xe5\x04z\x83\x9car\xad\x925\x99\x9f=\xb2\x00\xa3\xf6;G\x7f=bG\x10_.\xb5J0\n\x90\x13d\xa7,\xf4\xca&|\x03ʙ\tr\xa7N\xc0\x98\xef\x84\xe4\xce\xf0ʁ8\xf5\xe3\xa3g\x04r\xad_\xc3^$\xc4\xf5j\xb5#\x19\xbbN\xfb\xaeK\x8e\xe4\xb4*\rD\xdb$\x9e\xe3\xca`\x8fv\x15i\xd7(\xd6{\x12Ԓ\x18W*PS\x02q\xb5K:\xf3\x9a\x87>\x8d\x17\xd7\xca)S,\n\x93\u06ddm\x94.yAyr\xc3T\xd6T\xa8\x1a\xe2S\x15\xf2RNݧ_6\x9fa\xf4\xa4V\xaa\x16\xe5\xc9\xf4*/c}r6ɵ\xc8\xf5\\˾+\x98\xe8L\xf0\xe4\xa4\xfcі\xd0\tĴ\xedH2\r\xfeL\x18%\x97n\n{W\x94\t\xb6\b)\x18%h\xa6\x06\xef\x1dܩ\x0e흊\xf8?\xd7*W%6\xb9\bϪֹ\xdeN\x8dkz\xcf\x1bu\x90\xc9\x1b\xa5\x9dW\x84M@}\xd1x\x19\x85Z\x1a\x14\xa2\xf5i\x8b\x15\x10|;ý\x17\xb9\x9c?t\xa9\x9b#\xe2\xbb^\x91U[{-\t\r\xfc\xee\xd4\xcdݛş\xad\xe7\xd5b̏=\xb3\x06\xe1T\xb1\a\x96\r+\xff\x04\x00\x00\xff\xffNy\xc1Q\xa1\x0e\x00\x00"), -} - -var CRDs = crds() - -func crds() []*apiextv1.CustomResourceDefinition { - apiextinstall.Install(scheme.Scheme) - decode := scheme.Codecs.UniversalDeserializer().Decode - var objs []*apiextv1.CustomResourceDefinition - for _, crd := range rawCRDs { - gzr, err := gzip.NewReader(bytes.NewReader(crd)) - if err != nil { - panic(err) - } - bytes, err := io.ReadAll(gzr) - if err != nil { - panic(err) - } - gzr.Close() - - obj, _, err := decode(bytes, nil, nil) - if err != nil { - panic(err) - } - objs = append(objs, obj.(*apiextv1.CustomResourceDefinition)) - } - return objs -} diff --git a/config/crd/v1/crds/doc.go b/config/crd/v1/crds/doc.go deleted file mode 100644 index 9eed410f6..000000000 --- a/config/crd/v1/crds/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Package crds embeds the controller-tools generated CRD manifests -package crds - -//go:generate go run ../../../../hack/crd-gen/v1/main.go diff --git a/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml b/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml index 36ab864f9..71e662fe8 100644 --- a/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml +++ b/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml @@ -83,6 +83,34 @@ spec: Cancel indicates request to cancel the ongoing DataDownload. It can be set when the DataDownload is in InProgress phase type: boolean + csiSnapshot: + description: CSISnapshot provides the information of the CSI snapshot + used to do the incremental restore. + nullable: true + properties: + driver: + description: Driver is the driver used by the VolumeSnapshotContent + type: string + snapshotClass: + description: SnapshotClass is the name of the snapshot class that + the volume snapshot is created with + type: string + storageClass: + description: StorageClass is the name of the storage class of + the PVC that the volume snapshot is created from + type: string + volumeSnapshot: + description: VolumeSnapshot is the name of the volume snapshot + to be backed up + type: string + volumeSnapshotNamespace: + description: VolumeSnapshotNamespace is the namespece of the volume + snapshot to be backed up + type: string + required: + - storageClass + - volumeSnapshot + type: object dataMoverConfig: additionalProperties: type: string @@ -92,7 +120,7 @@ spec: datamover: description: |- DataMover specifies the data mover to be used by the backup. - If DataMover is "" or "velero", the built-in data mover will be used. + If DataMover is "" or "velero", the built-in fs data mover will be used. type: string nodeOS: description: NodeOS is OS of the node where the DataDownload is processed. @@ -106,6 +134,9 @@ spec: OperationTimeout specifies the time used to wait internal operations, before returning error as timeout. type: string + restoreType: + description: RestoreType indicates the type of the restore. + type: string snapshotID: description: SnapshotID is the ID of the Velero backup snapshot to be restored from. @@ -145,6 +176,7 @@ spec: required: - backupStorageLocation - operationTimeout + - restoreType - snapshotID - sourceNamespace - targetVolume diff --git a/config/crd/v2alpha1/bases/velero.io_datauploads.yaml b/config/crd/v2alpha1/bases/velero.io_datauploads.yaml index 15682739b..a3d7dbe80 100644 --- a/config/crd/v2alpha1/bases/velero.io_datauploads.yaml +++ b/config/crd/v2alpha1/bases/velero.io_datauploads.yaml @@ -110,6 +110,10 @@ spec: description: VolumeSnapshot is the name of the volume snapshot to be backed up type: string + volumeSnapshotNamespace: + description: VolumeSnapshotNamespace is the namespece of the volume + snapshot to be backed up + type: string required: - storageClass - volumeSnapshot @@ -124,13 +128,20 @@ spec: datamover: description: |- DataMover specifies the data mover to be used by the backup. - If DataMover is "" or "velero", the built-in data mover will be used. + If DataMover is "" or "velero", the built-in fs data mover will be used. type: string operationTimeout: description: |- OperationTimeout specifies the time used to wait internal operations, before returning error as timeout. type: string + parentSnapshot: + description: |- + ParentSnapshot specifies the parent snapshot that current backup is based on. + If its value is "" or "auto", the data mover finds the recent backup of the same volume as parent. + If its value is "none", the data mover will do a full backup + If its value is a specific snapshotID, the data mover finds the specific snapshot as parent. + type: string snapshotType: description: SnapshotType is the type of the snapshot to be backed up. @@ -185,8 +196,12 @@ spec: nullable: true type: object incrementalBytes: - description: IncrementalBytes holds the number of bytes new or changed - since the last backup + description: |- + IncrementalBytes holds the number of bytes new or changed since the last backup. + A nil value means the uploader did not report a figure; a pointer to 0 means it + reported zero, i.e. nothing changed and nothing was transferred. The two are + distinct: erasing a measured zero makes a perfect incremental indistinguishable + from a full transfer in every downstream report. format: int64 type: integer message: diff --git a/config/crd/v2alpha1/crds.go b/config/crd/v2alpha1/crds.go new file mode 100644 index 000000000..111b68cdc --- /dev/null +++ b/config/crd/v2alpha1/crds.go @@ -0,0 +1,58 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package crds embeds the controller-tools generated CRD manifests from +// ./bases into the binary via go:embed. +package crds + +import ( + "embed" + + apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/client-go/kubernetes/scheme" +) + +//go:embed bases/*.yaml +var basesFS embed.FS + +var CRDs = crds() + +func crds() []*apiextv1.CustomResourceDefinition { + apiextinstall.Install(scheme.Scheme) + decode := scheme.Codecs.UniversalDeserializer().Decode + + entries, err := basesFS.ReadDir("bases") + if err != nil { + panic(err) + } + + objs := make([]*apiextv1.CustomResourceDefinition, 0, len(entries)) + for _, entry := range entries { + data, err := basesFS.ReadFile("bases/" + entry.Name()) + if err != nil { + panic(err) + } + + obj, _, err := decode(data, nil, nil) + if err != nil { + panic(err) + } + objs = append(objs, obj.(*apiextv1.CustomResourceDefinition)) + } + + return objs +} diff --git a/config/crd/v2alpha1/crds/crds.go b/config/crd/v2alpha1/crds/crds.go deleted file mode 100644 index 59af9e6f0..000000000 --- a/config/crd/v2alpha1/crds/crds.go +++ /dev/null @@ -1,60 +0,0 @@ -/* -Copyright the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by crds_generate.go; DO NOT EDIT. - -package crds - -import ( - "bytes" - "compress/gzip" - "io" - - apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" - apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/client-go/kubernetes/scheme" -) - -var rawCRDs = [][]byte{ - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcYK\x93\xe3\xb6\x11\xbeϯ\xe8\xda\x1c\xf6\xb2\xd2d\xf3p\xa5t\xdb\xd1\xc4US\xf1Ϊ\xac\xc9\xdcA\xb2E\xc1\v\x02\b\x1e\x92\xe5$\xff\xdd\xd5\x00I\x81$4z\xd8^\xdd\x044>|\xe8n\xf4\x03\x9c\xcdfwL\xf3W4\x96+\xb9\x00\xa69\xfe\xecP\xd2?;\xff\xfa\x0f;\xe7\xea~\xf7\xf1\xee+\x97\xd5\x02\x96\xde:\xd5\xfc\x88VyS\xe2#n\xb8\xe4\x8e+yנc\x15slq\a\xc0\xa4T\x8eѰ\xa5\xbf\x00\xa5\x92\xce(!\xd0\xccj\x94\xf3\xaf\xbe\xc0\xc2sQ\xa1\t\xe0\xddֻ?\xcf?~7\xff\xfb\x1d\x80d\r.\x80\xf0*\xb5\x97B\xb1\xca\xcew(Ш9WwVcI\xc0\xb5Q^/\xe08\x11\x17\xb6\x9bF\u008f̱\xc7\x16#\f\vnݿ&S?p\xeb´\x16\xde01\xda;\xccح2\xee\xf9\x88?\x83*\"Z.k/\x98\x19.\xba\x03\xb0\xa5Ҹ\x80\xb0F\xb3\x12i\xac=l\xc0\x98\x01\xab\xaa\xa0>&V\x86K\x87f\xa9\x84o\xe4q\a\xb4\xa5\xe1\xda\x05\xf5\xa4|\xc1:\xe6\xbc\x05\xeb\xcb-0\vϸ\xbf\x7f\x92+\xa3j\x836\xf2\x05\xf8\xc9*\xb9bn\xbb\x80y\x14\x9f\xeb-\xb3\xd8\xceF\x1d\xaf\xc3D;\xe4\x0e\xc4\xd7:\xc3e\x9dc\xf0\xc2\x1b\x84ʛ`[:w\x89\xe0\xb6\xdc\x0e\xa9\xed\x99%z\xc6au\x92H\x98'8\xebX\xa3nj\x92\xa5\x91R\xc5\x1c\xe6\b-U\xa3\x05:\xac\xa088쎱Q\xa6an\x01\\\xba\xef\xfevZ\x17\xad\xb2\xe6a飒C\xc5<\xd0($Ñ\tY\xa9F\x93ՎrL\xfc\x16\"\x8e\x00\x1e\x92\xf5\x91I\xc4M\xc7\xcfR!\x97\x03\xb5\x01\xb7Ex`\xe5W\xafa\xed\x94a5\xc2\x0f\xaa\x8c\xe6\xdbo\xd1`\x90(\xa2\x04y/p\xb2\x9d2Y\xd3i,\xe7Q\xb6\x05\xeb\xb0F\xf6\x1bn\xf4\xbb\xfbVi\x90e}\xab\x8bA\xf3 \xc1\x95\xcc;ا\x1a/r\xaeT\x89RU\x98hl\xc0\x89[\xd0F\x95h\xed\x1b\x0eO\x00\x03\x16\xcfǁ\x89j\xa2\xc4\xee/L\xe8-\xfb\x18\x83L\xb9ņ-\xda\x15J\xa3\xfc\xb4zz\xfd\xebz0\fo\x04\fV:K\x91\x82\xe8k\xa3\x9c*\x95\x80\x02\xdd\x1eQF\xd37j\x87\x86\x02`ͥ\xed\x11)\x9cW\xa9\xc01\x98\x93\x7f\a<\x9a\x8d\x93\x06\x83\xf7\x10A\x93Z\x1fhO\x8d\xc6\xf1.|\xb6\xd8\xc7̓\x8c\x8e\xce\xf1\xbf\xd9`\x0e\x80\x8e\x1eWAE)\b\xe3\xb1\xda؊U\xab\xadhn\xa1\x94@6\xd6\"y\xe1gJ\vK%7\xbc\x9e\x1e<-\x7fO\xb9\xc8\x19\x9df\x1c6ْNA\xdeILf!C\xcd:ץо\xe1\xb57\xa7\xec\xbf\xe1(\xaaI\xfc9y\x93\xba\x03\x87]n\xb1qO\xbd\xbb]mVKR\xafS!B\xd9P\xef&\xae9%\t\xf0\xb4I\x10\xb9\x85w\xef@\x19x\x17\x9b\xa5w\x1f\xe2jυ\x9b\xf1A\xfe\xdfs!\xba]\xae\xf2n\xaap\xbe\xacϜ\xfc9\b\x11\x9f/\xebkk\xab)\x1b\x94\xbe\x99n8\x03\xe6\x9d\xca\f\v.\xfdϙ\xf1=\x97\x95\xda\xdbk\x0e\xdb\xd77Tb*\xefn1\xf8\x97\x11\xc6\xc8\xee\x8e\n\xe2`k\xa7`\xcfxRc\xf4\xbb\xdb\x0f\x19\xdc\x027\x94\x90\f:o$\x85\x034\x86\"\xb4\r\x90\xcaOj\x9e7Oj%\xd3v\xab\xdc\xd3\xe3\x993\xae{\xc1.\xee>=v&~\r^\xd7\a\xdfV\x122V\"\xfa]\x15Y\x85\xb4~\x13\xdb5\xff\x05/\xe4K\xa2\x1dc\xa1j^2\x016\x8cɶ\tl\x0f\xd1aO\t\xe5\xfa\xbc1ݴ[K\xf8\x86ڧ\x7f!\xb8ō\xd6C\x88\xee(\xca\U0001a4f3\xc8~\xe6x\xc7vJ\xf8&\x88\x92I\xb0\x02\xafO\xe8\x1a(}P\xb1U T|\xb3AC\x15U(\xb7\xe2ƫ\xd7\xe5{\x9bl\xc27\xe9\x1f\xcaT\r\xd3\x1a+\xea\xed\xc8\x19[\xdb^eU\xc7L\x8d\xee5\x90>\xa3\xa2\x97D\xb4S\x05\x95fd\xa0\xb6\xf6\x0f\x97+\x88\xc1\xeau\x99\xa9\xd4\xe9\xb7z\x9d2<]\xc7\xd0oc_\xe8\x04\x99\x99\x11\xc5\xef\xd7$ؑ\xdbp\x81`\x0f\xd6a\x13T0b\x18-\x95\xb3˙\xb4\bG3\\\xc0i\xe2>\xed\xf6=\xc6-\x04\xf4\ue09dW\xaf\xb92\xad\xb7\x0f\xb8-s$\xd1v\xfdP\x1c\xb2\x98\xd0Řֿn\xe3[^Dx\xf9&\xe3\xe5\x98\xf2\t\xbe\xc5\xe17S\xa6*\x90\x1b\xacr9\xf0\xb4\xe5f\xa0w\xd9\xc1\xf2\xf2Z'\xbf\xf3,_Џdƹs4}L8\xe3\x89a\xa0\x1bͦ1\xe2\xa2\xce'\xbc\xcb\\\xda\xfb\xc4\xd7\xd6\xd6\xec\xa57!\n\xb6o\xb0jsc\xf7\xc3\xca\x12\xb5\xc3\xea\xe1@e\xd1\x05\x95\x13\x11\x90o\xbfJ\xfd[\x1f\xeb&\xd4\xec\xda\x16\xa5\xa3Կ\x9cݒ\x91>\x8dA\xc2\U000c9a52\xbafJ7ֶ\xa7I\x03\xbcP\x0e\x0e\xed\xff\xfbX\xcaвP Q\x89?\xd9\xf4d\x96\xa6\xfe~F\xeb'\x12\xd2\v\xc1\n\x81\vpƟ\xeau\xf2\xad]|\x88N\xdf\x1co\xea\xf3\xa60Sݱ\xfe\x95-\xbc\x86vO\xe09\x95\x1d\xf1z\x85E8\xac\x00w(\x81\xbaw\xc6\x05V\x1df\xa6\xe19\xa7\xf9\f\xe9i-\xfdG*\xbfAkY}\xee\x02}\x8eR\xf1a\xaa]\x02\xac\xa0\xc2{\xdcv\xbc\xb7\xedݾ\xba\x01\xfa}.\xf1\x85\xed\xcf\x1b\\B\xb3~\x86̊dr1\xad\xa7v:\xa8\xc1\x1b\xdd\xd73\xee3\xa3\xdd\xfd\xccL\xad\xdaK\x9f\x99\x9a|\xd3J'\xe3\xabH.1vsY\xcc\xfe\xa3Qf\xee\xfbp\x19\xae\xd2t\xcb\xef\x96\xeb\u07bf\xadl\x95\xe8nx\xf8\xd8#}S\xa0!3\x14\xb9\x0e$<\xc9'V\xcb\x15\x7f=B\xdfL\x05\xa89\xbcl\xa94\x89\x0fB]{Yq\xab\x05;\xf4\x87IK\xe6\f\xf8\xf1\xd6L\xde\xfb\xaf\xad\x9a\xfb\x8fo\xf9\xca\xeb\xed\xce\n\xcetWa\xbe\xff\xa8\xf6\xc7\xec\xf0\xc6s\xd0\xf0#\xe7M\xbd\xdd\x00\xe1\\*h?\xba^\x1f\xc1\x87\xdb|\xcb\xe0\x9d\xd5\xded00\xaf\x12\xec\xf6\xf96\x1d\xf1E\xffMc\x01\xff\xfd\xffݯ\x01\x00\x00\xff\xff];\x85{\xd8 \x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcZIs\xe36\x16\xbe\xfbW\xbc\xea9\xe4b\xc9\xe9YRS\xba\xb5\xe5I\x95j\xd2nW\xcb\xe3;D>\x89\x88A\x80\x83E\x8af\xf9\xef\xa9\ap\x01IH\x94\x94Nx\xe8jcyx\x1b\xbe\xb7@\xb3\xd9\xec\x8eU\xfc\r\xb5\xe1J.\x80U\x1c\x7f\xb1(\xe9/3\x7f\xff\xbb\x99s\xf5\xb0\xffx\xf7\xcee\xbe\x80\xa53V\x95_\xd1(\xa73|\xc2-\x97\xdcr%\xefJ\xb4,g\x96-\xee\x00\x98\x94\xca2\x1a6\xf4'@\xa6\xa4\xd5J\bԳ\x1d\xca\xf9\xbb\xdb\xe0\xc6q\x91\xa3\xf6ě\xa3\xf7\xdf\xcf?\xfe0\xff\xdb\x1d\x80d%.\x80\xe8\xb9J(\x96\x9b\xf9\x1e\x05j5\xe7\xea\xceT\x98\x11ٝV\xaeZ@7\x11\xb6\xd5G\x06v\x9f\x98e\xff\xf2\x14\xfc\xa0\xe0\xc6\xfes0\xf1\x137\xd6OV\xc2i&z\xa7\xfaqS(m\x9f;\xca3\xc8]\x98\xe0r\xe7\x04\xd3\xf1\x96;\x00\x93\xa9\n\x17\xe0wT,C\x1a\xabE\xf4\x14f\xc0\xf2\xdc+\x8d\x89\x17ͥE\xbdT\u0095\xb2\xa3\x8f&Ӽ\xb2^)\x1d\xa7`,\xb3\u0380qY\x01\xcc\xc03\x1e\x1eV\xf2E\xab\x9dF\x13x\x05\xf8\xd9(\xf9\xc2l\xb1\x80yX>\xaf\nf\xb0\x9e\rz]\xfb\x89z\xc8\x1e\x89[c5\x97\xbb\xd4\xf9\xaf\xbcDȝ\xf6\xf6$\x993\x04[p\x133v`\x86\x98\xd3\x16\xf3\x93l\xf8y\"f,+\xab!?\xd1\xd6\xc0P\xce,\xa6\xd8Y\xaa\xb2\x12h1\x87\xcd\xd1b#\xc4V\xe9\x92\xd9\x05pi\x7f\xf8\xebiMԪ\x9a\xfb\xadOJ\xf6\xd5\xf2H\xa3\x10\r\aN\xc8B;\xd4I\xdd(\xcb\xc4oa\xc4\x12\x81\xc7h\x7f\xe0$Ѝ\xc7'YY\xc9Lc\x89\xf26\x86x\xb7{\xccML:\x9e\xad4W\x9a\xdb\xe3\x02>~\x7f)\x9bt+@m\xc1\x16\b\x8f,{w\x15\xac\xad\xd2l\x87\xf0\x93ʂ\x8f\x1d\nԵ\x8fm\xc2\x12S('r\xd84\x86\x010V餳U\x98\xcdî\x9anCv\xe0q\xfd3\xbf\xf1]\xc84\xb2\xe4]hPr\xeeWp%\xd3\x17\xe2\xd3\x0e/\xba\f\xb16\xa5ʱU\x1d\xc6\x1cq\x03\x95V\x19\x1as\xe6z\xd2\xf6\x1e\x0f\xcf\xdd\xc0H-a\xc5\xfe\xcfLT\x05\xfb\x18\xc00+\xb0d\x8bz\x87\xaaP~zY\xbd\xfde\xdd\x1b\x86\x93\xd0\xc62k\bӈ\xf5J+\xab2%`\x83\xf6\x80(=\xbcB\xa9\xf6\xa8\t\xa4w\\\x1a`2oiB\xbc\xa0\v5\xe4\xfa\x9e\x1e͆\xc9ڝT\x85:6;\xb92\x8dY\xde`|\xf8\xa2\xb0\x18\x8d\x0e\x84\xf8߬7\a@r\x87]\x90S|\xc4 U\x1d\x020\xafU\x15\xec\xc6\rh\xac4\x1a\xba^ޫ\xd4\x16\x98\x04\xb5\xf9\x193;\x1f\x90^\xa3&2\xcd}Ȕܣ\xb6\xa01S;\xc9\xff\xd3\xd26`\x95?T0\x8b\xc6\xfa\v\xa9%\x13\xb0g\xc2\xe1\xfd@{\xf4\x95\xec\b\x1a\xe9Lp2\xa2\xe77\x98!\x1f\x9f\x95F\xe0r\xab\x16PX[\x99\xc5\xc3Î\xdb&Y\xc8TY:\xc9\xed\xf1\xc1\x1b\x83o\x9cU\xda<\xe4\xb8G\xf1`\xf8n\xc6tVp\x8b\x99u\x1a\x1fX\xc5g^\x10\xe9\x13\x86y\x99\xffI\xd7\xe9\x85\xe9\x1d;\xf2\xc2\xf0\xf9@\x7f\x85y(\xfeӕ`5\xa9 bg\x05\x1a\"\xd5}\xfd\xc7\xfa\x15\x1aN\x82\xa5\x82Q\xba\xa5#\xbd4\xf6!mr\xb9E\x1d\xf6m\xb5*=M\x94y\xa5\xb8\xb4\xfe\x8fLp\x94\x16\x8c۔ܒ\x1b\xfcۡ\xb1d\xba!٥O\xa8`\x83\xe0*\x82\x82|\xb8`%a\xc9J\x14Kf\xf0\x0f\xb6\x15Y\xc5\xcc\xc8\b\x17Y+N\x13\x87\x8b\x83z\xa3\x89&\xd3;a\xda\x0e>\xd6\x15fdSR+m\xe2[^\xc7\x12\xc2\x00\x16\xad\xeck'}\xed\xe9K\x86\x90\xe1\xa2)W\xa3\xef1E\xa8\xe1UF\xf8݄\xba:2\x89~d\x8a\xbf\x0e\xe4\xeb=\x1a+e\xb8U\xfaH\x84Ch\x1c\xba\xc1I\x8bЗ1\x99\xa1\xb8E\xbc\xa5\xdf\t\\\xe6\xa4qlݘ\x00(P\xf5\x8c*\xb9St\xb1\"C\xc0\xca\xd2\n\xf2j\x836-\xa6L\x842.\xa1Kz!Nn\x87\xa2n\x94\x12Ȇ\x1a\xcc\f_KV\x99B\xd9\t\x81W[hV\xbe\x1e+\xa4×\xeb\xd5=\xfdӌ\x93\a\xedy^C<\xdd2ʶ\xd2f\xab\xed\xbc\\\xaf\xc0\xd4\xdb\xc7F\x92N\b\xb6\x11\xb8\x00\xab\xddX\xb0\xd3\x0e\xeb\xb9\xd7|\x8f:53\xbc9~a\xe3\x85a\x1b8\xe3\x93j?\xf4F\x05\t6R.\x95\xb4(S6:\xebU\xf45\x92.\x053I\x9e\a\x9c\xad\xe3\xf5\xa9k\xd2\x10\x84̯\xb0\x05K\xf3\x05!\xe8z9\xbaM\xbc\xcd\xcd\xe0\xc0mq\x93D\xe1\x82^,P\xb4<)O}߃8j{F\x98\x97\xb7\xa5\x97wJ2\n7\xb7H\xb6\xef\x19\xfd\x02\xd9\xfa^\x92\x92n\xc0\xe5)\xe1\x14\xa1\x00\x81\x19\xe6\xe0\xaa\xeby'\xd0\xe1\x1a\xf31ϳ\x9e\xbd\x12\xd3}\xa1O \xc9(2A\x9dt~\xa6\xb4r\xa9\xe4\x96\xef\xc6g\xc7e\xfe\xb9k{V\xb4Qċ\x8e$\x8dS\x80#Nf>Ý5яr\xc3-\xdf9}\n\x8d\xb6\x1cE>J`&\x01hB\x1f\x9e\x89[\xe2H+Y\x13\xbfkH\x8d2\xfb\xe0%1J\x85\xf07\x96\x01\b\xba;\x8a\xdc\xc0\x87\x0f\xa04|\b\xbd\xa2\x0f\xf7a\xb7\xe3\xc2\xcex\xaf\xbc8p!\x9aS\xae\x8a\xa0mIA\x05\x9drS\xa1%\xa9\x83/\x03\x1a\x03UX*>\xbd\xf8V\xc1\x81\xf1(\xadoO7\xf7\t\xba\x1b\xdcR\x0e\xa8\xd1:-)\n\xa3֔\x16\x19OR\xb9D\x18:#\xa9\x89B℔\xc3\xe8饠\xff\x0f\xb1<\x06\x80\x84\x00)\x1b\x9f\xe3Ч\xec?\xae/\xe10Z\xdap\xb8\xe5\x02\xc1\x1c\x8dŲ\xcfm\xa8\x04\x02`\xdc\xc0P\xdb\x10\xbc\xc57\xd6}\x12\r\xafJ\xf3\x1d'\x0f\x90\xedL\x97\x1d\xd6\xe0[\xb7Q<\xb4\xfaؐ\xbc0-|\x1b\x82\xef\x8e\x1c\xe1K8\x9c\xc2\x0f\x93\xb9O`\xda\xf9\xbcƂ\x04\x92L*\xe4\xe5my\x91y\xe8\xe0Dl\xa1\xe1C\xc1\xb3\xa2\xefK|\x8c\xf2\x00\x96\xbd\xa3/\x06\xae`3\x1dTf\xe9\xd2`\xb0f\b\a\x83\xe9\xf8\x0e\r\xa7\xfa\x86Nξ\xbc-/*\x9f|g\xe7\xb2\x02*t\x96k-gNk_\x9a\x86Q\xb5\xbd\xa9\x84bY\x86\x95\xc5\xfc\xf1\xf8\xac\xf2)\xa7\xff\xd4[L\x8c\xc8Kz[\tS\xfbn\x17V\xec\xda\x1a\xa8a\xb7\xed\xc8\xddrM?\r\x89\xf8ތ\xce#\x04\x1fW4\x01\xfdN3\r\xf0J\x0e\xee{\v\xdf\x05Цm>\x14\xd0\xf5\x1c\x1d:\xa2\xd04\x81sfqF\xfbo\v\xfb\xe9\xda14\xe4\xe3^\xe6M\x85\xe4\x98\xccXw\xac\xa9x}\x93\xb5y\tHi\xac#\xd7\xea+P\xc3\x1cp\x8f\x12\x94\x84-を\tO2\x01`\xe7\xa9\xd4Q5<\xfb4M\xa3\xa6\xc1\x98\xec\xdeM[2\xa1\x841\x9a\xfd\x9e\xc6lsگh\x9cHd1\xbfcN\x1b\x8e\f\xed\v\x93\xcci\xcf\xd7\xd7\xcc\x00\x03\x1d\x88Ըq\n\xb4.VR2\xd1\x1d>\x96L\xb5\x11\x06ˡP\xa2vj\xe9\xca\rj\xe2\xd6?ـ\xc4\x03\xe5\xa9Y\xc1\xe4.\x99\t5O\x0e\b\x82\x19[\xbb\xdbI\x0f\x89\xdf|\x86\x92\xc5o4\xddW\xa21l7\x05֟ê\xd0E\xad\xb7\x00\xdbP\xca\xda\xd7\xfaw\xa6\x8e!W!\xb1\x9c\x0e\x17W\x05\x89\xde\x03\xc8՜|Y_\xc0˗5\x1d\xf2e\xfd[yA\xe9\xcaT\x11˜U\x89a\xc1\xa5\xfb%1~\xe02W\x871t\x9c\x11\xb5b\xb6\x98\x10\xf4\x85٢M\x92\x9d\x10~\xcf(\x97\xaf\xb3\xce\r\x12&~\xab\x94\u07b7\xf9\xa6أ5\xa9\x14\x06/\x81\x83S\x9a\x7f\xc6Cb\xb4\t\xb9\x89\xa9\x97:\x8e'\xa6F\x8f\xf5\xf1d褦ಙK\xd2l\xdf\xc3\x13s?\xfa\x00w\x95\x9ek\xfen\x89\xe0mO\xb6\xc37\xff\xbc=B\xb9~o\x88J\x8a\xc8b\t\xc2\xd1\xfe\xb6\x8e\xf1\x94\xe6\xf0Zp\xd3t\x91\x9b\xd28\xe7\xa6\x12\xec\xd8\xca2\x156Z\xdc\x1a\xbe\x0e\x8e\x9d\xe4|\xfb\xb5\xfdUA\xbauv\x1e\x95a\x02\x99\xfd\xbc:\x1dr\xbe\xc5\tgb^s\xbdWO\x17\xd6\xfc\xab\xa7\xe6*\xf2\x1c\xa5\xe5[\x1e\xbd\xc8vŚ\xef\xf0\xa7t9|ٸ\xae\xbe\xec\xfd\xd6\xe4\xa6z\xbbGa\"\x13\xad\x7f\xfa\x92\xca\xf7\xd6\x04\x06\x04A\xfe\rp9|\xf5\xbfo#:\xb3\xf5Cd\b\xfe\xa9\"VIJo|zt}j\xd9\x17\xe8\x8f\xcc*\x93^5\x1a\xf4\x9c\xe7\x11\xed\xbao\x1b\x8f\xb8M\xfb2\xbc\x80\xff\xfe\xff\xee\xd7\x00\x00\x00\xff\xffʖ\x89F\xbb&\x00\x00"), -} - -var CRDs = crds() - -func crds() []*apiextv1.CustomResourceDefinition { - apiextinstall.Install(scheme.Scheme) - decode := scheme.Codecs.UniversalDeserializer().Decode - var objs []*apiextv1.CustomResourceDefinition - for _, crd := range rawCRDs { - gzr, err := gzip.NewReader(bytes.NewReader(crd)) - if err != nil { - panic(err) - } - bytes, err := io.ReadAll(gzr) - if err != nil { - panic(err) - } - gzr.Close() - - obj, _, err := decode(bytes, nil, nil) - if err != nil { - panic(err) - } - objs = append(objs, obj.(*apiextv1.CustomResourceDefinition)) - } - return objs -} diff --git a/config/crd/v2alpha1/crds/doc.go b/config/crd/v2alpha1/crds/doc.go deleted file mode 100644 index 9eed410f6..000000000 --- a/config/crd/v2alpha1/crds/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Package crds embeds the controller-tools generated CRD manifests -package crds - -//go:generate go run ../../../../hack/crd-gen/v1/main.go diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index ea669c709..f8f27a521 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -4,6 +4,16 @@ kind: ClusterRole metadata: name: velero-perms rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - create + - delete + - get + - list - apiGroups: - "" resources: diff --git a/design/backup-filter-enhancement/fine-grained-backup-filters-design.md b/design/backup-filter-enhancement/fine-grained-backup-filters-design.md index 853d47fa1..93e3f3981 100644 --- a/design/backup-filter-enhancement/fine-grained-backup-filters-design.md +++ b/design/backup-filter-enhancement/fine-grained-backup-filters-design.md @@ -41,7 +41,7 @@ This creates three critical gaps for common backup scenarios: - Maintain full backward compatibility — existing backups with no `namespacedFilterPolicies` behave exactly as they do today - Define clear precedence rules for how per-namespace filters interact with global filters - Add corresponding validation within the Resource Policies validation pipeline using existing Velero wildcard validation functions -- Update `velero backup describe` output to display per-namespace filter information when present +- Update `velero backup describe` output to display the referenced ResourcePolicy ConfigMap name when configured - Ensure the restore process works correctly with backups produced by namespace-scoped filters, without requiring restore-side code changes in the initial phase ## Non-Goals @@ -77,7 +77,8 @@ clusterScopedFilterPolicy: names: ["my-app-*"] - kinds: [CustomResourceDefinition] labelSelector: - app: my-app + matchLabels: + app: my-app namespacedFilterPolicies: # NEW: per-namespace filter overrides - namespaces: @@ -85,7 +86,8 @@ namespacedFilterPolicies: resourceFilters: - kinds: [ConfigMap, Secret, Deployment] labelSelector: - app: my-app + matchLabels: + app: my-app - namespaces: - ns-b resourceFilters: @@ -93,7 +95,8 @@ namespacedFilterPolicies: names: [app-1, app-2] - kinds: [ConfigMap] labelSelector: - app: my-service + matchLabels: + app: my-service ``` All four sections coexist in the same ConfigMap. They are independent — `volumePolicies` handles volume backup strategy, `includeExcludePolicy` handles global resource type filtering, `clusterScopedFilterPolicy` handles cluster-scoped resource filtering by kind/name/label, and `namespacedFilterPolicies` handles per-namespace, per-kind overrides. @@ -107,7 +110,9 @@ namespacedFilterPolicies: - namespaces: [ns-a] resourceFilters: - kinds: [ConfigMap, Secret] # these kinds share a selector - labelSelector: {app: my-app} + labelSelector: + matchLabels: + app: my-app names: ["app-*"] - kinds: [Deployment] # this kind has its own selector names: [workload-1, workload-2] @@ -116,6 +121,24 @@ namespacedFilterPolicies: This model has one way to express filters — there is no ambiguity about how to structure the configuration. Only resource kinds listed in `resourceFilters` entries are included in the backup for the matched namespaces; unlisted kinds are implicitly excluded. +#### Label selectors (`matchLabels` / `matchExpressions`) + +`labelSelector` and each entry of `orLabelSelectors` use the standard Kubernetes selector shape (same as `BackupSpec.labelSelector`): + +```yaml +labelSelector: + matchLabels: + app: my-app + matchExpressions: + - key: environment + operator: In + values: [prod, staging] + - key: do-not-backup + operator: DoesNotExist +``` + +Supported `matchExpressions` operators: `In`, `NotIn`, `Exists`, `DoesNotExist`. Prefer `In` for value-OR on one key; use `orLabelSelectors` for OR across independent multi-key groups. `labelSelector` and `orLabelSelectors` cannot co-exist in the same `resourceFilters` entry. + #### Catch-All Resource Filter (Empty `kinds` or `["*"]`) A `ResourceFilter` entry with an empty (or omitted) `kinds` field, or a field explicitly set to `["*"]`, acts as a **catch-all**. Its `labelSelector` or `orLabelSelectors` (if provided) is applied to **all resource types in the namespace that are not already matched by a kind-specific filter entry**. If no selectors are provided, all unlisted resources are included. Using `["*"]` is highly recommended as it makes the catch-all intention explicit and self-documenting. @@ -319,9 +342,10 @@ resourceFilters: resourceFilters: - kinds: ["Pod"] labelSelector: - "invalid label key!": "value" # invalid key syntax + matchLabels: + "invalid label key!": "value" # invalid key syntax ``` -**Behavior:** Validation error during backup creation when `labels.SelectorFromSet()` fails: +**Behavior:** Validation error during backup creation when `metav1.LabelSelectorAsSelector()` fails: ``` namespacedFilterPolicies[0].resourceFilters[0]: invalid label selector: "invalid label key!" is not a valid label key ``` @@ -340,7 +364,33 @@ This is consistent with how other discovery-dependent features handle this error ## ResourceFilter Field Notes -**`labelSelector`** supports equality-based selectors only (`key=value`). Set-based requirements (e.g., `environment in (prod, staging)`) are not supported. To match resources with any of several label combinations, use `orLabelSelectors` with multiple maps — each map is AND-evaluated internally, and the maps are OR-evaluated across the list. `labelSelector` and `orLabelSelectors` cannot co-exist in the same entry. +**`labelSelector`** uses the standard Kubernetes shape: `matchLabels` (equality) and `matchExpressions` (set-based: `In`, `NotIn`, `Exists`, `DoesNotExist`). All requirements within one selector are AND-ed. Example: + +```yaml +labelSelector: + matchLabels: + app: my-app + matchExpressions: + - key: environment + operator: In + values: [prod, staging] + - key: do-not-backup + operator: DoesNotExist +``` + +**`orLabelSelectors`** is a list of the same selector shape. Match if **any** entry matches (AND within each entry, OR across the list). Prefer `In` for value-OR on one key; use `orLabelSelectors` for OR of independent multi-key groups. `labelSelector` and `orLabelSelectors` cannot co-exist in the same entry. + +```yaml +orLabelSelectors: + - matchLabels: + tier: frontend + matchExpressions: + - key: track + operator: In + values: [canary] + - matchLabels: + tier: backend +``` **`names` / `excludedNames`** accept exact resource names or glob patterns. If `names` is empty, all resource names are included (subject to label filters). `excludedNames` takes precedence over `names` when a name matches both. @@ -420,7 +470,8 @@ data: resourceFilters: - kinds: [ConfigMap, Secret, Deployment] labelSelector: - app: my-app + matchLabels: + app: my-app # ns-b has no filter policy entry, so global filters apply (include everything) ``` @@ -462,10 +513,12 @@ data: resourceFilters: - kinds: [Deployment] labelSelector: - app: production-workload-1 + matchLabels: + app: production-workload-1 - kinds: [StatefulSet] labelSelector: - app: production-workload-2 + matchLabels: + app: production-workload-2 ``` ### Per-Kind Exact Names @@ -561,7 +614,8 @@ data: resourceFilters: - kinds: ["*"] # catch-all: applies to every kind not listed below labelSelector: - backup: "true" # back up any resource carrying this label + matchLabels: + backup: "true" # back up any resource carrying this label ``` **Result:** Every resource type in `production` that has the label `backup=true` is backed up. Resources without that label are excluded. No kind enumeration is required. @@ -589,7 +643,8 @@ data: names: [db-credentials, tls-cert] # these exact Secrets by name - kinds: ["*"] # catch-all for all other kinds labelSelector: - backup: "true" # back up by label + matchLabels: + backup: "true" # back up by label ``` **Result:** @@ -666,7 +721,8 @@ data: names: [workload-1, workload-2] - kinds: [StatefulSet] labelSelector: - app: my-app + matchLabels: + app: my-app - kinds: [ConfigMap, Secret] names: ["app-*"] excludedNames: ["*-tmp-*", "*-debug-*", "*-tmp", "*-debug"] @@ -697,7 +753,7 @@ spec: ### `velero backup describe` -The output is extended to display namespace-scoped filter policies when present in the ResourcePolicy ConfigMap: +The output displays the referenced ResourcePolicy ConfigMap name when configured on the backup. It intentionally avoids resolving and displaying the live ConfigMap contents, because the ConfigMap content in the cluster may be modified or deleted after the backup execution, which could lead to displaying out-of-sync or inaccurate information: ``` Name: selective-backup @@ -721,46 +777,9 @@ Resources: Label selector: -Resource Policy: backup-filter-policy - -Namespace-Scoped Filter Policies: - ns-a: - Resource Filters: - ConfigMap, Secret, Deployment: - Label selector: app=my-app - Included names: - Excluded names: - target-namespace: - Resource Filters: - Deployment: - Label selector: app=production-workload-1 - Included names: - Excluded names: - StatefulSet: - Label selector: app=production-workload-2 - Included names: - Excluded names: - production: - Resource Filters: - Deployment: - Label selector: - Included names: [api-server, worker] - Excluded names: - (all other kinds): - Label selector: backup=true - Included names: - Excluded names: - -Fine-Grained Global Filter Policy: - Resource Filters: - ClusterRole, ClusterRoleBinding: - Label selector: - Included names: [my-app-*] - Excluded names: - CustomResourceDefinition: - Label selector: app=my-app - Included names: - Excluded names: +Resource policies: + Type: configmap + Name: backup-filter-policy Storage Location: default @@ -795,7 +814,7 @@ Notes: - Global filters (--include-resources, --selector, etc.) apply to all included namespaces - Namespace-scoped filters defined in --resource-policies-configmap override global filters for matching namespaces - Fine-grained global filter policies defined in --resource-policies-configmap override global filters for cluster-scoped resources -- Use 'velero backup describe' to view resolved filter policies after backup creation +- Use 'velero backup describe' to view the referenced ResourcePolicy ConfigMap name after backup creation ``` ### CLI Integration Points @@ -808,12 +827,12 @@ Notes: **Help and Discovery:** - `velero backup create --help` includes updated filtering documentation -- `velero backup describe` shows resolved filter policies for troubleshooting +- `velero backup describe` shows the referenced ResourcePolicy ConfigMap name - Validation errors include ConfigMap field references for easy debugging **Configuration Discovery:** - `velero backup create --help` includes namespace-scoped filtering documentation -- `velero backup describe` shows resolved filter policies for verification +- `velero backup describe` shows the referenced ResourcePolicy ConfigMap name for verification ## User Perspective @@ -823,7 +842,7 @@ This design provides fine-grained, per-namespace, per-kind control over backup f - **For users adopting namespace-scoped filter policies**: Create a ConfigMap with the `namespacedFilterPolicies` section and reference it via `BackupSpec.ResourcePolicy` (or the existing `--resource-policies-configmap` flag). The backup will selectively include/exclude resources per namespace based on the filter rules. - **For users already using ResourcePolicy for volume policies**: Add the `namespacedFilterPolicies` section to the same ConfigMap. Both volume policies and namespace-scoped filters coexist. - **For restore from a namespace-filtered backup**: No changes to restore workflow. Restore processes whatever is in the archive. Users can use existing `RestoreSpec.IncludedNamespaces` for additional filtering at restore time. -- **`velero backup describe` output**: Extended to show per-namespace, per-kind filter details when the ResourcePolicy ConfigMap contains `namespacedFilterPolicies`. +- **`velero backup describe` output**: Displays the referenced ResourcePolicy ConfigMap name when configured on the backup. - **Validation errors**: Reported at backup start when the ResourcePolicy ConfigMap contains invalid `namespacedFilterPolicies` configurations. Consistent with how volume policy validation errors are reported today. ## Alternatives Considered diff --git a/design/restore-filter-enhancement/fine-grained-restore-filters-design.md b/design/restore-filter-enhancement/fine-grained-restore-filters-design.md index eadc68088..4dd1b89ea 100644 --- a/design/restore-filter-enhancement/fine-grained-restore-filters-design.md +++ b/design/restore-filter-enhancement/fine-grained-restore-filters-design.md @@ -108,14 +108,16 @@ clusterScopedFilterPolicy: names: ["my-app-*"] - kinds: [CustomResourceDefinition] labelSelector: - app: my-app + matchLabels: + app: my-app namespacedFilterPolicies: - namespaces: - ns-a resourceFilters: - kinds: [ConfigMap, Secret, Deployment] labelSelector: - app: my-app + matchLabels: + app: my-app - namespaces: - ns-b resourceFilters: @@ -123,7 +125,8 @@ namespacedFilterPolicies: names: [app-1, app-2] - kinds: [ConfigMap] labelSelector: - app: my-service + matchLabels: + app: my-service ``` The restore-side ConfigMap does **not** require `volumePolicies` or `includeExcludePolicy` sections. Those are backup-specific. The YAML parser will ignore unknown fields gracefully, so a user can technically point to the same ConfigMap used for backup — the restore pipeline will only read `namespacedFilterPolicies` and `clusterScopedFilterPolicy`. @@ -137,7 +140,9 @@ namespacedFilterPolicies: - namespaces: [ns-a] resourceFilters: - kinds: [ConfigMap, Secret] # these kinds share a selector - labelSelector: {app: my-app} + labelSelector: + matchLabels: + app: my-app names: ["app-*"] - kinds: [Deployment] # this kind has its own selector names: [workload-1, workload-2] @@ -146,6 +151,36 @@ namespacedFilterPolicies: Only resource kinds listed in `resourceFilters` entries are restored for the matched namespaces; unlisted kinds are implicitly excluded (globally excluded kinds cannot be re-included — see precedence model). +#### Label selectors (`matchLabels` / `matchExpressions`) + +`labelSelector` and each entry of `orLabelSelectors` use the standard Kubernetes selector shape (same as `RestoreSpec.labelSelector`): + +```yaml +labelSelector: + matchLabels: + app: my-app + matchExpressions: + - key: environment + operator: In + values: [prod, staging] + - key: do-not-restore + operator: DoesNotExist +``` + +Supported `matchExpressions` operators: `In`, `NotIn`, `Exists`, `DoesNotExist`. Prefer `In` for value-OR on one key; use `orLabelSelectors` for OR across independent multi-key groups. `labelSelector` and `orLabelSelectors` cannot co-exist in the same `resourceFilters` entry. + +```yaml +orLabelSelectors: + - matchLabels: + tier: frontend + matchExpressions: + - key: track + operator: In + values: [canary] + - matchLabels: + tier: backend +``` + #### Peek-and-Map Fallback for Unresolved Kinds The `kinds` field accepts both plural resource names (e.g., `configmaps`, `mycustomkinds.mygroup.io`) and singular `Kind` names (e.g., `ConfigMap`, `MyCustomKind`). @@ -382,9 +417,10 @@ resourceFilters: resourceFilters: - kinds: ["Deployment"] labelSelector: - "invalid label key!": "value" # invalid key syntax + matchLabels: + "invalid label key!": "value" # invalid key syntax ``` -**Behavior:** Validation error during restore creation when `labels.ValidatedSelectorFromSet()` fails: +**Behavior:** Validation error during restore creation when `metav1.LabelSelectorAsSelector()` fails: ``` namespacedFilterPolicies[0].resourceFilters[0]: invalid label selector: "invalid label key!" is not a valid label key ``` @@ -420,7 +456,8 @@ namespacedFilterPolicies: resourceFilters: - kinds: [ConfigMap, Secret] # Secret listed here is ineffective — globally excluded labelSelector: - app: my-app + matchLabels: + app: my-app - kinds: [Deployment] ``` @@ -461,8 +498,8 @@ After existing filter setup, the filter policies are resolved into the runtime m The `resolveRestoreNamespacedFilterPolicies` function: - For each `NamespacedFilterPolicy`, iterates its `ResourceFilters` entries - Resolves kind names to fully-qualified group-resource strings using the discovery helper -- Converts `labelSelector` maps into `labels.Selector` objects using `labels.ValidatedSelectorFromSet()` -- Converts `orLabelSelectors` maps into `[]labels.Selector` +- Converts `labelSelector` into a `labels.Selector` via `ToMetaV1LabelSelector` + `metav1.LabelSelectorAsSelector()` +- Converts `orLabelSelectors` into `[]labels.Selector` the same way - Creates `IncludesExcludes` instances for `names`/`excludedNames` patterns - Identifies catch-all entries (empty or `["*"]` kinds) and stores them in `catchAllFilter` - Builds a `resourceFilterMap` keyed by the resolved group-resource string @@ -537,7 +574,8 @@ data: resourceFilters: - kinds: [Deployment, ConfigMap] labelSelector: - app: my-app + matchLabels: + app: my-app # ns-b has no filter policy entry, so global filters apply (restore everything) ``` @@ -631,7 +669,8 @@ data: names: [db-credentials, tls-cert] # these exact Secrets by name - kinds: ["*"] # catch-all for all other kinds labelSelector: - backup: "true" # restore by label + matchLabels: + backup: "true" # restore by label ``` **Result:** @@ -658,7 +697,8 @@ data: names: ["my-app-*"] - kinds: [CustomResourceDefinition] labelSelector: - app: my-app + matchLabels: + app: my-app namespacedFilterPolicies: - namespaces: - production diff --git a/design/ria-must-include-addtional-items-design.md b/design/ria-must-include-addtional-items-design.md new file mode 100644 index 000000000..95f6863fd --- /dev/null +++ b/design/ria-must-include-addtional-items-design.md @@ -0,0 +1,357 @@ +# RestoreItemAction Must-Include Additional Items + +## Abstract + +Backup Item Actions (BIAs) can already mark additional items as must-include via `backup.velero.io/must-include-additional-items`, so Velero bypasses resource and namespace exclusion filters when backing those dependencies up. +This proposal adds the same plugin-controlled escape hatch on restore: `restore.velero.io/must-include-additional-items`, so Restore Item Actions (RIAs) can force-restore declared `AdditionalItems` even when they would otherwise be dropped by global restore filters. + +## Glossary & Abbreviation + +**Additional Item**: A resource identifier returned by a Backup/Restore Item Action's `Execute()` result that Velero should process as a dependency of the current item. +**BIA**: Backup Item Action plugin. +**RIA**: Restore Item Action plugin. +**Must-Include**: A plugin-set annotation on the action's `UpdatedItem` that tells Velero to bypass global include/exclude filters for that action's `AdditionalItems`. +**Global Restore Filter**: `RestoreSpec` filters applied uniformly — `IncludedNamespaces`/`ExcludedNamespaces`, `IncludedResources`/`ExcludedResources`, `IncludeClusterResources`, and label selectors. +**Fine-Grained Restore Filter**: Per-namespace / cluster-scoped policies from `RestoreSpec.ResourcePolicy` (`namespacedFilterPolicies`, `clusterScopedFilterPolicy`), as described in [Fine Grained Restore Filters via Resource Policies](https://github.com/velero-io/velero/blob/main/design/restore-filter-enhancement/fine-grained-restore-filters-design.md). +**`resourceMustHave`**: A small hardcoded server-side set of resource types that bypass resource and namespace I/E checks inside `restoreItem()` today (but not `IncludeClusterResources=false`). + +## Background + +### Backup-side precedent + +On backup, a BIA may set `backup.velero.io/must-include-additional-items: "true"` on the returned `UpdatedItem`. +Velero strips that annotation (it is an internal signal, not intended to land on the live object) and passes `mustInclude=true` into recursive `backupItem` calls for that action's `AdditionalItems`. +When `mustInclude` is true, `itemInclusionChecks` skips namespace/resource exclusion checks (and related exclusion labels / fine-grained name filters) so plugin-declared dependencies are not dropped by the user's backup filters. +In-tree CSI BIAs already rely on this for VolumeSnapshot / VolumeSnapshotContent / VolumeSnapshotClass style dependency chains. + +### Restore-side gap + +On restore, RIAs can return `AdditionalItems`, and Velero recursively calls `restoreItem()` for each of them. +That path already bypasses fine-grained restore filters and global label selectors, because those are evaluated earlier in `getOrderedResourceCollection` / `getSelectedRestoreableItems`. +However, `restoreItem()` still enforces global resource includes/excludes, namespace includes/excludes, and `IncludeClusterResources=false`. + +The fine-grained restore filters design explicitly documents this remaining floor: + +> Note that these additional items must still pass global resource/namespace exclusions. + +There is no restore-side equivalent of the BIA must-include annotation. +Plugins that need a hard dependency restored despite a selective restore configuration have no opt-in way to express that, short of relying on the server-side `resourceMustHave` list (which is global, not plugin-scoped, and does not bypass `IncludeClusterResources=false`). + +### Motivating scenario + +Consider a selective restore that includes only application namespaces and excludes storage/snapshot resource types, while a plugin knows that restoring a PVC correctly requires a related cluster-scoped or cross-namespace dependency that exists in the backup archive. +Today the RIA can request that dependency as an `AdditionalItem`, but Velero will skip it at the global exclusion checks inside `restoreItem()`. +With a restore must-include annotation, the plugin can declare the dependency as required and Velero will restore it (provided the object is present in the backup tarball). + +## Goals + +- Add `restore.velero.io/must-include-additional-items` with the same parent-annotation contract as the backup-side must-include annotation. +- When an RIA sets the annotation on `UpdatedItem`, bypass global resource I/E, namespace I/E, and `IncludeClusterResources=false` for that RIA's `AdditionalItems`. +- Keep the change opt-in and backward compatible: restores and plugins that do not set the annotation behave exactly as today. +- Document the trust model, precedence rules, and interaction with existing restore gates for plugin authors and operators. + +## Non-Goals + +- Changing the plugin protobuf / `RestoreItemAction` interface shape (no new RPC fields). +- Changing CRDs or adding CLI flags. +- Changing the `resourceMustHave` list (including any narrowing related to VolumeSnapshotContent). +- Updating in-tree RIAs (CSI or otherwise) to set the new annotation as part of this change. +- Per-additional-item granularity (the annotation applies blanket to all `AdditionalItems` from that RIA invocation, matching BIA). +- Materializing items that were never backed up. + +## High-Level Design + +Mirror the backup workflow: + +1. Introduce annotation constant `restore.velero.io/must-include-additional-items`. +2. After each RIA `Execute()`, if `UpdatedItem` carries the annotation with value `"true"`, strip it and set `mustIncludeAdditionalItems=true`. +3. Pass that boolean into recursive `restoreItem(..., mustInclude)` calls for the action's `AdditionalItems`. +4. When `mustInclude` is true, skip the global resource/namespace/`IncludeClusterResources` exclusion checks inside `restoreItem()`. +5. Keep all non-filter gates unchanged (tarball presence, already-restored, completed Jobs, API errors, wait-for-additional-items, etc.). + +Top-level items from the archive continue to be restored with `mustInclude=false`, so user filters still apply to the primary restore set. + +```mermaid +flowchart TD + startRestore[Start Restore] --> readTarball[Read Item from Backup Tarball] + readTarball --> topLevelRestoreItem["restoreItem(..., mustInclude=false)"] + + topLevelRestoreItem --> checkMustInclude{"mustInclude == true?"} + + checkMustInclude -- No --> checkFilters{"Pass Global Resource/Namespace Filters?"} + checkFilters -- No --> skipItem[Skip Restore] + checkFilters -- Yes --> nonFilterGates["Other gates: isCompleted, already-restored, ..."] + + checkMustInclude -- Yes --> nonFilterGates + + nonFilterGates --> executeRIA[Execute RestoreItemAction] + + executeRIA --> checkSkip{"SkipRestore?"} + checkSkip -- Yes --> skipItem + checkSkip -- No --> checkAnnotation{"Has must-include annotation?"} + + checkAnnotation -- Yes --> stripAnnotation[Strip Annotation] + stripAnnotation --> setFlagTrue["mustIncludeAdditionalItems = true"] + + checkAnnotation -- No --> setFlagFalse["mustIncludeAdditionalItems = false"] + + setFlagTrue --> loopAdditionalItems[Loop over AdditionalItems] + setFlagFalse --> loopAdditionalItems + + loopAdditionalItems --> existsInBackup{"Item file in tarball?"} + existsInBackup -- No --> warnSkip[Warn and skip] + existsInBackup -- Yes --> recursiveRestoreItem["restoreItem(..., mustInclude=mustIncludeAdditionalItems)"] + recursiveRestoreItem --> checkMustInclude +``` + +> The edge `recursiveRestoreItem --> checkMustInclude` is a recursive call (new `restoreItem` stack frame), not a same-frame loop. + +## Detailed Design + +### Annotation constant + +In `pkg/apis/velero/v1/labels_annotations.go`, next to the existing backup constant: + +```go +// Velero checks this annotation to determine whether to skip resource excluding check. +MustIncludeAdditionalItemAnnotation = "backup.velero.io/must-include-additional-items" + +// MustIncludeAdditionalItemRestoreAnnotation is set by RestoreItemActions on the UpdatedItem +// to tell Velero to bypass global resource/namespace exclusion checks (and IncludeClusterResources=false) +// for that action's AdditionalItems. Value must be "true". The annotation is stripped before +// the item is applied to the cluster. +// +// Notice: SkipRestore on the Execute output takes precedence. If SkipRestore is true, the +// annotation is never inspected and AdditionalItems are not processed. +MustIncludeAdditionalItemRestoreAnnotation = "restore.velero.io/must-include-additional-items" +``` + +Only the string value `"true"` enables the bypass (same as backup). + +### `restoreItem` signature + +```go +func (ctx *restoreContext) restoreItem( + obj *unstructured.Unstructured, + groupResource schema.GroupResource, + namespace string, + mustInclude bool, +) (results.Result, results.Result, bool) +``` + +Call sites: + +| Site | `mustInclude` value | +|---|---| +| Top-level restore loop | `false` | +| Recursive additional-item restore after an RIA | derived from that RIA's `UpdatedItem` annotation | + +### Bypass exclusion checks; keep namespace creation + +Today, namespace exclusion and `EnsureNamespaceExistsAndIsReady` share one `if namespace != ""` block in `restoreItem()`. +If must-include only skipped the exclusion check without refactoring, an additional item targeting an excluded namespace would fail because its target namespace was never ensured. + +Required structure: + +```go +if mustInclude { + restoreLogger.Info("Skipping the resource/namespace exclusion checks because the item is marked as must-include") +} else { + if !ctx.resourceIncludesExcludes.ShouldInclude(groupResource.String()) && !ctx.resourceMustHave.Has(groupResource.String()) { + restoreLogger.Info("Not restoring item because resource is excluded") + return warnings, errs, itemExists + } + + if namespace != "" { + if !ctx.namespaceIncludesExcludes.ShouldInclude(obj.GetNamespace()) && !ctx.resourceMustHave.Has(groupResource.String()) { + restoreLogger.Info("Not restoring item because namespace is excluded") + return warnings, errs, itemExists + } + } else { + if boolptr.IsSetToFalse(ctx.restore.Spec.IncludeClusterResources) { + restoreLogger.Info("Not restoring item because it's cluster-scoped") + return warnings, errs, itemExists + } + } +} + +// Namespace creation runs regardless of mustInclude. +if namespace != "" { + nsToEnsure := getNamespace(restoreLogger, archive.GetItemFilePath(ctx.restoreDir, "namespaces", "", obj.GetNamespace()), namespace) + _, nsCreated, err := kube.EnsureNamespaceExistsAndIsReady(nsToEnsure, ctx.namespaceClient, ctx.resourceTerminatingTimeout, ctx.resourceDeletionStatusTracker) + // ... existing error handling and restoredItems bookkeeping ... +} +``` + +Namespace remapping is unchanged: exclusion checks use the original namespace (`obj.GetNamespace()`); namespace creation uses the remapped target `namespace` parameter. + +### Process the annotation after each RIA + +Inside the applicable-actions loop in `restoreItem()`, after `SkipRestore` handling and type-asserting `UpdatedItem`: + +```go +obj = unstructuredObj + +mustIncludeAdditionalItems := false +if annotations := obj.GetAnnotations(); annotations != nil && + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] == "true" { + mustIncludeAdditionalItems = true + restoreLogger.Info("RestoreItemAction marked additional items as must-include; bypassing resource/namespace exclusion checks for them") + delete(annotations, velerov1api.MustIncludeAdditionalItemRestoreAnnotation) + obj.SetAnnotations(annotations) +} + +for _, additionalItem := range executeOutput.AdditionalItems { + // existing tarball stat / unmarshal / namespace mapping ... + w, e, additionalItemExists := ctx.restoreItem( + additionalObj, + additionalItem.GroupResource, + additionalItemNamespace, + mustIncludeAdditionalItems, + ) + // existing merge / filteredAdditionalItems bookkeeping ... +} +``` + +### Filter bypass matrix + +| Gate | Plain AdditionalItem | `resourceMustHave` | RIA `mustInclude=true` | BIA `mustInclude=true` (parity target) | +|---|---|---|---|---| +| Fine-grained policies (kind/name/label) | Bypass (never enter selection Phase B filters) | N/A in `restoreItem` | Bypass (same) | Bypass | +| Global label selectors | Bypass (never re-enter selection) | N/A in `restoreItem` | Bypass (same) | Bypass | +| Global resource I/E | Honored | Bypass | Bypass | Bypass | +| Global namespace I/E | Honored | Bypass | Bypass | Bypass | +| `IncludeClusterResources=false` | Honored | Honored (not bypassed) | Bypass | Bypass | +| Item must exist in backup tarball | Required | Required | Required | N/A (fetched from cluster) | +| `isCompleted` / already-restored / API errors | Still apply | Still apply | Still apply | `DeletionTimestamp` still applies on backup | + +RIA must-include is intentionally a **stronger** override than `resourceMustHave` because it also bypasses `IncludeClusterResources=false`. +That matches BIA must-include semantics (plugin-trusted hard dependencies), rather than widening the hardcoded server list. + +### Interaction with fine-grained restore filters + +Per [Fine Grained Restore Filters via Resource Policies](../restore-filter-enhancement/fine-grained-restore-filters-design.md), plugin additional items already bypass `namespacedFilterPolicies` / `clusterScopedFilterPolicy` kind, name, and label checks. +Those filters live in the selection phases; additional items enter `restoreItem()` directly. + +This proposal only changes the remaining global gates inside `restoreItem()`. +With must-include set, an additional item effectively bypasses **all** restore filters (fine-grained and global). +Without the annotation, behavior is unchanged: fine-grained filters are still bypassed, global exclusions still apply. + +### Interaction with existing restore gates + +#### `SkipRestore` precedence + +If `Execute()` returns `SkipRestore: true`, `restoreItem()` returns before inspecting the annotation, and no `AdditionalItems` are processed. +This mirrors backup-side precedence where `velero.io/skip-from-backup` outranks must-include. + +#### Multi-RIA semantics + +Annotation handling is per RIA invocation inside the actions loop: + +1. RIA N executes → inspect/strip annotation on that `UpdatedItem` → restore that RIA's `AdditionalItems` with the derived flag. +2. RIA N+1 sees the already-stripped object unless it sets the annotation again. + +A later RIA does not inherit an earlier RIA's must-include decision. + +#### Transitive propagation + +The parent's `mustInclude` flag admits the child additional item through filters. +It does **not** automatically force-include grandchildren. +Each RIA level that needs the escape hatch must set the annotation on its own `UpdatedItem`, matching BIA behavior. + +#### Non-filter gates that still apply + +Even when `mustInclude=true`: + +- Missing archive file → warn and skip (existing behavior). +- `isCompleted` resources (e.g. completed Jobs) → skip. +- Already present in `ctx.restoredItems` → skip. +- Create/update API failures → errors as today. +- `WaitForAdditionalItems` / `AreAdditionalItemsReady` polling after the additional-item loop → unchanged. + +### Relationship to `resourceMustHave` + +| Mechanism | Who decides | Bypasses resource/ns I/E | Bypasses `IncludeClusterResources=false` | +|---|---|---|---| +| `resourceMustHave` | Velero server (hardcoded) | Yes | No | +| RIA must-include | Plugin author (annotation) | Yes | Yes | + +The two mechanisms coexist. +This proposal does not migrate in-tree CSI (or other) RIAs onto the annotation. +Doing so would be a separate behavior change: it could force-restore types users explicitly excluded, and would newly restore cluster-scoped dependencies even when `IncludeClusterResources=false`. + +### Plugin usage sketch + +```go +func (p *myRestoreAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + item.SetAnnotations(annotations) + + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: schema.GroupResource{Group: "example.io", Resource: "dependencies"}, Namespace: "dep-ns", Name: "dep-1"}, + }, + }, nil +} +``` + +Plugin authors must ensure the additional item was actually captured in the backup (typically via the corresponding BIA also using `backup.velero.io/must-include-additional-items`). + +### Tests + +Extend restore coverage (existing `TestRestoreActionAdditionalItems` patterns / focused cases) for: + +1. Resource exclusion bypass with annotation; still skipped without annotation. +2. Namespace exclusion bypass **and** target namespace creation. +3. `IncludeClusterResources=false` bypass for cluster-scoped additional items. +4. Annotation stripped from the object applied to the cluster. +5. `SkipRestore: true` prevents additional-item processing even if the annotation is set. +6. Missing tarball entry still warns and skips. +7. Transitive case: child RIA must re-set the annotation for grandchildren. +8. Top-level restore path still passes `mustInclude=false` and honors filters. + +### Documentation + +- Constant doc comment (including `SkipRestore` precedence). +- Plugin-author docs for Restore Item Actions: annotation key/value, blanket scope, filter-bypass matrix, namespace-creation side effect, tarball requirement. + +## Security Considerations + +Installing an RIA that sets this annotation grants that plugin authority to restore dependencies outside the operator's restore filters, including: + +- resources in namespaces the restore excluded (and creation of those target namespaces if needed); +- resource types the restore excluded; +- cluster-scoped resources even when `IncludeClusterResources=false`. + +This matches the existing BIA trust model: item-action plugins are already privileged components of the Velero deployment. +Operators should treat RIA installation as a trust decision. +The annotation is stripped before apply so it does not persist as attacker-controlled cluster state from the backup archive alone; a matching RIA must run and return `AdditionalItems` for the bypass to take effect. + +## Compatibility + +- No CRD or plugin interface changes. +- Existing restores unchanged when no RIA sets the annotation. +- Existing tests that assert additional items are dropped under namespace filters / `IncludeClusterResources=false` remain valid for the no-annotation path. +- Compatible with fine-grained restore filters: additional items already bypass those filters; this proposal only addresses the documented global-exclusion floor. + +## Alternatives Considered + +### Per-item must-include on each `ResourceIdentifier` + +Pros: selective control within one `AdditionalItems` list. +Cons: requires API changes to `ResourceIdentifier` or a parallel structure; diverges from BIA; plugins that need selectivity can already split across actions or omit non-required items. + +Rejected for this proposal; may be revisited later if plugin authors demonstrate a concrete need. + +### Widen `resourceMustHave` instead of a plugin annotation + +Pros: no plugin contract change. +Cons: server-forced, global, not scoped to a plugin call; does not give third-party plugins a general tool; does not match BIA; conflicts with efforts to keep hardcoded force-include lists narrow. + +Rejected — wrong trust model for a general plugin escape hatch. diff --git a/design/volume-data-inplace-restore/volume-data-inplace-restore.md b/design/volume-data-inplace-restore/volume-data-inplace-restore.md new file mode 100644 index 000000000..b178b9314 --- /dev/null +++ b/design/volume-data-inplace-restore/volume-data-inplace-restore.md @@ -0,0 +1,368 @@ +# Volume Data In-place Full/Incremental Restore + +## Table of Contents + +- [Background](#background) +- [Goals](#goals) +- [Non-Goals](#non-goals) +- [Overview](#overview) +- [Detailed Design](#detailed-design) + - [CRD Changes](#crd-changes) + - [CLI](#cli) + - [Workload Management](#workload-management) + - [Handling Cross-Zone Scheduling (WaitForFirstConsumer)](#handling-cross-zone-scheduling-waitforfirstconsumer) + - [Namespace Mapping](#namespace-mapping) + - [Pre-flight Checks](#pre-flight-checks) + - [1. PVC is Not Actively Used by a Running Pod](#1-pvc-is-not-actively-used-by-a-running-pod) + - [2. PVC is Bound to the Original PV](#2-pvc-is-bound-to-the-original-pv) + - [3. Volume Size Validation](#3-volume-size-validation) + - [Error Handling](#error-handling) + - [Restore Workflow Update](#restore-workflow-update) + - [In-place Incremental Restore for CSI Snapshot with Block Data Move for Block Volumes](#in-place-incremental-restore-for-csi-snapshot-with-block-data-move-for-block-volumes) + - [In-place Full Restore for CSI Snapshot with Block Data Move for Block Volumes](#in-place-full-restore-for-csi-snapshot-with-block-data-move-for-block-volumes) + - [In-place Incremental Restore for CSI Snapshot with File System Data Move for File System Volumes](#in-place-incremental-restore-for-csi-snapshot-with-file-system-data-move-for-file-system-volumes) + - [In-place Full Restore for CSI Snapshot with File System Data Move for File System Volumes](#in-place-full-restore-for-csi-snapshot-with-file-system-data-move-for-file-system-volumes) + - [In-place Incremental Restore for CSI Snapshot with Block Data Move for File System Volumes](#in-place-incremental-restore-for-csi-snapshot-with-block-data-move-for-file-system-volumes) + - [In-place Full Restore for CSI Snapshot with Block Data Move for File System Volumes](#in-place-full-restore-for-csi-snapshot-with-block-data-move-for-file-system-volumes) + - [In-place Incremental Restore for File System Backup for File System Volumes](#in-place-incremental-restore-for-file-system-backup-for-file-system-volumes) + - [In-place Full Restore for File System Backup for File System Volumes](#in-place-full-restore-for-file-system-backup-for-file-system-volumes) +- [Installation](#installation) +- [Upgrade](#upgrade) + +## Background + +Currently, Velero only supports restoring volume data to a newly provisioned PVC. If the target PVC already exists in the cluster, Velero skips the data restoration entirely and leaves the existing volume untouched. + +This design introduces the "in-place restore" capability, allowing Velero to restore volume data directly into an existing, bound PVC. When performing an in-place restore, users can choose to either overwrite the volume entirely (in-place full restore) or only restore the modified data to optimize performance (in-place incremental restore). + +To ensure data consistency and allow Velero to safely recreate the PVC during the process, users must manually delete any pods consuming the target volume before initiating an in-place restore. + +## Goals + +- Enable Velero to restore volume data directly into an existing, bound PVC without requiring the user to manually delete the PVC and PV. +- Support both Full (overwrite all) and Incremental (overwrite only changed data) in-place restores. +- Support in-place restores for Windows workloads. +- Ensure data consistency and correct Kubernetes scheduling constraints (e.g., handling `WaitForFirstConsumer` and zonal topologies) are respected during and after the restore. + +## Non-Goals + +- Automating the deletion of workloads before the restore. It remains the user's responsibility to ensure the volume is not actively consumed and the Pods are completely removed before triggering the restore to prevent data corruption and allow PVC recreation. +- In-place restore for CSI snapshot without data move. +- In-place restore for Native Snapshots (cloud provider snapshots without CSI). +- Fine-grained, per-volume control over in-place restores. The newly introduced in-place restore policies apply globally to all volumes within a single restore operation. Allowing users to specify different restore strategies for individual volumes is deferred to a future enhancement. + +## Overview + +This design focuses exclusively on volume data restoration. To support this, we are introducing a new field, `ExistingVolumeDataPolicy`, to the `Restore` spec. This feature operates independently of Kubernetes resource restoration, which remains controlled by the existing `ExistingResourcePolicy` field. + +Depending on how unchanged data is handled during the restoration process, in-place volume data restores are categorized into two types: + +- **In-place full restore**: Overwrites the volume with the backup data, regardless of whether the existing data has changed. +- **In-place incremental restore**: Optimizes the process by restoring only the data that has changed since the backup, leaving unmodified data intact. This is achieved by leveraging Changed Block Tracking (CBT) for block data and file metadata comparisons for file system data. + +Support for in-place full and incremental restores varies depending on the underlying backup method, as detailed in the following table: + +| Backup Method | In-place Full Restore | In-place Incremental Restore | +| --------------------------------------- | --------------------- | ---------------------------- | +| CSI Snapshot with Block Data Move | Yes | Yes | +| CSI Snapshot with File System Data Move | Yes | Yes | +| CSI Snapshot without Data Move | No | No | +| File System Backup | Yes | Yes | +| Native Snapshot | No | No | + +Additionally, a new boolean field `DeleteExtraFiles` is added to the `UploaderConfig` within the `Restore` spec. When performing a file system restore (either via PodVolumeBackup or CSI File System Data Move), this flag controls whether files present in the target volume but absent in the backup should be deleted. Setting this to `true` ensures the target volume's file system exactly mirrors the backup state. Note that this setting is ignored for block data mover restores, as block-level operations inherently overwrite the entire file system structure. + +Because Velero must create a temporary restore Pod in the Velero namespace to mount the volume and restore the data, it cannot directly use the existing PVC, which resides in the workload namespace. Velero must delete the existing PVC, recreate a temporary restore PVC in the Velero namespace, and bind it to the existing PV. The core strategy for implementing an in-place restore involves the following sequence: + +```mermaid +flowchart TD + subgraph PVC CSI RIA + A[Patch existing PV's reclaim policy to Retain] --> B[Delete existing PVC] + end + subgraph Exposer + B --> C[Create temporary restore PVC in Velero namespace
and bind it to existing PV] + C --> D[Create temporary restore Pod
that mounts temporary restore PVC] + end + subgraph Block/File System Uploader + D --> E[Restore data directly into the volume] + end + subgraph Exposer Post-Restore + E --> F[Delete temporary restore Pod and PVC] + end + F --> G[Target workload Pod mounts target PVC
once it is recreated] +``` + +When restoring a file system volume using the block data mover, the PV must temporarily have its `volumeMode` set to `Block` so the restore Pod can mount it as a raw block device. Because the `volumeMode` field in a PV spec is immutable, reusing the existing PV directly is not possible. Instead, Velero must delete the existing PV and create a temporary one. The sequence for this scenario is as follows: + +```mermaid +flowchart TD + subgraph PVC CSI RIA + A[Patch existing PV's reclaim policy to Retain] --> B[Delete existing PVC] + end + subgraph Exposer + B --> C[Delete existing PV] + C --> D[Create temporary restore PV with volumeMode: Block
using same volume handle] + D --> E[Create temporary restore PVC in Velero namespace
with volumeMode: Block and bind to temporary PV] + E --> F[Create temporary restore Pod
that mounts temporary restore PVC] + end + subgraph Block Uploader + F --> G[Restore data directly into the volume] + end + subgraph Exposer Post-Restore + G --> H[Delete temporary restore Pod, PVC, and PV] + H --> I[Recreate original PV with volumeMode: Filesystem] + end + I --> J[Recreate original PVC in workload namespace
and allow it to bind to recreated PV] +``` + + +## Detailed Design + +### CRD Changes + +To support the new in-place restore policies and incremental data transfer, several Custom Resource Definitions (CRDs) will be updated. + +**Restore CRD** +A new field `existingVolumeDataPolicy` is added to the `Restore` spec to allow users to define how existing volume data should be handled. Additionally, a new field `deleteExtraFiles` is added to the `uploaderConfig` to control file deletion during file system restores. + +```yaml +spec: + existingVolumeDataPolicy: "" # Valid values: "", none, full, incremental + uploaderConfig: + deleteExtraFiles: false +``` + +- `existingVolumeDataPolicy`: + - `""` (default) or `none`: Do not restore volume data if the target PVC already exists. + - `full`: Perform an in-place full restore, overwriting all existing data on the volume. + - `incremental`: Perform an in-place incremental restore, only overwriting data that has changed since the backup. +- `uploaderConfig.deleteExtraFiles`: A boolean flag that controls whether files present in the target volume but absent from the backup should be deleted. **Note:** This setting is *only* applicable to File System restores (PodVolumeBackup or CSI File System Data Move) and has no effect on Block Data Move restores. Furthermore, it is ignored for non-in-place restores (where `existingVolumeDataPolicy` is not set to `full` or `incremental`). + +If the target PVC does not exist, Velero will fall back to its default behavior and provision a new PVC for the restore, regardless of whether `existingVolumeDataPolicy` is set to `full` or `incremental`. Furthermore, if `existingVolumeDataPolicy` is set to `incremental` but the underlying storage does not support incremental restores, Velero will automatically fall back to a `full` restore. + +The following table summarizes the expected behavior for different combinations of `existingResourcePolicy` and `existingVolumeDataPolicy` when the target PVC already exists: + +| `existingResourcePolicy` | `existingVolumeDataPolicy` | PVC Resource Action | Volume Data Restore | +| ------------------------ | -------------------------- | ------------------- | ------------------- | +| `none` | `none` | Untouched | Untouched | +| `none` | `full` | Untouched | Full | +| `none` | `incremental` | Untouched | Incremental | +| `update` | `none` | Patched | Untouched | +| `update` | `full` | Patched | Full | +| `update` | `incremental` | Patched | Incremental | + +**DataDownload CRD** +To support incremental restores, the `DataDownload` spec is extended with a new `restoreType` string flag (valid values are `full` and `incremental`) to instruct the data mover to perform an incremental restore. It also introduces a new `csiSnapshot` field, which captures the metadata of a snapshot taken from the existing PVC, acting as the baseline for Changed Block Tracking (CBT) delta calculations during an in-place incremental block restore. Additionally, the `deleteExtraFiles` configuration is passed to the underlying data mover via the existing `dataMoverConfig` map. + +```yaml +spec: + restoreType: "incremental" + csiSnapshot: + volumeSnapshot: "" + storageClass: "" + snapshotClass: "" + driver: "" +``` + +- `restoreType`: A string flag indicating whether the data mover should perform a `full` or `incremental` restore. +- `csiSnapshot`: + - `volumeSnapshot`: the name of the volume snapshot + - `storageClass`: the name of the storage class of the PVC that the volume snapshot is created from + - `snapshotClass`: the name of the snapshot class that the volume snapshot is created with + - `driver`: the driver used by the VolumeSnapshotContent + +**PodVolumeRestore CRD** +A new `restoreType` string flag (valid values are `full` and `incremental`) is added to the `PodVolumeRestore` spec to instruct the file system data mover (e.g., Kopia) to perform an incremental restore. Additionally, the `deleteExtraFiles` configuration is passed to the underlying uploader via the existing `uploaderSettings` map. + +```yaml +spec: + restoreType: "incremental" +``` + +- `restoreType`: A string flag indicating whether the data mover should perform a `full` or `incremental` restore. + +### CLI + +New flags will be added to the `velero restore create` command to support the new policy: + +- `--existing-volume-data-policy`: Accepts the values `none`, `full`, or `incremental`, mapping to `existingVolumeDataPolicy`. +- `--delete-extra-files`: A boolean flag mapping to `uploaderConfig.deleteExtraFiles`. + +### Workload Management + +To ensure data consistency and allow for necessary configuration changes, users must delete any Pods actively using the target volume before initiating an in-place restore. This is required for three primary reasons: + +1. **Preventing Data Corruption:** It is critical to prevent the active workload Pods and the temporary restore Pods from writing to the volume simultaneously, which would lead to data corruption. +2. **PVC Recreation:** Velero creates a temporary restore Pod in the Velero namespace to mount the volume and restore the data. Since it cannot directly use the existing PVC located in the workload namespace, Velero must delete the existing PVC, create a temporary restore PVC in the Velero namespace, and bind it to the existing PV. However, Kubernetes' `pvc-protection` finalizer prevents the deletion of any PVC actively used by a running Pod. Consequently, simply pausing the workload is insufficient; the Pods must be completely removed to allow the PVC deletion to proceed. +3. **ReadWriteOncePod Access Mode:** If the volume is configured with the `ReadWriteOncePod` access mode, Kubernetes strictly enforces that the volume can only be mounted by a single Pod at a time. The existing workload Pod must be completely deleted to release the volume, allowing Velero's temporary restore Pod to successfully mount it and perform the data transfer. + +Users must manage the lifecycle of their workloads before starting the restore. This applies to various workload types: + +- **Standard Controllers (Deployments, StatefulSets, Jobs, CronJobs):** The required action depends on the restore method: + - **For CSI Snapshot Restores:** Users can scale these controllers down to zero replicas to terminate the underlying Pods. + - **For File System Restores (PodVolumeRestore):** Users must completely delete the controllers. Simply scaling down to zero is insufficient because file system restores rely on an init container injected into the restored target Pod to process the data transfer. If the controller is only scaled down, it will immediately terminate the Pod restored by Velero to maintain its zero-replica count. Although the controller may subsequently spawn a new Pod, that new Pod will lack the required restore init container, causing the restore to fail. +- **DaemonSets:** Since Kubernetes lacks a mechanism to scale DaemonSets to zero, users must either delete the DaemonSet entirely or use node selectors/cordoning to evict the Pods. +- **Operator-Managed Pods:** Custom controllers (like ArgoCD) may have fast reconciliation loops that aggressively recreate Pods. These operators must be paused or suspended, and their managed Pods deleted. +- **Out-of-Cluster Clients:** External consumers accessing the storage directly (e.g., via NFS or storage APIs) are invisible to Kubernetes and must be manually disconnected to ensure no external writes occur during the restore. + +**Note:** Automating the deletion of these workloads is explicitly out of scope for this feature. It remains the user's responsibility to ensure the volume is not actively consumed and the Pods are removed before triggering the restore. + +### Handling Cross-Zone Scheduling (WaitForFirstConsumer) + +When performing an in-place restore, Velero deletes the existing target PVC and recreates it. For StorageClasses using the `WaitForFirstConsumer` volume binding mode, this recreation resets the scheduling lifecycle. Even though Velero adds a selector to the PVC spec to ensure it binds exclusively to the original PV, a scheduling issue can still occur. If the target PVC loses its node affinity, the Kubernetes Scheduler might schedule the recreated business Pod to a different availability zone. Because the original PV is physically constrained to its original zone, the Pod will fail to mount the volume and remain stuck in the `ContainerCreating` state with an attachment error. + +**Solution**: +During the PVC CSI Restore Item Action (RIA), right before deleting the existing PVC, Velero extracts the `volume.kubernetes.io/selected-node` annotation from that PVC and carries it on the PVC to be restored via a Velero-internal carrier annotation (`restore.velero.io/inplace-restore-selected-node`). After all Restore Item Actions have run, the restore engine translates the carrier back to the `volume.kubernetes.io/selected-node` annotation and strips the carrier so it never lands on the cluster. + +A carrier annotation is used instead of the Kubernetes annotation directly because the generic PVC RIA unconditionally strips the `selected-node` annotation during restore, and the execution order of Restore Item Actions is not a documented contract. With the carrier, the behavior is independent of the RIA execution order: the Kubernetes annotation is stripped by default on every path (including when the target PVC does not exist and Velero falls back to provisioning a new PVC), and preservation only happens when the PVC CSI RIA explicitly captured a value from the existing PVC. +By preserving the `selected-node` annotation, the Kubernetes Scheduler is forced to schedule the recreated business Pod to the original node/zone, ensuring it successfully mounts the restored PV. + +### Namespace Mapping +When namespace mapping is configured, in-place restores work normally in most scenarios. However, in-place incremental restores using CSI snapshots with a block data mover are not natively supported across different namespaces. Velero cannot use Changed Block Tracking (CBT) to calculate data deltas when the target volume is in a different namespace, as the volumes may belong to different lineages. + +Despite this, users can achieve a fast cross-namespace "clone and restore" workflow. For example, to quickly clone a large production workload into a test namespace ((e.g., for debugging, testing, or auditing)), a standard full restore would be too slow. Instead, users can manually take a CSI snapshot of the source PVC and provision a new PVC in the destination namespace from that snapshot. + +When a Velero restore is triggered against this new PVC, Velero detects the snapshot and uses CBT to write only the blocks that changed since the backup. This effectively "rolls back" the clone to the backup's state, drastically reducing data transfer and speeding up the restore. + +The key requirements for this approach are: +1. **Manual Cloning:** Users must manually snapshot the source PVC and clone it to the destination namespace before the restore. *(Note: Users must manually recreate the `VolumeSnapshotContent` and `VolumeSnapshot` in the destination namespace, or use `CrossNamespaceVolumeDataSource` if supported).* +2. **Workload Management:** Ensure no Pods are mounting the destination PVC during the restore to prevent data corruption. +3. **Snapshot Detection:** Velero inspects the destination PVC's `dataSource`. If it is a `VolumeSnapshot`, Velero uses its `SnapshotHandle` along with the backup's handle to calculate CBT. +4. **One-Shot Operation:** This is a one-time process. To restore a different backup later, users must clean up the destination namespace and repeat the workflow. +5. **Snapshot Cleanup:** Users must manually delete the temporary snapshot after the restore completes. + +### Pre-flight Checks + +Before initiating an in-place restore for a volume, Velero performs the following pre-flight checks to ensure the operation is safe and valid: + +#### 1. PVC is Not Actively Used by a Running Pod +Velero verifies that the target PVC is not currently mounted or consumed by any running Pods in the cluster. If the PVC is in use, Velero will skip the in-place restore for that volume and log an error. This enforces the prerequisite that users must completely delete consuming workloads prior to the restore, which prevents data corruption and avoids deadlocks caused by the Kubernetes `pvc-protection` finalizer during PVC recreation. + +#### 2. PVC is Bound to the Original PV +Velero checks whether the existing PVC in the cluster is still bound to the same PersistentVolume (PV) it was bound to at the time of the backup. If the PVC is bound to a different PV, performing an in-place restore (especially an incremental one that relies on Changed Block Tracking) may be unsafe or result in unpredictable behavior. If this check fails, Velero will log an error and skip the in-place restore for that volume. + + +#### 3. Volume Size Validation + +For in-place restores, the target volume must be large enough to accommodate the backed-up data. While the data path performs size checks during the actual restoration (only for block data mover), Velero will fail early to prevent unnecessary operations (such as taking a temporary snapshot). + +Before initiating an in-place restore, Velero compares the existing PV's size (`pv.spec.capacity.storage`) against the backup's data size (retrieved from the backup volume info). If the target PV is smaller than the backup data size, Velero will log an error and skip the volume data restoration. + +### Error Handling + +It is highly recommended that users create a backup (e.g., a CSI snapshot backup without data movement, if possible) before initiating an in-place restore. This ensures that the original state can be recovered in the event of a restore failure. + +If an in-place restore fails, Velero will intentionally leave certain temporary resources intact, such as the temporary PVC bound to the existing PV. Velero does not automatically clean up these resources because doing so could inadvertently trigger the deletion of the underlying storage volume. In such failure scenarios, users must manually clean up these temporary resources and, if necessary, use their pre-restore backup to recover the system's state. + +### Restore Workflow Update + +This section outlines the step-by-step control path and data path workflows for in-place restores. The exact sequence of operations depends on the backup method (CSI snapshot vs. file system backup), the chosen data mover (block vs. file system), and the target volume mode (block vs. file system). The following subsections detail the mechanisms for each supported scenario. + +#### In-place Incremental Restore for CSI Snapshot with Block Data Move for Block Volumes + +**Control Path** + +PVC CSI RIA: +- Capture the `volume.kubernetes.io/selected-node` annotation from the existing PVC into the Velero-internal carrier annotation before deleting the PVC, so the restore engine can re-apply it to the recreated target PVC (see [Handling Cross-Zone Scheduling](#handling-cross-zone-scheduling-waitforfirstconsumer)). +- Create a snapshot of the existing `PVC` to serve as the baseline for CBT delta calculations. +- Patch the existing PV's reclaim policy to `Retain`. +- Delete the existing PVC. +- Create a `DataDownload` resource referencing this snapshot and the existing `PV`, with `restoreType` set to `incremental`. + +Restore Exposer: +- Create a temporary restore PVC and bind it to the existing PV. +- Create a temporary restore Pod that mounts the temporary restore PVC. + +**Data Path** + +Block Uploader: +- The block uploader leverages Changed Block Tracking (CBT) to calculate the delta between the volume's current state and the backup snapshot. By skipping unchanged blocks and exclusively overwriting the modified ones, it significantly reduces I/O operations and accelerates the overall restore process. If the underlying storage system lacks CBT support, Velero will automatically fall back to performing an in-place full restore. + +#### In-place Full Restore for CSI Snapshot with Block Data Move for Block Volumes + +The workflow is identical to the **In-place Incremental Restore for CSI Snapshot with Block Data Move for Block Volumes**, with the following exceptions: +- No baseline snapshot is taken. +- The uploader does not use CBT to calculate deltas; instead, it overwrites all data on the volume. + +#### In-place Incremental Restore for CSI Snapshot with File System Data Move for File System Volumes + +**Control Path** + +The control path workflow is identical to the **In-place Incremental Restore for CSI Snapshot with Block Data Move for Block Volumes**, with the following exceptions: +- No baseline snapshot is taken. + +**Data Path** + +Kopia Uploader: +- Set the `incremental` flag to `true` when initiating the restore with the Kopia uploader. +- Pass the `deleteExtraFiles` configuration to the Kopia uploader based on the user's settings. +- Kopia evaluates file metadata (e.g., modification times and sizes) to identify changed files. It skips downloading and overwriting files that are identical to the backup, only restoring those that are modified, missing, or corrupted. + +#### In-place Full Restore for CSI Snapshot with File System Data Move for File System Volumes + +The workflow is identical to the **In-place Incremental Restore for CSI Snapshot with File System Data Move for File System Volumes**, with the following exceptions: +- The `restoreType` flag set to `full`. +- The Kopia uploader does not evaluate file metadata to skip unchanged files; instead, it overwrites all data on the target volume. + +#### In-place Incremental Restore for CSI Snapshot with Block Data Move for File System Volumes + +**Control Path** + +PVC CSI RIA: +- Capture the `volume.kubernetes.io/selected-node` annotation from the existing PVC into the Velero-internal carrier annotation before deleting the PVC, so the restore engine can re-apply it to the recreated target PVC (see [Handling Cross-Zone Scheduling](#handling-cross-zone-scheduling-waitforfirstconsumer)). +- Create a snapshot of the existing `PVC` to serve as the baseline for CBT delta calculations. +- Patch the existing `PV` to set its `persistentVolumeReclaimPolicy` to `Retain`. +- Delete the existing `PVC`. +- Create a `DataDownload` resource referencing the snapshot and the existing `PV`, with `restoreType` set to `incremental`. + +Restore Exposer: +- Delete the existing `PV`. +- Create a temporary restore `PV` with `volumeMode` set to `Block`, using the same volume handle as the original `PV`. +- Reset the bind information of the temporary restore `PV` to ensure it only binds to the temporary restore `PVC`. +- Create a temporary restore `PVC` with `volumeMode` set to `Block`. +- Create a temporary restore Pod that mounts the temporary restore `PVC`. + +**Data Path** + +Block Uploader: +- The block uploader leverages Changed Block Tracking (CBT) to calculate the delta between the volume's current state and the backup snapshot. By skipping unchanged blocks and exclusively overwriting the modified ones, it significantly reduces I/O operations and accelerates the overall restore process. If the underlying storage system lacks CBT support, Velero will automatically fall back to performing an in-place full restore. + +**Control Path (Post-Restore)** + +Restore Exposer: +- Delete the temporary restore Pod, `PVC`, and `PV`. +- Recreate the original `PV` with its `volumeMode` set back to `Filesystem`. +- Proceed with the standard process to allow the target `PVC` to bind to the recreated `PV`. + +#### In-place Full Restore for CSI Snapshot with Block Data Move for File System Volumes + +The workflow is identical to the **In-place Incremental Restore for CSI Snapshot with Block Data Move for File System Volumes**, with the following exceptions: +- No baseline snapshot is taken. +- The uploader does not use CBT to calculate deltas; instead, it overwrites all data on the volume. + +#### In-place Incremental Restore for File System Backup for File System Volumes + +**Control Path** + +- Create a `PodVolumeRestore` resource with `restoreType` set to `incremental`. + +**Data Path** + +Kopia Uploader: +- Set the `incremental` flag to `true` when initiating the restore with the Kopia uploader. +- Pass the `deleteExtraFiles` configuration to the Kopia uploader based on the user's settings. +- Similar to the CSI File System Data Move, Kopia evaluates file metadata to skip unchanged files and only restores those that are modified or missing. + +#### In-place Full Restore for File System Backup for File System Volumes + +The workflow is identical to the **In-place Incremental Restore for File System Backup for File System Volumes**, with the following exceptions: +- The `restoreType` flag is set to `full`. +- The Kopia uploader does not evaluate file metadata to skip unchanged files; instead, it overwrites all data on the target volume. + +## Installation + +No change to Installation. + +## Upgrade + +No impacts to Upgrade. The new fields in the CRDs are all optional fields and have backwards compatible values. \ No newline at end of file diff --git a/examples/default-resource-modifier-cni.yaml b/examples/default-resource-modifier-cni.yaml new file mode 100644 index 000000000..352180f5a --- /dev/null +++ b/examples/default-resource-modifier-cni.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: default-restore-resource-modifiers + namespace: velero +data: + resource-modifiers.yaml: | + version: v1 + resourceModifierRules: + - conditions: + groupResource: pods + mergePatches: + - patchData: | + metadata: + annotations: + k8s.ovn.org/pod-networks: null + k8s.v1.cni.cncf.io/network-status: null + k8s.v1.cni.cncf.io/networks-status: null diff --git a/go.mod b/go.mod index a2c41faf6..c38ab050a 100644 --- a/go.mod +++ b/go.mod @@ -44,13 +44,13 @@ require ( github.com/vmware-tanzu/velero/pkg/apis v0.0.0 go.uber.org/zap v1.28.0 go.yaml.in/yaml/v3 v3.0.4 - golang.org/x/mod v0.36.0 + golang.org/x/mod v0.40.0 golang.org/x/oauth2 v0.36.0 - golang.org/x/sys v0.46.0 - golang.org/x/text v0.37.0 + golang.org/x/sys v0.47.0 + golang.org/x/text v0.41.0 google.golang.org/api v0.283.0 - google.golang.org/grpc v1.81.1 - google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af + google.golang.org/grpc v1.82.1 + google.golang.org/protobuf v1.36.12 k8s.io/api v0.36.0 k8s.io/apiextensions-apiserver v0.36.0 k8s.io/apimachinery v0.36.0 @@ -76,7 +76,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect github.com/Masterminds/semver/v3 v3.5.0 // indirect @@ -189,7 +189,7 @@ require ( github.com/zeebo/blake3 v0.2.4 // indirect github.com/zeebo/xxh3 v1.1.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.42.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.43.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect go.opentelemetry.io/otel v1.44.0 // indirect @@ -200,13 +200,13 @@ require ( go.starlark.net v0.0.0-20241226192728-8dfa5b98479f // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect - golang.org/x/crypto v0.52.0 // indirect + golang.org/x/crypto v0.55.0 // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect - golang.org/x/net v0.55.0 // indirect - golang.org/x/sync v0.21.0 // indirect - golang.org/x/term v0.43.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/term v0.45.0 // indirect golang.org/x/time v0.15.0 // indirect - golang.org/x/tools v0.44.0 // indirect + golang.org/x/tools v0.49.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect diff --git a/go.sum b/go.sum index ed0070272..f744bb2e4 100644 --- a/go.sum +++ b/go.sum @@ -48,8 +48,8 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMs github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 h1:IEjq88XO4PuBDcvmjQJcQGg+w+UaafSy8G5Kcb5tBhI= github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5/go.mod h1:exZ0C/1emQJAw5tHOaUDyY1ycttqBAPcxuzf7QbY6ec= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 h1:rIkQfkCOVKc1OiRCNcSDD8ml5RJlZbH/Xsq7lbpynwc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0/go.mod h1:RD2SsorTmYhF6HkTmDw7KmPYQk8OBYwTkuasChwv7R4= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ= @@ -466,8 +466,8 @@ github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/detectors/gcp v1.42.0 h1:kpt2PEJuOuqYkPcktfJqWWDjTEd/FNgrxcniL7kQrXQ= -go.opentelemetry.io/contrib/detectors/gcp v1.42.0/go.mod h1:W9zQ439utxymRrXsUOzZbFX4JhLxXU4+ZnCt8GG7yA8= +go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmfElKI4d/Myu2OxDFU= +go.opentelemetry.io/contrib/detectors/gcp v1.43.0/go.mod h1:RyaZMFY7yi1kAs45S6mbFGz8O8rqB0dTY14uzvG4LCs= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= @@ -501,27 +501,27 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= -golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= -golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= +golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -532,22 +532,22 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -564,10 +564,10 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1: google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= -google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= -google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= -google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/hack/build-image/Dockerfile b/hack/build-image/Dockerfile index 88dedde95..8978855b2 100644 --- a/hack/build-image/Dockerfile +++ b/hack/build-image/Dockerfile @@ -15,6 +15,8 @@ FROM --platform=$TARGETPLATFORM golang:1.26-trixie ARG GOPROXY +ARG PROTOC_GEN_GO_VERSION +ARG GOIMPORTS_VERSION ENV GO111MODULE=on # Use a proxy for go modules to reduce the likelihood of various hosts being down and breaking the build @@ -27,16 +29,26 @@ RUN go install sigs.k8s.io/controller-runtime/tools/setup-envtest@v0.0.0-2026030 ENVTEST_ASSETS_DIR=$(setup-envtest use 1.33.0 --bin-dir /usr/local/kubebuilder/bin -p path) && \ cp -r ${ENVTEST_ASSETS_DIR}/* /usr/local/kubebuilder/bin/ -RUN wget --quiet https://github.com/kubernetes-sigs/kubebuilder/releases/download/v3.2.0/kubebuilder_linux_$(go env GOARCH) && \ - mv kubebuilder_linux_$(go env GOARCH) /usr/local/kubebuilder/bin/kubebuilder && \ +RUN set -eux; \ + ARCH="$(go env GOARCH)"; \ + case "$ARCH" in \ + amd64) KUBEBUILDER_SHA256="102bb0f586dcb50951aded67856483a2ee114057c56475b3cda6051a12832a72" ;; \ + arm64) KUBEBUILDER_SHA256="0a340ea925c801aa71344becdefce96eda6fa0bc92352b9c7bcb36a4f8c56314" ;; \ + ppc64le) KUBEBUILDER_SHA256="74473d094908caad852a77088f64bb64eb4c79497f6695eb5e9e8bc4bacd9409" ;; \ + *) echo "Unsupported kubebuilder architecture: $ARCH" >&2; exit 1 ;; \ + esac; \ + FILE="kubebuilder_linux_$ARCH"; \ + wget --quiet "https://github.com/kubernetes-sigs/kubebuilder/releases/download/v3.2.0/$FILE"; \ + echo "$KUBEBUILDER_SHA256 $FILE" | sha256sum -c -; \ + mv "$FILE" /usr/local/kubebuilder/bin/kubebuilder; \ chmod +x /usr/local/kubebuilder/bin/kubebuilder # get controller-tools RUN go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.16.5 -# get goimports (the revision is pinned so we don't indiscriminately update, but the particular commit -# is not important) -RUN go install golang.org/x/tools/cmd/goimports@v0.33.0 +# get goimports, version derived from go.mod's golang.org/x/tools requirement +# (see https://github.com/velero-io/velero/issues/10023) +RUN go install golang.org/x/tools/cmd/goimports@${GOIMPORTS_VERSION} # get protoc compiler and golang plugin WORKDIR /root @@ -50,28 +62,29 @@ RUN apt-get update && apt-get install -y unzip # cpu = "ppcle_64" # snippet from: https://github.com/protocolbuffers/protobuf/blob/d445953603e66eb8992a39b4e10fcafec8501f24/protobuf_release.bzl#L18-L24 # cpu names: https://github.com/bazelbuild/platforms/blob/main/cpu/BUILD -RUN ARCH=$(go env GOARCH) && \ - if [ "$ARCH" = "s390x" ] ; then \ - ARCH="s390_64"; \ - elif [ "$ARCH" = "arm64" ] ; then \ - ARCH="aarch_64"; \ - elif [ "$ARCH" = "ppc64le" ] ; then \ - ARCH="ppcle_64"; \ - elif [ "$ARCH" = "ppc64" ] ; then \ - ARCH="ppcle_64"; \ - else \ - ARCH=$(uname -m); \ - fi && echo "ARCH=$ARCH" && \ - wget --quiet https://github.com/protocolbuffers/protobuf/releases/download/v25.2/protoc-25.2-linux-$ARCH.zip && \ - unzip protoc-25.2-linux-$ARCH.zip; \ - rm *.zip && \ - mv bin/protoc /usr/bin/protoc && \ - mv include/google /usr/include && \ - chmod a+x /usr/include/google && \ - chmod a+x /usr/include/google/protobuf && \ - chmod a+r -R /usr/include/google && \ +RUN set -eux; \ + GOARCH="$(go env GOARCH)"; \ + case "$GOARCH" in \ + amd64) ARCH="x86_64"; PROTOC_SHA256="78ab9c3288919bdaa6cfcec6127a04813cf8a0ce406afa625e48e816abee2878" ;; \ + 386) ARCH="x86_32"; PROTOC_SHA256="cc1c6e31a9b333c3e6d026aac5fdc1f7d70c6cd8851631505188ca9826acee5a" ;; \ + arm64) ARCH="aarch_64"; PROTOC_SHA256="07683afc764e4efa3fa969d5f049fbc2bdfc6b4e7786a0b233413ac0d8753f6b" ;; \ + ppc64|ppc64le) ARCH="ppcle_64"; PROTOC_SHA256="cea283337101ed08ff6c76a98461b1d871bac21f41dc1dabdfddaa5d99df9339" ;; \ + s390x) ARCH="s390_64"; PROTOC_SHA256="8a13ec6518585f7664d58f929417c9e6d0c4aeedf3bcdd854aeafceb5ef0a389" ;; \ + *) echo "Unsupported protoc architecture: $GOARCH" >&2; exit 1 ;; \ + esac; \ + echo "ARCH=$ARCH"; \ + FILE="protoc-25.2-linux-$ARCH.zip"; \ + wget --quiet "https://github.com/protocolbuffers/protobuf/releases/download/v25.2/$FILE"; \ + echo "$PROTOC_SHA256 $FILE" | sha256sum -c -; \ + unzip "$FILE"; \ + rm "$FILE"; \ + mv bin/protoc /usr/bin/protoc; \ + mv include/google /usr/include; \ + chmod a+x /usr/include/google; \ + chmod a+x /usr/include/google/protobuf; \ + chmod a+r -R /usr/include/google; \ chmod +x /usr/bin/protoc -RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.33.0 \ +RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@${PROTOC_GEN_GO_VERSION} \ && go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0 # get goreleaser @@ -82,17 +95,21 @@ RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.33.0 \ # {{- else if eq .Arch "386" }}i386 # {{- else }}{{ .Arch }}{{ end }} # {{- if .Arm }}v{{ .Arm }}{{ end -}} -RUN ARCH=$(go env GOARCH) && \ - if [ "$ARCH" = "amd64" ] ; then \ - ARCH="x86_64"; \ - elif [ "$ARCH" = "386" ] ; then \ - ARCH="i386"; \ - elif [ "$ARCH" = "ppc64le" ] ; then \ - ARCH="ppc64"; \ - fi && \ - wget --quiet "https://github.com/goreleaser/goreleaser/releases/download/v1.26.2/goreleaser_Linux_$ARCH.tar.gz" && \ - tar xvf goreleaser_Linux_$ARCH.tar.gz; \ - mv goreleaser /usr/bin/goreleaser && \ +RUN set -eux; \ + GOARCH="$(go env GOARCH)"; \ + case "$GOARCH" in \ + amd64) ARCH="x86_64"; GORELEASER_SHA256="cfbdf12e3ea20e4c3a209d07311f43c2e0baf20d5cce09bcdc232567e0f34307" ;; \ + 386) ARCH="i386"; GORELEASER_SHA256="21c236575cccd29588182b570b4ffe83ad8fb96cd3b13b2af79feafd8ae37b1b" ;; \ + arm64) ARCH="arm64"; GORELEASER_SHA256="2b984e2932b24be0d638c7dab7357a59d86eb79ca7fee1afd31be5ebb1847cbb" ;; \ + arm) ARCH="armv7"; GORELEASER_SHA256="6db2899885be19f123b36192a42dcfb3bb2b3e1009fec7277517969e96d8a7c6" ;; \ + ppc64|ppc64le) ARCH="ppc64"; GORELEASER_SHA256="76d060ebb8d48e76fde45983f87040fe3ac0ca37c5ace4648a956959b81bfdf0" ;; \ + *) echo "Unsupported goreleaser architecture: $GOARCH" >&2; exit 1 ;; \ + esac; \ + FILE="goreleaser_Linux_$ARCH.tar.gz"; \ + wget --quiet "https://github.com/goreleaser/goreleaser/releases/download/v1.26.2/$FILE"; \ + echo "$GORELEASER_SHA256 $FILE" | sha256sum -c -; \ + tar xvf "$FILE"; \ + mv goreleaser /usr/bin/goreleaser; \ chmod +x /usr/bin/goreleaser # get golangci-lint @@ -100,10 +117,5 @@ RUN ARCH=$(go env GOARCH) && \ # release API/CDN, which has been returning intermittent/persistent HTTP 504s. RUN go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.5.0 -# install kubectl -RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/$(go env GOARCH)/kubectl -RUN chmod +x ./kubectl -RUN mv ./kubectl /usr/local/bin - # Fix the "dubious ownership" issue from git when running goreleaser.sh RUN echo "[safe] \n\t directory = *" > /.gitconfig diff --git a/hack/crd-gen/v1/main.go b/hack/crd-gen/v1/main.go deleted file mode 100644 index 5f45b04e0..000000000 --- a/hack/crd-gen/v1/main.go +++ /dev/null @@ -1,134 +0,0 @@ -/* -Copyright the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This code embeds the CRD manifests in ../bases in ../crds/crds.go - -package main - -import ( - "bytes" - "compress/gzip" - "fmt" - "io" - "log" - "os" - "text/template" -) - -// This is relative to config/crd/crds -const goHeaderFile = "../../../../hack/boilerplate.go.txt" - -const tpl = `{{.GoHeader}} -// Code generated by crds_generate.go; DO NOT EDIT. - -package crds - -import ( - "bytes" - "compress/gzip" - "io" - - apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" - apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/client-go/kubernetes/scheme" -) - -var rawCRDs = [][]byte{ -{{- range .RawCRDs }} - []byte({{ . }}), -{{- end }} -} - -var CRDs = crds() - -func crds() []*apiextv1.CustomResourceDefinition { - apiextinstall.Install(scheme.Scheme) - decode := scheme.Codecs.UniversalDeserializer().Decode - var objs []*apiextv1.CustomResourceDefinition - for _, crd := range rawCRDs { - gzr, err := gzip.NewReader(bytes.NewReader(crd)) - if err != nil { - panic(err) - } - bytes, err := io.ReadAll(gzr) - if err != nil { - panic(err) - } - gzr.Close() - - obj, _, err := decode(bytes, nil, nil) - if err != nil { - panic(err) - } - objs = append(objs, obj.(*apiextv1.CustomResourceDefinition)) - } - return objs -} -` - -type templateData struct { - GoHeader string - RawCRDs []string -} - -func main() { - headerBytes, err := os.ReadFile(goHeaderFile) - if err != nil { - log.Fatalln(err) - } - - data := templateData{ - GoHeader: string(headerBytes), - } - - // This is relative to config/crd/crds - manifests, err := os.ReadDir("../bases") - if err != nil { - log.Fatalln(err) - } - - for _, crd := range manifests { - file, err := os.Open("../bases/" + crd.Name()) - if err != nil { - log.Fatalln(err) - } - - // gzip compress manifest - var buf bytes.Buffer - gzw := gzip.NewWriter(&buf) - if _, err := io.Copy(gzw, file); err != nil { - log.Fatalln(err) - } - file.Close() - gzw.Close() - - data.RawCRDs = append(data.RawCRDs, fmt.Sprintf("%q", buf.Bytes())) - } - - t, err := template.New("crd").Parse(tpl) - if err != nil { - log.Fatalln(err) - } - - out, err := os.Create("crds.go") - if err != nil { - log.Fatalln(err) - } - - if err := t.Execute(out, data); err != nil { - log.Fatalln(err) - } -} diff --git a/hack/update-3generated-crd-code.sh b/hack/update-3generated-crd-code.sh index 720639a40..cc7b92eda 100755 --- a/hack/update-3generated-crd-code.sh +++ b/hack/update-3generated-crd-code.sh @@ -55,6 +55,6 @@ controller-gen \ paths=./pkg/controller/... \ rbac:roleName=velero-perms -go generate ./config/crd/v1/crds - -go generate ./config/crd/v2alpha1/crds +# The CRD manifests above are embedded directly into the binary via +# go:embed (see config/crd/v1/crds.go and config/crd/v2alpha1/crds.go), +# so no further code generation step is required. diff --git a/hack/verify-build-image-tool-checksums.sh b/hack/verify-build-image-tool-checksums.sh new file mode 100755 index 000000000..a11f258ab --- /dev/null +++ b/hack/verify-build-image-tool-checksums.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# Copyright 2026 the Velero contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +DOCKERFILE="${ROOT_DIR}/hack/build-image/Dockerfile" + +verify_block() { + local tool=$1 + local start=$2 + local end=$3 + local hash_variable=$4 + local install_pattern=$5 + shift 5 + local expected_arches=("$@") + local block + + block=$(awk -v start="${start}" -v end="${end}" ' + $0 ~ start { printing = 1 } + printing { print } + printing && $0 ~ end { exit } + ' "${DOCKERFILE}") + + if [[ -z "${block}" ]]; then + echo "Unable to find ${tool} install block" >&2 + return 1 + fi + + local actual_arches + actual_arches=$(printf '%s\n' "${block}" | + sed -nE "s/^[[:space:]]*([[:alnum:]_|]+)\).*${hash_variable}=\"([[:xdigit:]]+)\".*/\1 \2/p") + + local expected_arch + for expected_arch in "${expected_arches[@]}"; do + if ! printf '%s\n' "${actual_arches}" | awk -v arch="${expected_arch}" ' + $1 == arch && length($2) == 64 && $2 ~ /^[0-9a-f]+$/ { found = 1 } + END { exit !found } + '; then + echo "${tool} is missing a lowercase 64-hex SHA-256 for ${expected_arch}" >&2 + return 1 + fi + done + + local actual_count + actual_count=$(printf '%s\n' "${actual_arches}" | sed '/^$/d' | wc -l | tr -d ' ') + if [[ "${actual_count}" -ne "${#expected_arches[@]}" ]]; then + echo "${tool} architecture mapping changed; update this verification gate" >&2 + printf '%s\n' "${actual_arches}" >&2 + return 1 + fi + + if ! printf '%s\n' "${block}" | grep -Eq '^ \*\).*Unsupported .+ architecture:.+exit 1'; then + echo "${tool} does not fail closed for unknown architectures" >&2 + return 1 + fi + + local download_line checksum_line install_line + download_line=$(printf '%s\n' "${block}" | grep -n 'wget --quiet' | head -1 | cut -d: -f1) + checksum_line=$(printf '%s\n' "${block}" | grep -n "echo \"\$${hash_variable} \$FILE\" | sha256sum -c -" | head -1 | cut -d: -f1) + install_line=$(printf '%s\n' "${block}" | grep -nE "${install_pattern}" | head -1 | cut -d: -f1) + + if [[ -z "${download_line}" || -z "${checksum_line}" || -z "${install_line}" || + "${download_line}" -ge "${checksum_line}" || "${checksum_line}" -ge "${install_line}" ]]; then + echo "${tool} must download, verify, then install/extract in that order" >&2 + return 1 + fi + + if ! printf '%s\n' "${block}" | grep -q '^RUN set -eux;'; then + echo "${tool} install block must use strict shell error handling" >&2 + return 1 + fi +} + +verify_block \ + kubebuilder \ + '^RUN set -eux;.*$' \ + '^# get controller-tools$' \ + KUBEBUILDER_SHA256 \ + 'mv "\$FILE"' \ + amd64 arm64 ppc64le + +verify_block \ + protoc \ + '^# cpu names:' \ + '^RUN go install google.golang.org/protobuf' \ + PROTOC_SHA256 \ + 'unzip "\$FILE"' \ + amd64 386 arm64 'ppc64|ppc64le' s390x + +verify_block \ + goreleaser \ + '^# goreleaser name template' \ + '^# get golangci-lint$' \ + GORELEASER_SHA256 \ + 'tar xvf "\$FILE"' \ + amd64 386 arm64 arm 'ppc64|ppc64le' + +echo "Verified pinned build-tool checksums and fail-closed install ordering" diff --git a/hack/verify-generated-crd-code.sh b/hack/verify-generated-crd-code.sh deleted file mode 100755 index 1d9f23cab..000000000 --- a/hack/verify-generated-crd-code.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -e -# -# Copyright the Velero contributors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -HACK_DIR=$(dirname "${BASH_SOURCE}") - -${HACK_DIR}/update-3generated-crd-code.sh - -# ensure no changes to generated CRDs -if ! git diff --exit-code config/crd/v1/crds/crds.go config/crd/v2alpha1/crds/crds.go &> /dev/null; then - # revert changes to state before running CRD generation to stay consistent - # with code-generator `--verify-only` option which discards generated changes - git checkout config/crd - - echo "CRD verification - failed! Generated CRDs are out-of-date, please run 'make update' and 'git add' the generated file(s)." - exit 1 -fi diff --git a/internal/delete/actions/csi/volumesnapshotcontent_action.go b/internal/delete/actions/csi/volumesnapshotcontent_action.go index c57a0eb1a..4b7895341 100644 --- a/internal/delete/actions/csi/volumesnapshotcontent_action.go +++ b/internal/delete/actions/csi/volumesnapshotcontent_action.go @@ -86,7 +86,7 @@ func (p *volumeSnapshotContentDeleteItemAction) Execute( // This handles legacy (pre-1.15) backups where the original VSC // with DeletionPolicy=Retain still exists in the cluster. originalVSCName := snapCont.Name - if cleaned := p.tryDeleteOriginalVSC(context.TODO(), originalVSCName); cleaned { + if cleaned := p.tryDeleteOriginalVSC(context.TODO(), originalVSCName, input.Backup.Name); cleaned { p.log.Infof("Successfully deleted original VolumeSnapshotContent %s from cluster, skipping temp VSC creation", originalVSCName) return nil } @@ -149,10 +149,11 @@ func (p *volumeSnapshotContentDeleteItemAction) Execute( // the cluster (legacy pre-1.15 backups). It patches the DeletionPolicy to // Delete so the CSI driver also removes the cloud snapshot, then deletes // the VSC object itself. -// Returns true if the original VSC was found and deletion was initiated. +// Returns true if the original VSC was found, carries the backup label, and deletion was initiated. func (p *volumeSnapshotContentDeleteItemAction) tryDeleteOriginalVSC( ctx context.Context, vscName string, + backupName string, ) bool { existing := new(snapshotv1api.VolumeSnapshotContent) if err := p.crClient.Get(ctx, crclient.ObjectKey{Name: vscName}, existing); err != nil { @@ -164,6 +165,15 @@ func (p *volumeSnapshotContentDeleteItemAction) tryDeleteOriginalVSC( return false } + if !kubeutil.HasBackupLabel(&existing.ObjectMeta, backupName) { + p.log.Warnf( + "Original VolumeSnapshotContent %s in cluster does not belong to backup %s, skipping direct deletion", + vscName, + backupName, + ) + return false + } + p.log.Debugf("Found original VolumeSnapshotContent %s in cluster (legacy backup), cleaning up directly", vscName) // Patch DeletionPolicy to Delete so the CSI driver removes the cloud snapshot diff --git a/internal/delete/actions/csi/volumesnapshotcontent_action_test.go b/internal/delete/actions/csi/volumesnapshotcontent_action_test.go index e8a0b5865..25cc69b82 100644 --- a/internal/delete/actions/csi/volumesnapshotcontent_action_test.go +++ b/internal/delete/actions/csi/volumesnapshotcontent_action_test.go @@ -122,7 +122,29 @@ func TestVSCExecute(t *testing.T) { backup: builder.ForBackup("velero", "backup").Result(), expectErr: false, preExistingVSC: &snapshotv1api.VolumeSnapshotContent{ - ObjectMeta: metav1.ObjectMeta{Name: "bar"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "backup", + }, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, + Driver: "disk.csi.azure.com", + Source: snapshotv1api.VolumeSnapshotContentSource{SnapshotHandle: stringPtr("snap-123")}, + VolumeSnapshotRef: corev1api.ObjectReference{Name: "vs-1", Namespace: "default"}, + }, + }, + }, + { + name: "Original VSC exists in cluster without backup label, falls through to temp VSC flow", + vsc: builder.ForVolumeSnapshotContent("bar").ObjectMeta(builder.WithLabelsMap(map[string]string{velerov1api.BackupNameLabel: "backup"})).Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: &snapshotHandleStr}).Result(), + backup: builder.ForBackup("velero", "backup").Result(), + expectErr: false, + preExistingVSC: &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, Driver: "disk.csi.azure.com", @@ -200,22 +222,51 @@ func TestNewVolumeSnapshotContentDeleteItemAction(t *testing.T) { func TestTryDeleteOriginalVSC(t *testing.T) { tests := []struct { - name string - vscName string - existing *snapshotv1api.VolumeSnapshotContent - createIt bool - expectRet bool + name string + vscName string + backupName string + existing *snapshotv1api.VolumeSnapshotContent + createIt bool + expectRet bool }{ { - name: "VSC not found in cluster, returns false", - vscName: "not-found", + name: "VSC not found in cluster, returns false", + vscName: "not-found", + backupName: "test-backup", + expectRet: false, + }, + { + name: "VSC found in cluster without backup label, returns false", + vscName: "unlabeled-vsc", + backupName: "test-backup", + existing: &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{Name: "unlabeled-vsc"}, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, + Driver: "disk.csi.azure.com", + Source: snapshotv1api.VolumeSnapshotContentSource{ + SnapshotHandle: stringPtr("snap-123"), + }, + VolumeSnapshotRef: corev1api.ObjectReference{ + Name: "vs-1", + Namespace: "default", + }, + }, + }, + createIt: true, expectRet: false, }, { - name: "VSC found with Retain policy, patches and deletes", - vscName: "legacy-vsc", + name: "VSC found with Retain policy and matching backup label, patches and deletes", + vscName: "legacy-vsc", + backupName: "test-backup", existing: &snapshotv1api.VolumeSnapshotContent{ - ObjectMeta: metav1.ObjectMeta{Name: "legacy-vsc"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "legacy-vsc", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, Driver: "disk.csi.azure.com", @@ -232,10 +283,16 @@ func TestTryDeleteOriginalVSC(t *testing.T) { expectRet: true, }, { - name: "VSC found with Delete policy already, just deletes", - vscName: "already-delete-vsc", + name: "VSC found with Delete policy and matching backup label, just deletes", + vscName: "already-delete-vsc", + backupName: "test-backup", existing: &snapshotv1api.VolumeSnapshotContent{ - ObjectMeta: metav1.ObjectMeta{Name: "already-delete-vsc"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "already-delete-vsc", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ DeletionPolicy: snapshotv1api.VolumeSnapshotContentDelete, Driver: "disk.csi.azure.com", @@ -266,7 +323,7 @@ func TestTryDeleteOriginalVSC(t *testing.T) { require.NoError(t, crClient.Create(t.Context(), test.existing)) } - result := p.tryDeleteOriginalVSC(t.Context(), test.vscName) + result := p.tryDeleteOriginalVSC(t.Context(), test.vscName, test.backupName) require.Equal(t, test.expectRet, result) // If cleanup succeeded, verify the VSC is gone @@ -289,13 +346,18 @@ func TestTryDeleteOriginalVSC(t *testing.T) { log: logrus.StandardLogger(), crClient: errClient, } - require.False(t, p.tryDeleteOriginalVSC(t.Context(), "some-vsc")) + require.False(t, p.tryDeleteOriginalVSC(t.Context(), "some-vsc", "test-backup")) }) t.Run("Patch fails, returns false", func(t *testing.T) { realClient := velerotest.NewFakeControllerRuntimeClient(t) vsc := &snapshotv1api.VolumeSnapshotContent{ - ObjectMeta: metav1.ObjectMeta{Name: "patch-fail-vsc"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "patch-fail-vsc", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, Driver: "disk.csi.azure.com", @@ -313,13 +375,18 @@ func TestTryDeleteOriginalVSC(t *testing.T) { log: logrus.StandardLogger(), crClient: errClient, } - require.False(t, p.tryDeleteOriginalVSC(t.Context(), "patch-fail-vsc")) + require.False(t, p.tryDeleteOriginalVSC(t.Context(), "patch-fail-vsc", "test-backup")) }) t.Run("Delete fails, returns false", func(t *testing.T) { realClient := velerotest.NewFakeControllerRuntimeClient(t) vsc := &snapshotv1api.VolumeSnapshotContent{ - ObjectMeta: metav1.ObjectMeta{Name: "delete-fail-vsc"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "delete-fail-vsc", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, Spec: snapshotv1api.VolumeSnapshotContentSpec{ DeletionPolicy: snapshotv1api.VolumeSnapshotContentDelete, Driver: "disk.csi.azure.com", @@ -337,7 +404,7 @@ func TestTryDeleteOriginalVSC(t *testing.T) { log: logrus.StandardLogger(), crClient: errClient, } - require.False(t, p.tryDeleteOriginalVSC(t.Context(), "delete-fail-vsc")) + require.False(t, p.tryDeleteOriginalVSC(t.Context(), "delete-fail-vsc", "test-backup")) }) } diff --git a/internal/delete/delete_item_action_handler.go b/internal/delete/delete_item_action_handler.go index 2a16044ee..89a638331 100644 --- a/internal/delete/delete_item_action_handler.go +++ b/internal/delete/delete_item_action_handler.go @@ -114,7 +114,10 @@ func InvokeDeleteActions(ctx *Context) error { // Process individual items from the backup for _, item := range items { - itemPath := archive.GetItemFilePath(dir, resource, namespace, item) + itemPath, err := archive.GetItemFilePath(dir, resource, namespace, item) + if err != nil { + return errors.Wrapf(err, "could not build item path: %v", item) + } // obj is the Unstructured item from the backup obj, err := archive.Unmarshal(ctx.Filesystem, itemPath) diff --git a/internal/hook/item_hook_handler.go b/internal/hook/item_hook_handler.go index bed48c5ea..a2a58fc4a 100644 --- a/internal/hook/item_hook_handler.go +++ b/internal/hook/item_hook_handler.go @@ -365,6 +365,12 @@ func getPodExecHookFromAnnotations(annotations map[string]string, phase HookPhas func parseStringToCommand(commandValue string) []string { var command []string + // An empty command means the container image's own entrypoint should be used. + // Callers that require a command already return early; getInitContainerFromAnnotation + // deliberately allows this case, so return nil rather than indexing an empty string. + if commandValue == "" { + return nil + } // check for json array if commandValue[0] == '[' { if err := json.Unmarshal([]byte(commandValue), &command); err != nil { @@ -419,7 +425,7 @@ func getInitContainerFromAnnotation(podName string, annotations map[string]strin return nil } if command == "" { - log.Infof("RestoreHook init container for pod %s is using container's default entrypoint", podName, containerImage) + log.Infof("RestoreHook init container for pod %s is using the default entrypoint of image %s", podName, containerImage) } if containerName == "" { uid, err := uuid.NewRandom() diff --git a/internal/hook/item_hook_handler_test.go b/internal/hook/item_hook_handler_test.go index 1f2df9469..792bcd66e 100644 --- a/internal/hook/item_hook_handler_test.go +++ b/internal/hook/item_hook_handler_test.go @@ -1287,6 +1287,25 @@ func TestGetInitContainerFromAnnotations(t *testing.T) { podRestoreHookInitContainerCommandAnnotationKey: "[foobarbaz", }, }, + { + name: "should use the image's default entrypoint when the command annotation is empty", + expectNil: false, + expected: builder.ForContainer("restore-init1", "busy-box").Result(), + inputAnnotations: map[string]string{ + podRestoreHookInitContainerImageAnnotationKey: "busy-box", + podRestoreHookInitContainerNameAnnotationKey: "restore-init", + podRestoreHookInitContainerCommandAnnotationKey: "", + }, + }, + { + name: "should use the image's default entrypoint when the command annotation is missing", + expectNil: false, + expected: builder.ForContainer("restore-init1", "busy-box").Result(), + inputAnnotations: map[string]string{ + podRestoreHookInitContainerImageAnnotationKey: "busy-box", + podRestoreHookInitContainerNameAnnotationKey: "restore-init", + }, + }, } for _, tc := range testCases { diff --git a/internal/resourcepolicies/resource_policies.go b/internal/resourcepolicies/resource_policies.go index 235f48ed5..ad8f06ee4 100644 --- a/internal/resourcepolicies/resource_policies.go +++ b/internal/resourcepolicies/resource_policies.go @@ -21,16 +21,17 @@ import ( "fmt" "strings" - "k8s.io/apimachinery/pkg/util/sets" - "github.com/cockroachdb/errors" "github.com/gobwas/glob" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" crclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - datamover "github.com/vmware-tanzu/velero/pkg/util/datamover" + "github.com/vmware-tanzu/velero/pkg/util/datamover" "github.com/vmware-tanzu/velero/pkg/util/wildcard" ) @@ -53,11 +54,16 @@ const ( // DataMoverParameter is the key of the action parameter that selects the data // mover to be used for the matched volumes when the action type is snapshot. DataMoverParameter = "dataMover" + + // SnapshotClassParameter is the key of the action parameter that selects the + // VolumeSnapshotClass to use for CSI snapshots when the action type is snapshot. + SnapshotClassParameter = "snapshotClass" ) // validDataMovers is the set of data mover values accepted in the snapshot // action's dataMover parameter. var validDataMovers = map[string]struct{}{ + datamover.DataMoverTypeEmpty: {}, datamover.DataMoverTypeVelero: {}, datamover.DataMoverTypeVeleroFs: {}, datamover.DataMoverTypeVeleroBlock: {}, @@ -89,24 +95,108 @@ func (a *Action) GetDataMover() (string, error) { if !ok { return datamover.GetDefaultBuiltInDataMover(), nil } + dataMover, ok := raw.(string) if !ok { return "", fmt.Errorf("parameter %q must be a string, got %T", DataMoverParameter, raw) } if _, ok := validDataMovers[dataMover]; !ok { - return "", fmt.Errorf("invalid %q value %q, valid values are %q, %q, %q", - DataMoverParameter, dataMover, datamover.DataMoverTypeVelero, datamover.DataMoverTypeVeleroFs, datamover.DataMoverTypeVeleroBlock) + return "", fmt.Errorf("invalid %q value %q, valid values are %q, %q, %q, %q", + DataMoverParameter, dataMover, datamover.DataMoverTypeEmpty, datamover.DataMoverTypeVelero, datamover.DataMoverTypeVeleroFs, datamover.DataMoverTypeVeleroBlock) } + + // Return default data mover for backup's volume policy, when the data mover's original value is legacy value: "" or "velero". + if dataMover == datamover.DataMoverTypeEmpty || dataMover == datamover.DataMoverTypeVelero { + dataMover = datamover.GetDefaultBuiltInDataMover() + } + return dataMover, nil } +// GetSnapshotClass returns the VolumeSnapshotClass name configured in the +// snapshot action's snapshotClass parameter. The snapshotClass parameter is +// only meaningful for the snapshot action, so it returns an error when the +// action is nil or its type is not snapshot. When the parameter is absent, +// it returns an empty string, meaning the caller should fall back to the +// existing VolumeSnapshotClass selection logic. +func (a *Action) GetSnapshotClass() (string, error) { + if a == nil || a.Type != Snapshot { + return "", fmt.Errorf("the %q parameter is only supported for the %q action", SnapshotClassParameter, Snapshot) + } + if len(a.Parameters) == 0 { + return "", nil + } + raw, ok := a.Parameters[SnapshotClassParameter] + if !ok { + return "", nil + } + snapshotClass, ok := raw.(string) + if !ok { + return "", fmt.Errorf("parameter %q must be a string, got %T", SnapshotClassParameter, raw) + } + return snapshotClass, nil +} + +// PolicyLabelSelector mirrors metav1.LabelSelector with yaml tags for ConfigMap decode. +// metav1.LabelSelector only has json tags, which do not populate under go.yaml.in/yaml/v3. +type PolicyLabelSelector struct { + MatchLabels map[string]string `yaml:"matchLabels,omitempty"` + MatchExpressions []PolicyLabelSelectorRequirement `yaml:"matchExpressions,omitempty"` +} + +// PolicyLabelSelectorRequirement mirrors metav1.LabelSelectorRequirement with yaml tags. +type PolicyLabelSelectorRequirement struct { + Key string `yaml:"key"` + Operator string `yaml:"operator"` + Values []string `yaml:"values,omitempty"` +} + +// IsPresentLabelSelector reports whether s defines any label constraints. +// Empty {} (nil MatchLabels and empty MatchExpressions) is treated as absent. +func IsPresentLabelSelector(s *PolicyLabelSelector) bool { + return s != nil && (len(s.MatchLabels) > 0 || len(s.MatchExpressions) > 0) +} + +// ToMetaV1LabelSelector converts the YAML mirror type to metav1.LabelSelector. +// Conversion itself is infallible; call LabelSelectorAsSelector (or +// SelectorFromPolicyLabelSelector) to validate operators and values. +func ToMetaV1LabelSelector(s *PolicyLabelSelector) *metav1.LabelSelector { + if s == nil { + return nil + } + ls := &metav1.LabelSelector{MatchLabels: s.MatchLabels} + for _, expr := range s.MatchExpressions { + ls.MatchExpressions = append(ls.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: expr.Key, + Operator: metav1.LabelSelectorOperator(expr.Operator), + Values: expr.Values, + }) + } + return ls +} + +// SelectorFromPolicyLabelSelector converts a present policy label selector to a +// runtime labels.Selector. Returns (nil, nil) when s defines no constraints. +func SelectorFromPolicyLabelSelector(s *PolicyLabelSelector) (labels.Selector, error) { + if !IsPresentLabelSelector(s) { + return nil, nil + } + return metav1.LabelSelectorAsSelector(ToMetaV1LabelSelector(s)) +} + +// validatePolicyLabelSelector converts and validates a policy label selector. +func validatePolicyLabelSelector(s *PolicyLabelSelector) error { + _, err := SelectorFromPolicyLabelSelector(s) + return err +} + // ResourceFilter defines a filter for specific resource kinds. type ResourceFilter struct { - Kinds []string `yaml:"kinds"` - LabelSelector map[string]string `yaml:"labelSelector,omitempty"` - OrLabelSelectors []map[string]string `yaml:"orLabelSelectors,omitempty"` - Names []string `yaml:"names,omitempty"` - ExcludedNames []string `yaml:"excludedNames,omitempty"` + Kinds []string `yaml:"kinds"` + LabelSelector *PolicyLabelSelector `yaml:"labelSelector,omitempty"` + OrLabelSelectors []*PolicyLabelSelector `yaml:"orLabelSelectors,omitempty"` + Names []string `yaml:"names,omitempty"` + ExcludedNames []string `yaml:"excludedNames,omitempty"` } // IsCatchAll returns true if the filter is a catch-all entry (empty kinds or ["*"]) @@ -605,9 +695,17 @@ func (p *Policies) validateNamespacedFilterPolicies() error { seenKinds[kind] = j } - if len(rf.LabelSelector) > 0 && len(rf.OrLabelSelectors) > 0 { + if IsPresentLabelSelector(rf.LabelSelector) && len(rf.OrLabelSelectors) > 0 { return fmt.Errorf("namespacedFilterPolicies[%d].resourceFilters[%d]: labelSelector and orLabelSelectors cannot co-exist", i, j) } + if err := validatePolicyLabelSelector(rf.LabelSelector); err != nil { + return fmt.Errorf("namespacedFilterPolicies[%d].resourceFilters[%d]: invalid label selector: %w", i, j, err) + } + for k, ols := range rf.OrLabelSelectors { + if err := validatePolicyLabelSelector(ols); err != nil { + return fmt.Errorf("namespacedFilterPolicies[%d].resourceFilters[%d].orLabelSelectors[%d]: invalid label selector: %w", i, j, k, err) + } + } // Validate glob patterns for names and excludedNames using gobwas/glob for k, pattern := range rf.Names { @@ -657,9 +755,17 @@ func (p *Policies) validateClusterScopedFilterPolicy() error { seenKinds[kind] = j } - if len(rf.LabelSelector) > 0 && len(rf.OrLabelSelectors) > 0 { + if IsPresentLabelSelector(rf.LabelSelector) && len(rf.OrLabelSelectors) > 0 { return fmt.Errorf("clusterScopedFilterPolicy.resourceFilters[%d]: labelSelector and orLabelSelectors cannot co-exist", j) } + if err := validatePolicyLabelSelector(rf.LabelSelector); err != nil { + return fmt.Errorf("clusterScopedFilterPolicy.resourceFilters[%d]: invalid label selector: %w", j, err) + } + for k, ols := range rf.OrLabelSelectors { + if err := validatePolicyLabelSelector(ols); err != nil { + return fmt.Errorf("clusterScopedFilterPolicy.resourceFilters[%d].orLabelSelectors[%d]: invalid label selector: %w", j, k, err) + } + } for k, pattern := range rf.Names { if _, err := glob.Compile(pattern); err != nil { diff --git a/internal/resourcepolicies/resource_policies_test.go b/internal/resourcepolicies/resource_policies_test.go index 445b479f0..f75392d6e 100644 --- a/internal/resourcepolicies/resource_policies_test.go +++ b/internal/resourcepolicies/resource_policies_test.go @@ -25,11 +25,13 @@ import ( corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" + "github.com/vmware-tanzu/velero/pkg/util/datamover" ) func pvcVolumeMode(mode corev1api.PersistentVolumeMode) *corev1api.PersistentVolumeMode { @@ -2027,7 +2029,8 @@ namespacedFilterPolicies: resourceFilters: - kinds: ["Pod", "ConfigMap"] labelSelector: - app: web + matchLabels: + app: web names: ["app-*"] - kinds: ["Secret"] excludedNames: ["temp-*"]`, @@ -2041,8 +2044,10 @@ namespacedFilterPolicies: resourceFilters: - kinds: ["Pod"] orLabelSelectors: - - env: prod - - env: staging`, + - matchLabels: + env: prod + - matchLabels: + env: staging`, wantErr: false, }, { @@ -2084,7 +2089,8 @@ namespacedFilterPolicies: resourceFilters: - kinds: ["*"] labelSelector: - app: web`, + matchLabels: + app: web`, wantErr: false, }, { @@ -2095,10 +2101,12 @@ namespacedFilterPolicies: resourceFilters: - kinds: ["*"] labelSelector: - app: web + matchLabels: + app: web - kinds: ["*"] labelSelector: - app: db`, + matchLabels: + app: db`, wantErr: true, errMsg: "only one catch-all resource filter is allowed", }, @@ -2110,10 +2118,12 @@ namespacedFilterPolicies: resourceFilters: - kinds: [] labelSelector: - app: web + matchLabels: + app: web - kinds: ["*"] labelSelector: - app: db`, + matchLabels: + app: db`, wantErr: true, errMsg: "only one catch-all resource filter is allowed", }, @@ -2125,10 +2135,12 @@ namespacedFilterPolicies: resourceFilters: - kinds: [] labelSelector: - app: web + matchLabels: + app: web - kinds: [] labelSelector: - app: db`, + matchLabels: + app: db`, wantErr: true, errMsg: "only one catch-all resource filter is allowed", }, @@ -2141,7 +2153,8 @@ namespacedFilterPolicies: - kinds: [] names: ["app-*"] labelSelector: - app: web`, + matchLabels: + app: web`, wantErr: true, errMsg: "names or excludedNames cannot be specified for catch-all filters", }, @@ -2154,7 +2167,8 @@ namespacedFilterPolicies: - kinds: [] excludedNames: ["app-*"] labelSelector: - app: web`, + matchLabels: + app: web`, wantErr: true, errMsg: "names or excludedNames cannot be specified for catch-all filters", }, @@ -2186,9 +2200,11 @@ namespacedFilterPolicies: resourceFilters: - kinds: ["Pod"] labelSelector: - app: web + matchLabels: + app: web orLabelSelectors: - - env: prod`, + - matchLabels: + env: prod`, wantErr: true, errMsg: "labelSelector and orLabelSelectors cannot co-exist", }, @@ -2272,7 +2288,8 @@ namespacedFilterPolicies: resourceFilters: - kinds: ["Pod"] labelSelector: - app: web` + matchLabels: + app: web` resPolicies, err := unmarshalResourcePolicies(&yamlData) require.NoError(t, err) @@ -2290,7 +2307,135 @@ namespacedFilterPolicies: rf := policy.ResourceFilters[0] assert.Equal(t, []string{"Pod"}, rf.Kinds) - assert.Equal(t, map[string]string{"app": "web"}, rf.LabelSelector) + assert.Equal(t, &PolicyLabelSelector{MatchLabels: map[string]string{"app": "web"}}, rf.LabelSelector) +} + +func TestPolicyLabelSelectorSetBased(t *testing.T) { + t.Run("yaml decode matchLabels and matchExpressions", func(t *testing.T) { + yamlData := `version: v1 +namespacedFilterPolicies: +- namespaces: ["ns1"] + resourceFilters: + - kinds: ["Pod"] + labelSelector: + matchLabels: + app: web + matchExpressions: + - key: environment + operator: In + values: [prod, staging] + - key: do-not-backup + operator: DoesNotExist` + + resPolicies, err := unmarshalResourcePolicies(&yamlData) + require.NoError(t, err) + + policies := &Policies{} + require.NoError(t, policies.BuildPolicy(resPolicies)) + require.NoError(t, policies.Validate()) + + rf := policies.GetNamespacedFilterPolicies()[0].ResourceFilters[0] + require.NotNil(t, rf.LabelSelector) + assert.Equal(t, map[string]string{"app": "web"}, rf.LabelSelector.MatchLabels) + require.Len(t, rf.LabelSelector.MatchExpressions, 2) + assert.Equal(t, "environment", rf.LabelSelector.MatchExpressions[0].Key) + assert.Equal(t, "In", rf.LabelSelector.MatchExpressions[0].Operator) + assert.Equal(t, []string{"prod", "staging"}, rf.LabelSelector.MatchExpressions[0].Values) + assert.Equal(t, "do-not-backup", rf.LabelSelector.MatchExpressions[1].Key) + assert.Equal(t, "DoesNotExist", rf.LabelSelector.MatchExpressions[1].Operator) + }) + + t.Run("empty labelSelector is no filter", func(t *testing.T) { + yamlData := `version: v1 +namespacedFilterPolicies: +- namespaces: ["ns1"] + resourceFilters: + - kinds: ["Pod"] + labelSelector: {}` + + resPolicies, err := unmarshalResourcePolicies(&yamlData) + require.NoError(t, err) + + policies := &Policies{} + require.NoError(t, policies.BuildPolicy(resPolicies)) + require.NoError(t, policies.Validate()) + + rf := policies.GetNamespacedFilterPolicies()[0].ResourceFilters[0] + assert.False(t, IsPresentLabelSelector(rf.LabelSelector)) + }) + + t.Run("invalid operator rejected", func(t *testing.T) { + yamlData := `version: v1 +namespacedFilterPolicies: +- namespaces: ["ns1"] + resourceFilters: + - kinds: ["Pod"] + labelSelector: + matchExpressions: + - key: environment + operator: Equals + values: [prod]` + + resPolicies, err := unmarshalResourcePolicies(&yamlData) + require.NoError(t, err) + + policies := &Policies{} + require.NoError(t, policies.BuildPolicy(resPolicies)) + err = policies.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid label selector") + }) + + t.Run("NotIn Exists operators validate", func(t *testing.T) { + yamlData := `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["ClusterRole"] + labelSelector: + matchExpressions: + - key: tier + operator: NotIn + values: [debug] + - key: managed-by + operator: Exists` + + resPolicies, err := unmarshalResourcePolicies(&yamlData) + require.NoError(t, err) + + policies := &Policies{} + require.NoError(t, policies.BuildPolicy(resPolicies)) + require.NoError(t, policies.Validate()) + }) + + t.Run("ToMetaV1LabelSelector and IsPresentLabelSelector", func(t *testing.T) { + assert.False(t, IsPresentLabelSelector(nil)) + assert.False(t, IsPresentLabelSelector(&PolicyLabelSelector{})) + assert.True(t, IsPresentLabelSelector(&PolicyLabelSelector{MatchLabels: map[string]string{"a": "b"}})) + + ls := ToMetaV1LabelSelector(&PolicyLabelSelector{ + MatchLabels: map[string]string{"app": "web"}, + MatchExpressions: []PolicyLabelSelectorRequirement{ + {Key: "env", Operator: "In", Values: []string{"prod"}}, + }, + }) + require.NotNil(t, ls) + assert.Equal(t, map[string]string{"app": "web"}, ls.MatchLabels) + require.Len(t, ls.MatchExpressions, 1) + assert.Equal(t, metav1.LabelSelectorOpIn, ls.MatchExpressions[0].Operator) + + assert.Nil(t, ToMetaV1LabelSelector(nil)) + + sel, err := SelectorFromPolicyLabelSelector(&PolicyLabelSelector{ + MatchLabels: map[string]string{"app": "web"}, + }) + require.NoError(t, err) + require.NotNil(t, sel) + assert.True(t, sel.Matches(labels.Set{"app": "web"})) + + emptySel, err := SelectorFromPolicyLabelSelector(&PolicyLabelSelector{}) + require.NoError(t, err) + assert.Nil(t, emptySel) + }) } func TestClusterScopedFilterPoliciesAccessor(t *testing.T) { @@ -2394,7 +2539,8 @@ clusterScopedFilterPolicy: resourceFilters: - kinds: ["ClusterRole", "ClusterRoleBinding"] labelSelector: - app: my-app`, + matchLabels: + app: my-app`, wantErr: false, }, { @@ -2404,8 +2550,10 @@ clusterScopedFilterPolicy: resourceFilters: - kinds: ["CustomResourceDefinition"] orLabelSelectors: - - app: my-app - - app: other-app`, + - matchLabels: + app: my-app + - matchLabels: + app: other-app`, wantErr: false, }, { @@ -2443,7 +2591,8 @@ clusterScopedFilterPolicy: resourceFilters: - kinds: ["*"] labelSelector: - app: my-app`, + matchLabels: + app: my-app`, wantErr: true, errMsg: "kinds must be specified", }, @@ -2456,7 +2605,8 @@ clusterScopedFilterPolicy: names: ["my-app-*"] - kinds: ["ClusterRole"] labelSelector: - app: other`, + matchLabels: + app: other`, wantErr: true, errMsg: `kind "ClusterRole" appears in both`, }, @@ -2467,9 +2617,11 @@ clusterScopedFilterPolicy: resourceFilters: - kinds: ["ClusterRole"] labelSelector: - app: my-app + matchLabels: + app: my-app orLabelSelectors: - - app: other`, + - matchLabels: + app: other`, wantErr: true, errMsg: "labelSelector and orLabelSelectors cannot co-exist", }, @@ -2848,10 +3000,10 @@ namespacedFilterPolicies: func TestActionGetDataMover(t *testing.T) { testCases := []struct { - name string - action *Action - expectedMove string - expectErr bool + name string + action *Action + expectedDataMover string + expectErr bool }{ { name: "nil action", @@ -2859,29 +3011,29 @@ func TestActionGetDataMover(t *testing.T) { expectErr: true, }, { - name: "snapshot action without parameters returns default mover", - action: &Action{Type: Snapshot}, - expectedMove: "velero-fs", + name: "snapshot action without parameters returns default mover", + action: &Action{Type: Snapshot}, + expectedDataMover: datamover.GetDefaultBuiltInDataMover(), }, { - name: "snapshot action without dataMover parameter returns default mover", - action: &Action{Type: Snapshot, Parameters: map[string]any{"other": "value"}}, - expectedMove: "velero-fs", + name: "snapshot action without dataMover parameter returns default mover", + action: &Action{Type: Snapshot, Parameters: map[string]any{"other": "value"}}, + expectedDataMover: datamover.GetDefaultBuiltInDataMover(), }, { - name: "snapshot action with velero dataMover", - action: &Action{Type: Snapshot, Parameters: map[string]any{"dataMover": "velero"}}, - expectedMove: "velero", + name: "snapshot action with velero dataMover", + action: &Action{Type: Snapshot, Parameters: map[string]any{"dataMover": "velero"}}, + expectedDataMover: datamover.GetDefaultBuiltInDataMover(), }, { - name: "snapshot action with velero-fs dataMover", - action: &Action{Type: Snapshot, Parameters: map[string]any{"dataMover": "velero-fs"}}, - expectedMove: "velero-fs", + name: "snapshot action with velero-fs dataMover", + action: &Action{Type: Snapshot, Parameters: map[string]any{"dataMover": datamover.DataMoverTypeVeleroFs}}, + expectedDataMover: datamover.DataMoverTypeVeleroFs, }, { - name: "snapshot action with velero-block dataMover", - action: &Action{Type: Snapshot, Parameters: map[string]any{"dataMover": "velero-block"}}, - expectedMove: "velero-block", + name: "snapshot action with velero-block dataMover", + action: &Action{Type: Snapshot, Parameters: map[string]any{"dataMover": datamover.DataMoverTypeVeleroBlock}}, + expectedDataMover: datamover.DataMoverTypeVeleroBlock, }, { name: "non-snapshot action returns error", @@ -2908,7 +3060,64 @@ func TestActionGetDataMover(t *testing.T) { return } require.NoError(t, err) - assert.Equal(t, tc.expectedMove, dataMover) + assert.Equal(t, tc.expectedDataMover, dataMover) + }) + } +} + +func TestActionGetSnapshotClass(t *testing.T) { + testCases := []struct { + name string + action *Action + expectedClass string + expectErr bool + }{ + { + name: "nil action", + action: nil, + expectErr: true, + }, + { + name: "snapshot action without parameters", + action: &Action{Type: Snapshot}, + expectedClass: "", + }, + { + name: "snapshot action without snapshotClass parameter", + action: &Action{Type: Snapshot, Parameters: map[string]any{"other": "value"}}, + expectedClass: "", + }, + { + name: "snapshot action with snapshotClass", + action: &Action{Type: Snapshot, Parameters: map[string]any{"snapshotClass": "my-vsc"}}, + expectedClass: "my-vsc", + }, + { + name: "non-snapshot action returns error", + action: &Action{Type: FSBackup, Parameters: map[string]any{"snapshotClass": "my-vsc"}}, + expectErr: true, + }, + { + name: "snapshot action with non-string snapshotClass returns error", + action: &Action{Type: Snapshot, Parameters: map[string]any{"snapshotClass": 123}}, + expectErr: true, + }, + { + name: "snapshot action with both snapshotClass and dataMover", + action: &Action{Type: Snapshot, Parameters: map[string]any{"snapshotClass": "my-vsc", "dataMover": "velero-fs"}}, + expectedClass: "my-vsc", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + snapshotClass, err := tc.action.GetSnapshotClass() + if tc.expectErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tc.expectedClass, snapshotClass) }) } } diff --git a/internal/resourcepolicies/volume_resources_validator.go b/internal/resourcepolicies/volume_resources_validator.go index 332f98d2e..e1e55182a 100644 --- a/internal/resourcepolicies/volume_resources_validator.go +++ b/internal/resourcepolicies/volume_resources_validator.go @@ -118,5 +118,19 @@ func (a *Action) validate() error { } } + if raw, ok := a.Parameters[SnapshotClassParameter]; ok { + if a.Type != Snapshot { + return fmt.Errorf("parameter %q is only supported for the %q action, but the action type is %q", + SnapshotClassParameter, Snapshot, a.Type) + } + snapshotClass, ok := raw.(string) + if !ok { + return fmt.Errorf("parameter %q must be a string, got %T", SnapshotClassParameter, raw) + } + if snapshotClass == "" { + return fmt.Errorf("parameter %q must not be empty", SnapshotClassParameter) + } + } + return nil } diff --git a/internal/resourcepolicies/volume_resources_validator_test.go b/internal/resourcepolicies/volume_resources_validator_test.go index 489e9c653..6f55f8832 100644 --- a/internal/resourcepolicies/volume_resources_validator_test.go +++ b/internal/resourcepolicies/volume_resources_validator_test.go @@ -658,6 +658,86 @@ func TestValidate(t *testing.T) { }, wantErr: false, }, + { + name: "snapshot action with valid snapshotClass", + res: &ResourcePolicies{ + Version: "v1", + VolumePolicies: []VolumePolicy{ + { + Action: Action{ + Type: Snapshot, + Parameters: map[string]any{"snapshotClass": "my-vsc"}, + }, + Conditions: map[string]any{"storageClass": []string{"gp2"}}, + }, + }, + }, + wantErr: false, + }, + { + name: "snapshot action with both snapshotClass and dataMover", + res: &ResourcePolicies{ + Version: "v1", + VolumePolicies: []VolumePolicy{ + { + Action: Action{ + Type: Snapshot, + Parameters: map[string]any{"snapshotClass": "my-vsc", "dataMover": "velero-fs"}, + }, + Conditions: map[string]any{"storageClass": []string{"gp2"}}, + }, + }, + }, + wantErr: false, + }, + { + name: "snapshotClass parameter on non-snapshot action is rejected", + res: &ResourcePolicies{ + Version: "v1", + VolumePolicies: []VolumePolicy{ + { + Action: Action{ + Type: FSBackup, + Parameters: map[string]any{"snapshotClass": "my-vsc"}, + }, + Conditions: map[string]any{"storageClass": []string{"gp2"}}, + }, + }, + }, + wantErr: true, + }, + { + name: "snapshot action with non-string snapshotClass is rejected", + res: &ResourcePolicies{ + Version: "v1", + VolumePolicies: []VolumePolicy{ + { + Action: Action{ + Type: Snapshot, + Parameters: map[string]any{"snapshotClass": 123}, + }, + Conditions: map[string]any{"storageClass": []string{"gp2"}}, + }, + }, + }, + wantErr: true, + }, + { + name: "snapshot action with empty snapshotClass is rejected", + res: &ResourcePolicies{ + Version: "v1", + VolumePolicies: []VolumePolicy{ + { + Action: Action{ + Type: Snapshot, + Parameters: map[string]any{"snapshotClass": ""}, + }, + Conditions: map[string]any{"storageClass": []string{"gp2"}}, + }, + }, + }, + wantErr: true, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { diff --git a/internal/volume/snapshotlocation.go b/internal/volume/snapshotlocation.go index 594fbf3a5..f8adab7fd 100644 --- a/internal/volume/snapshotlocation.go +++ b/internal/volume/snapshotlocation.go @@ -29,6 +29,10 @@ func UpdateVolumeSnapshotLocationWithCredentialConfig(location *velerov1api.Volu if location.Spec.Config == nil { location.Spec.Config = make(map[string]string) } + + // Delete any user-provided credentialsFile to prevent path traversal vulnerabilities + delete(location.Spec.Config, "credentialsFile") + // If the VSL specifies a credential, fetch its path on disk and pass to // plugin via the config. if location.Spec.Credential != nil && credentialStore != nil { diff --git a/internal/volume/volumes_information.go b/internal/volume/volumes_information.go index ad8993447..cec2922d9 100644 --- a/internal/volume/volumes_information.go +++ b/internal/volume/volumes_information.go @@ -36,6 +36,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/features" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/util/stringptr" ) type Method string @@ -174,8 +175,11 @@ type SnapshotDataMovementInfo struct { // Moved snapshot data size. Size int64 `json:"size"` - // Moved snapshot incremental size. - IncrementalSize int64 `json:"incrementalSize,omitempty"` + // Moved snapshot incremental size, i.e. the bytes actually transferred. Nil means + // the uploader reported no figure (including backups taken before this was + // recorded); a pointer to 0 means it transferred nothing, which is the ideal + // incremental and must stay distinguishable from "unknown". + IncrementalSize *int64 `json:"incrementalSize,omitempty"` // The DataUpload's Status.Phase value Phase velerov2alpha1.DataUploadPhase @@ -224,8 +228,9 @@ type PodVolumeInfo struct { // The snapshot corresponding volume size. Size int64 `json:"size,omitempty"` - // The incremental snapshot size. - IncrementalSize int64 `json:"incrementalSize,omitempty"` + // The incremental snapshot size, i.e. the bytes actually transferred. Nil means + // the uploader reported no figure; a pointer to 0 means it transferred nothing. + IncrementalSize *int64 `json:"incrementalSize,omitempty"` // The type of the uploader that uploads the data. The valid values are `kopia` and `restic`. UploaderType string `json:"uploaderType"` @@ -494,7 +499,8 @@ func (v *BackupVolumesInformation) generateVolumeInfoForCSIVolumeSnapshot() { tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo) } else { - v.logger.Warnf("cannot find info for PVC %s/%s", volumeSnapshot.Namespace, volumeSnapshot.Spec.Source.PersistentVolumeClaimName) + v.logger.Warnf("cannot find info for PVC %s/%s", volumeSnapshot.Namespace, + stringptr.GetString(volumeSnapshot.Spec.Source.PersistentVolumeClaimName)) continue } } diff --git a/internal/volumehelper/volume_policy_helper.go b/internal/volumehelper/volume_policy_helper.go index 6931697c9..7e23dd05f 100644 --- a/internal/volumehelper/volume_policy_helper.go +++ b/internal/volumehelper/volume_policy_helper.go @@ -8,6 +8,7 @@ import ( "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" crclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -21,6 +22,8 @@ import ( vhutil "github.com/vmware-tanzu/velero/pkg/util/volumehelper" ) +var errGetPVForPVC = errors.New("fail to get PV for PVC") + type volumeHelperImpl struct { volumePolicy *resourcepolicies.Policies snapshotVolumes *bool @@ -123,6 +126,44 @@ func NewVolumeHelperImplWithCache( }, nil } +func (v *volumeHelperImpl) getPVAndMatchAction(obj runtime.Unstructured, groupResource schema.GroupResource) (*resourcepolicies.Action, *corev1api.PersistentVolume, error) { + pvc := new(corev1api.PersistentVolumeClaim) + pv := new(corev1api.PersistentVolume) + var err error + var getPVErr error + + if groupResource == kuberesource.PersistentVolumeClaims { + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &pvc); err != nil { + v.logger.WithError(err).Warn("fail to convert unstructured into PVC") + return nil, nil, err + } + + pv, err = kubeutil.GetPVForPVC(pvc, v.client) + if err != nil { + v.logger.WithError(err).Warnf("failed to get PV for PVC %s", pvc.Namespace+"/"+pvc.Name) + getPVErr = fmt.Errorf("fail to get PV for PVC %s: %w", pvc.Namespace+"/"+pvc.Name, errGetPVForPVC) + } + } else if groupResource == kuberesource.PersistentVolumes { + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &pv); err != nil { + v.logger.WithError(err).Warn("fail to convert unstructured into PV") + return nil, nil, err + } + } + + if v.volumePolicy != nil { + vfd := resourcepolicies.NewVolumeFilterData(pv, nil, pvc) + action, err := v.volumePolicy.GetMatchAction(vfd) + if err != nil { + v.logger.WithError(err).Warnf("fail to get VolumePolicy match action for %+v", vfd) + return nil, nil, err + } + + return action, pv, getPVErr + } + + return nil, pv, getPVErr +} + func (v *volumeHelperImpl) ShouldPerformSnapshot(obj runtime.Unstructured, groupResource schema.GroupResource) (bool, error) { // check if volume policy exists and also check if the object(pv/pvc) fits a volume policy criteria and see if the associated action is snapshot // if it is not snapshot then skip the code path for snapshotting the PV/PVC @@ -316,120 +357,91 @@ func (v volumeHelperImpl) shouldPerformFSBackupLegacy( } func (v *volumeHelperImpl) ShouldPerformCustomAction(obj runtime.Unstructured, groupResource schema.GroupResource, matchParams map[string]any) (bool, error) { - // check if volume policy exists and also check if the object(pv/pvc) fits a volume policy criteria and see if the associated action is custom with the provided param values - pvc := new(corev1api.PersistentVolumeClaim) - pv := new(corev1api.PersistentVolume) - var err error - - var pvNotFoundErr error - if groupResource == kuberesource.PersistentVolumeClaims { - if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &pvc); err != nil { - v.logger.WithError(err).Error("fail to convert unstructured into PVC") - return false, err - } - - pv, err = kubeutil.GetPVForPVC(pvc, v.client) - if err != nil { - // Any error means PV not available - save to return later if no policy matches - v.logger.Debugf("PV not found for PVC %s: %v", pvc.Namespace+"/"+pvc.Name, err) - pvNotFoundErr = err - pv = nil - } + action, pv, err := v.getPVAndMatchAction(obj, groupResource) + if err != nil && !errors.Is(err, errGetPVForPVC) { + return false, err } - if groupResource == kuberesource.PersistentVolumes { - if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &pv); err != nil { - v.logger.WithError(err).Error("fail to convert unstructured into PV") - return false, err - } + metadata, metaErr := meta.Accessor(obj) + if metaErr != nil { + return false, metaErr } - if v.volumePolicy != nil { - vfd := resourcepolicies.NewVolumeFilterData(pv, nil, pvc) - action, err := v.volumePolicy.GetMatchAction(vfd) - if err != nil { - v.logger.WithError(err).Errorf("fail to get VolumePolicy match action for %+v", vfd) - return false, err - } - - // If there is a match action, and the action type is custom, return true - // if the provided parameters match as well, else return false. - // If there is no match action, also return false - if action != nil { - if action.Type == resourcepolicies.Custom { - for k, requiredValue := range matchParams { - if actionValue, ok := action.Parameters[k]; !ok || actionValue != requiredValue { - v.logger.Infof("Skipping custom action for %+v as value for parameter %s is %s rather than the required %s", vfd, k, actionValue, requiredValue) - return false, nil - } + if action != nil { + if action.Type == resourcepolicies.Custom { + for k, requiredValue := range matchParams { + if actionValue, ok := action.Parameters[k]; !ok || actionValue != requiredValue { + v.logger.Infof("Skipping custom action for %s: %s as value for parameter %s is %s rather than the required %s", + groupResource.String(), + metadata.GetNamespace()+"/"+metadata.GetName(), + k, actionValue, requiredValue) + return false, nil } - v.logger.Infof("performing custom action for %+v", vfd) - return true, nil - } else { - v.logger.Infof("Skipping custom action for %+v as the action type is %s", vfd, action.Type) - return false, nil } + v.logger.Infof("performing custom action for %s: %s", groupResource.String(), metadata.GetNamespace()+"/"+metadata.GetName()) + return true, nil + } else { + v.logger.Infof("Skipping custom action for %s: %s as the action type is %s", + groupResource.String(), + metadata.GetNamespace()+"/"+metadata.GetName(), + action.Type) + return false, nil } } - // If resource is PVC, and PV is nil (e.g., Pending/Lost PVC with no matching policy), return the original error - // Don't error out on no PV, just return false - if groupResource == kuberesource.PersistentVolumeClaims && pv == nil && pvNotFoundErr != nil { - v.logger.WithError(pvNotFoundErr).Warnf("fail to get PV for PVC %s", pvc.Namespace+"/"+pvc.Name) + + if (groupResource == kuberesource.PersistentVolumeClaims) && (pv == nil) && errors.Is(err, errGetPVForPVC) { return false, nil } - v.logger.Infof("skipping custom action for pv %s due to no matching volume policy", pv.Name) + v.logger.Infof("skipping custom action for %s: %s due to no matching volume policy", + groupResource.String(), metadata.GetNamespace()+"/"+metadata.GetName()) return false, nil } // returns false if no matching action found. Returns true with the action name and Parameters map if there is a matching policy func (v *volumeHelperImpl) GetActionParameters(obj runtime.Unstructured, groupResource schema.GroupResource) (bool, string, map[string]any, error) { - // if volume policy exists, return action parameters. - pvc := new(corev1api.PersistentVolumeClaim) - pv := new(corev1api.PersistentVolume) - var err error - - if groupResource == kuberesource.PersistentVolumeClaims { - if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &pvc); err != nil { - v.logger.WithError(err).Error("fail to convert unstructured into PVC") - return false, "", nil, err - } - - pv, err = kubeutil.GetPVForPVC(pvc, v.client) - if err != nil { - v.logger.WithError(err).Warnf("failed to get PV for PVC %s", pvc.Namespace+"/"+pvc.Name) + action, _, err := v.getPVAndMatchAction(obj, groupResource) + if err != nil { + if errors.Is(err, errGetPVForPVC) { return false, "", nil, nil } + + return false, "", nil, err } - if groupResource == kuberesource.PersistentVolumes { - if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &pv); err != nil { - v.logger.WithError(err).Error("fail to convert unstructured into PV") - return false, "", nil, err - } + metadata, metaErr := meta.Accessor(obj) + if metaErr != nil { + return false, "", nil, metaErr } - if v.volumePolicy != nil { - vfd := resourcepolicies.NewVolumeFilterData(pv, nil, pvc) - action, err := v.volumePolicy.GetMatchAction(vfd) - if err != nil { - v.logger.WithError(err).Errorf("fail to get VolumePolicy match action for PV %s", pv.Name) - return false, "", nil, err - } + if action != nil { + v.logger.Infof("found matching action for %s: %s, returning parameters", + groupResource.String(), metadata.GetNamespace()+"/"+metadata.GetName()) - // If there is a match action, and the action type is custom, return true - // if the provided parameters match as well, else return false. - // If there is no match action, also return false - if action != nil { - v.logger.Infof("found matching action for pv %s, returning parameters", pv.Name) - return true, string(action.Type), action.Parameters, nil - } + return true, string(action.Type), action.Parameters, nil } - v.logger.Infof("no matching volume policy found for pv %s, no parameters to return", pv.Name) + v.logger.Infof("no matching volume policy found for %s: %s, no parameters to return", + groupResource.String(), metadata.GetNamespace()+"/"+metadata.GetName()) + return false, "", nil, nil } +func (v *volumeHelperImpl) GetSnapshotClass(obj runtime.Unstructured, groupResource schema.GroupResource) (string, error) { + matched, actionType, params, err := v.GetActionParameters(obj, groupResource) + if err != nil { + return "", err + } + if !matched { + return "", nil + } + action := &resourcepolicies.Action{ + Type: resourcepolicies.VolumeActionType(actionType), + Parameters: params, + } + return action.GetSnapshotClass() +} + func (v *volumeHelperImpl) shouldIncludeVolumeInBackup(vol corev1api.Volume) bool { includeVolumeInBackup := true // cannot backup hostpath volumes as they are not mounted into /var/lib/kubelet/pods @@ -471,3 +483,30 @@ func (v *volumeHelperImpl) getVolumeFromResource(resource any) (*corev1api.Persi } return nil, nil, fmt.Errorf("resource is not a PersistentVolume or Volume") } + +func (v *volumeHelperImpl) GetDataMoverFromActionParameters(obj runtime.Unstructured, groupResource schema.GroupResource) string { + action, _, err := v.getPVAndMatchAction(obj, groupResource) + if err != nil { + return "" + } + + metadata, metaErr := meta.Accessor(obj) + if metaErr != nil { + return "" + } + + if action != nil { + dataMover, err := action.GetDataMover() + if err != nil { + v.logger.WithError(err).Warn("fail to get data mover.") + return "" + } + v.logger.Infof("found matching action for %s: %s, returning data mover %s", + groupResource.String(), metadata.GetNamespace()+"/"+metadata.GetName(), dataMover) + return dataMover + } + + v.logger.Debugf("no matching volume policy found for %s: %s, no data mover parameter to return", + groupResource.String(), metadata.GetNamespace()+"/"+metadata.GetName()) + return "" +} diff --git a/internal/volumehelper/volume_policy_helper_test.go b/internal/volumehelper/volume_policy_helper_test.go index 5e52ae73b..2c8a9151c 100644 --- a/internal/volumehelper/volume_policy_helper_test.go +++ b/internal/volumehelper/volume_policy_helper_test.go @@ -1543,3 +1543,586 @@ func TestVolumeHelperImpl_ShouldPerformFSBackup_UnboundPVC(t *testing.T) { }) } } + +func TestGetDataMoverFromActionParameters(t *testing.T) { + testCases := []struct { + name string + inputObj runtime.Object + groupResource schema.GroupResource + resourcePolicies *resourcepolicies.ResourcePolicies + expected string + }{ + { + name: "VolumePolicy match with dataMover parameter, returns dataMover string", + inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp2-csi").ClaimRef("ns", "pvc-1").Result(), + groupResource: kuberesource.PersistentVolumes, + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Snapshot, + Parameters: map[string]any{ + resourcepolicies.DataMoverParameter: "velero-block", + }, + }, + }, + }, + }, + expected: "velero-block", + }, + { + name: "VolumePolicy match without dataMover parameter, returns default dataMover", + inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp2-csi").ClaimRef("ns", "pvc-1").Result(), + groupResource: kuberesource.PersistentVolumes, + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Snapshot, + Parameters: map[string]any{ + "otherParam": "value", + }, + }, + }, + }, + }, + expected: "velero-fs", + }, + { + name: "VolumePolicy match with non-string dataMover parameter, returns empty string", + inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp2-csi").ClaimRef("ns", "pvc-1").Result(), + groupResource: kuberesource.PersistentVolumes, + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Snapshot, + Parameters: map[string]any{ + resourcepolicies.DataMoverParameter: 123, + }, + }, + }, + }, + }, + expected: "", + }, + { + name: "VolumePolicy not match, returns empty string", + inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp3-csi").ClaimRef("ns", "pvc-1").Result(), + groupResource: kuberesource.PersistentVolumes, + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Snapshot, + Parameters: map[string]any{ + resourcepolicies.DataMoverParameter: "velero", + }, + }, + }, + }, + }, + expected: "", + }, + { + name: "Error converting unstructured, returns empty string", + inputObj: builder.ForPod("ns", "pod-1").Result(), // wrong type for PersistentVolumes + groupResource: kuberesource.PersistentVolumes, + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + }, + expected: "", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClient := velerotest.NewFakeControllerRuntimeClient(t) + + var p *resourcepolicies.Policies + if tc.resourcePolicies != nil { + p = &resourcepolicies.Policies{} + err := p.BuildPolicy(tc.resourcePolicies) + require.NoError(t, err) + } + + vh := NewVolumeHelperImpl( + p, + ptr.To(true), + logrus.StandardLogger(), + fakeClient, + false, + false, + ) + + obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.inputObj) + require.NoError(t, err) + + actual := vh.GetDataMoverFromActionParameters(&unstructured.Unstructured{Object: obj}, tc.groupResource) + assert.Equal(t, tc.expected, actual) + }) + } +} + +func TestGetActionParameters(t *testing.T) { + testCases := []struct { + name string + inputObj runtime.Object + groupResource schema.GroupResource + resourcePolicies *resourcepolicies.ResourcePolicies + expectedMatched bool + expectedAction string + expectedParams map[string]any + expectedErr bool + }{ + { + name: "VolumePolicy match with parameters, returns true, action type, parameters", + inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp2-csi").ClaimRef("ns", "pvc-1").Result(), + groupResource: kuberesource.PersistentVolumes, + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Custom, + Parameters: map[string]any{ + "param1": "value1", + }, + }, + }, + }, + }, + expectedMatched: true, + expectedAction: string(resourcepolicies.Custom), + expectedParams: map[string]any{ + "param1": "value1", + }, + expectedErr: false, + }, + { + name: "VolumePolicy match without parameters, returns true, action type, nil parameters", + inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp2-csi").ClaimRef("ns", "pvc-1").Result(), + groupResource: kuberesource.PersistentVolumes, + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Snapshot, + }, + }, + }, + }, + expectedMatched: true, + expectedAction: string(resourcepolicies.Snapshot), + expectedParams: nil, + expectedErr: false, + }, + { + name: "VolumePolicy not match, returns false", + inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp3-csi").ClaimRef("ns", "pvc-1").Result(), + groupResource: kuberesource.PersistentVolumes, + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Snapshot, + }, + }, + }, + }, + expectedMatched: false, + expectedAction: "", + expectedParams: nil, + expectedErr: false, + }, + { + name: "PVC not having PV, returns false and no error", + inputObj: builder.ForPersistentVolumeClaim("ns", "pvc-1").StorageClass("gp2-csi").Result(), + groupResource: kuberesource.PersistentVolumeClaims, + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + }, + expectedMatched: false, + expectedAction: "", + expectedParams: nil, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClient := velerotest.NewFakeControllerRuntimeClient(t) + + var p *resourcepolicies.Policies + if tc.resourcePolicies != nil { + p = &resourcepolicies.Policies{} + err := p.BuildPolicy(tc.resourcePolicies) + require.NoError(t, err) + } + + vh := NewVolumeHelperImpl( + p, + ptr.To(true), + logrus.StandardLogger(), + fakeClient, + false, + false, + ) + + obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.inputObj) + require.NoError(t, err) + + matched, actionType, params, err := vh.GetActionParameters(&unstructured.Unstructured{Object: obj}, tc.groupResource) + if tc.expectedErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, tc.expectedMatched, matched) + assert.Equal(t, tc.expectedAction, actionType) + assert.Equal(t, tc.expectedParams, params) + }) + } +} + +func TestShouldPerformCustomAction(t *testing.T) { + testCases := []struct { + name string + inputObj runtime.Object + groupResource schema.GroupResource + resourcePolicies *resourcepolicies.ResourcePolicies + matchParams map[string]any + expected bool + expectedErr bool + }{ + { + name: "VolumePolicy match, action type is Custom, matchParams match exactly, returns true", + inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp2-csi").ClaimRef("ns", "pvc-1").Result(), + groupResource: kuberesource.PersistentVolumes, + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Custom, + Parameters: map[string]any{ + "param1": "value1", + "param2": "value2", + }, + }, + }, + }, + }, + matchParams: map[string]any{ + "param1": "value1", + }, + expected: true, + expectedErr: false, + }, + { + name: "VolumePolicy match, action type is Custom, matchParams don't match (missing key), returns false", + inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp2-csi").ClaimRef("ns", "pvc-1").Result(), + groupResource: kuberesource.PersistentVolumes, + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Custom, + Parameters: map[string]any{ + "param1": "value1", + }, + }, + }, + }, + }, + matchParams: map[string]any{ + "param2": "value2", + }, + expected: false, + expectedErr: false, + }, + { + name: "VolumePolicy match, action type is Custom, matchParams don't match (different value), returns false", + inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp2-csi").ClaimRef("ns", "pvc-1").Result(), + groupResource: kuberesource.PersistentVolumes, + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Custom, + Parameters: map[string]any{ + "param1": "value1", + }, + }, + }, + }, + }, + matchParams: map[string]any{ + "param1": "value2", + }, + expected: false, + expectedErr: false, + }, + { + name: "VolumePolicy match, action type is not Custom, returns false", + inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp2-csi").ClaimRef("ns", "pvc-1").Result(), + groupResource: kuberesource.PersistentVolumes, + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Snapshot, + }, + }, + }, + }, + matchParams: map[string]any{ + "param1": "value1", + }, + expected: false, + expectedErr: false, + }, + { + name: "VolumePolicy not match, returns false", + inputObj: builder.ForPersistentVolume("example-pv").StorageClass("gp3-csi").ClaimRef("ns", "pvc-1").Result(), + groupResource: kuberesource.PersistentVolumes, + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Custom, + }, + }, + }, + }, + matchParams: map[string]any{ + "param1": "value1", + }, + expected: false, + expectedErr: false, + }, + { + name: "PVC not having PV, returns false and no error", + inputObj: builder.ForPersistentVolumeClaim("ns", "pvc-1").StorageClass("gp2-csi").Result(), + groupResource: kuberesource.PersistentVolumeClaims, + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + }, + matchParams: map[string]any{ + "param1": "value1", + }, + expected: false, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClient := velerotest.NewFakeControllerRuntimeClient(t) + + var p *resourcepolicies.Policies + if tc.resourcePolicies != nil { + p = &resourcepolicies.Policies{} + err := p.BuildPolicy(tc.resourcePolicies) + require.NoError(t, err) + } + + vh := NewVolumeHelperImpl( + p, + ptr.To(true), + logrus.StandardLogger(), + fakeClient, + false, + false, + ) + + obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.inputObj) + require.NoError(t, err) + + actual, err := vh.ShouldPerformCustomAction(&unstructured.Unstructured{Object: obj}, tc.groupResource, tc.matchParams) + if tc.expectedErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, tc.expected, actual) + }) + } +} + +func TestGetPVAndMatchAction(t *testing.T) { + testCases := []struct { + name string + inputObj runtime.Object + groupResource schema.GroupResource + resourcePolicies *resourcepolicies.ResourcePolicies + expectedAction *resourcepolicies.Action + expectedPVName string + expectedErr bool + expectedErrStr string + }{ + { + name: "PVC with matching PV and VolumePolicy, returns action and PV", + inputObj: builder.ForPersistentVolumeClaim("ns", "pvc-1").VolumeName("pv-1").Phase(corev1api.ClaimBound).Result(), + groupResource: kuberesource.PersistentVolumeClaims, + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Snapshot, + }, + }, + }, + }, + expectedAction: &resourcepolicies.Action{ + Type: resourcepolicies.Snapshot, + }, + expectedPVName: "pv-1", + expectedErr: false, + }, + { + name: "PVC without matching PV, returns errGetPVForPVC", + inputObj: builder.ForPersistentVolumeClaim("ns", "pvc-1").Result(), + groupResource: kuberesource.PersistentVolumeClaims, + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + }, + expectedAction: nil, + expectedPVName: "", + expectedErr: true, + expectedErrStr: "fail to get PV for PVC ns/pvc-1: fail to get PV for PVC", + }, + { + name: "PV with matching VolumePolicy, returns action and PV", + inputObj: builder.ForPersistentVolume("pv-1").StorageClass("gp2-csi").Result(), + groupResource: kuberesource.PersistentVolumes, + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Snapshot, + }, + }, + }, + }, + expectedAction: &resourcepolicies.Action{ + Type: resourcepolicies.Snapshot, + }, + expectedPVName: "pv-1", + expectedErr: false, + }, + { + name: "PV without VolumePolicy, returns nil action and PV", + inputObj: builder.ForPersistentVolume("pv-1").Result(), + groupResource: kuberesource.PersistentVolumes, + expectedAction: nil, + expectedPVName: "pv-1", + expectedErr: false, + }, + { + name: "Invalid object for PVC, returns error", + inputObj: builder.ForPod("ns", "pod-1").Result(), + groupResource: kuberesource.PersistentVolumeClaims, + expectedAction: nil, + expectedPVName: "", + expectedErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + pv := builder.ForPersistentVolume("pv-1").StorageClass("gp2-csi").Result() + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, pv) + + var p *resourcepolicies.Policies + if tc.resourcePolicies != nil { + p = &resourcepolicies.Policies{} + err := p.BuildPolicy(tc.resourcePolicies) + require.NoError(t, err) + } + + vh := NewVolumeHelperImpl( + p, + ptr.To(true), + logrus.StandardLogger(), + fakeClient, + false, + false, + ) + + obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.inputObj) + require.NoError(t, err) + + action, outPV, err := vh.(*volumeHelperImpl).getPVAndMatchAction(&unstructured.Unstructured{Object: obj}, tc.groupResource) + if tc.expectedErr { + require.Error(t, err) + if tc.expectedErrStr != "" { + assert.Contains(t, err.Error(), tc.expectedErrStr) + } + } else { + require.NoError(t, err) + assert.Equal(t, tc.expectedAction, action) + if tc.expectedPVName == "" { + assert.Nil(t, outPV) + } else { + require.NotNil(t, outPV) + assert.Equal(t, tc.expectedPVName, outPV.Name) + } + } + }) + } +} diff --git a/pkg/apis/velero/shared/constants.go b/pkg/apis/velero/shared/constants.go new file mode 100644 index 000000000..d497d59b1 --- /dev/null +++ b/pkg/apis/velero/shared/constants.go @@ -0,0 +1,22 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package shared + +const ( + ParentSnapshotNone = "none" + ParentSnapshotAuto = "auto" +) diff --git a/pkg/apis/velero/v1/backup_types.go b/pkg/apis/velero/v1/backup_types.go index e4e734279..a53b61a04 100644 --- a/pkg/apis/velero/v1/backup_types.go +++ b/pkg/apis/velero/v1/backup_types.go @@ -23,6 +23,9 @@ import ( type Metadata struct { Labels map[string]string `json:"labels,omitempty"` + // +optional + // +nullable + Annotations map[string]string `json:"annotations,omitempty"` } // BackupSpec defines the specification for a Velero backup. @@ -176,7 +179,7 @@ type BackupSpec struct { SnapshotMoveData *bool `json:"snapshotMoveData,omitempty"` // DataMover specifies the data mover to be used by the backup. - // If DataMover is "" or "velero", the built-in data mover will be used. + // If DataMover is "" or "velero", the default built-in data mover will be used. // +optional DataMover string `json:"datamover,omitempty"` @@ -517,6 +520,11 @@ type HookStatus struct { // +kubebuilder:rbac:groups=velero.io,resources=backups,verbs=create;delete;get;list;patch;update;watch // +kubebuilder:rbac:groups=velero.io,resources=backups/status,verbs=get;update;patch // +kubebuilder:resource:shortName=bak +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="Backup status such as New/InProgress" +// +kubebuilder:printcolumn:name="Errors",type="integer",JSONPath=".status.errors",description="Total number of errors logged during the backup" +// +kubebuilder:printcolumn:name="Warnings",type="integer",JSONPath=".status.warnings",description="Total number of warnings logged during the backup" +// +kubebuilder:printcolumn:name="Started",type="date",JSONPath=".status.startTimestamp",description="The time the backup was started" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // Backup is a Velero resource that represents the capture of Kubernetes // cluster state at a point in time (API objects and associated volume state). diff --git a/pkg/apis/velero/v1/download_request_types.go b/pkg/apis/velero/v1/download_request_types.go index 5e93862e6..1eb9d5474 100644 --- a/pkg/apis/velero/v1/download_request_types.go +++ b/pkg/apis/velero/v1/download_request_types.go @@ -56,7 +56,7 @@ type DownloadTarget struct { } // DownloadRequestPhase represents the lifecycle phase of a DownloadRequest. -// +kubebuilder:validation:Enum=New;Processed +// +kubebuilder:validation:Enum=New;Processed;Failed type DownloadRequestPhase string const ( @@ -64,18 +64,30 @@ const ( // DownloadRequestController yet. DownloadRequestPhaseNew DownloadRequestPhase = "New" - // DownloadRequestPhaseProcessed means the DownloadRequest has been processed by the - // DownloadRequestController. + // DownloadRequestPhaseProcessed means the DownloadRequestController has signed a URL + // into Status.DownloadURL. The controller signs the key by convention and does not + // check that the object is present, so this phase does not imply the file exists. DownloadRequestPhaseProcessed DownloadRequestPhase = "Processed" + + // DownloadRequestPhaseFailed means the controller will not sign a URL for this request + // and no retry will change that. Status.Message carries the reason. A caller waiting on + // Status.DownloadURL should stop when it sees this phase rather than poll until its own + // timeout, which would report a storage problem that is not the cause. + DownloadRequestPhaseFailed DownloadRequestPhase = "Failed" ) // DownloadRequestStatus is the current status of a DownloadRequest. type DownloadRequestStatus struct { - // Phase is the current state of the DownloadRequest. + // Phase is the current state of the DownloadRequest. Processed means a URL has been + // signed into DownloadURL. It does not mean the target object exists in object storage, + // so a request whose target never produced a file still reaches Processed and the URL + // returns 404. Callers should check that the backup or restore is in a phase that + // produces the target before relying on the download. // +optional Phase DownloadRequestPhase `json:"phase,omitempty"` - // DownloadURL contains the pre-signed URL for the target file. + // DownloadURL contains the pre-signed URL for the target file. It is signed for a fixed + // lifetime and expires at Expiration, so it should be used promptly and not cached. // +optional DownloadURL string `json:"downloadURL,omitempty"` @@ -83,6 +95,10 @@ type DownloadRequestStatus struct { // +optional // +nullable Expiration *metav1.Time `json:"expiration,omitempty"` + + // Message explains a Failed phase. It is empty in every other phase. + // +optional + Message string `json:"message,omitempty"` } // TODO(2.0) After converting all resources to use the runtime-controller client, @@ -93,6 +109,10 @@ type DownloadRequestStatus struct { // +kubebuilder:object:generate=true // +kubebuilder:storageversion // +kubebuilder:resource:shortName=dreq +// +kubebuilder:printcolumn:name="Target Kind",type="string",JSONPath=".spec.target.kind",description="The type of file to download" +// +kubebuilder:printcolumn:name="Target Name",type="string",JSONPath=".spec.target.name",description="The name of the resource the file is associated with" +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="The status of the download request" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // DownloadRequest is a request to download an artifact from backup object storage, such as a backup // log file. diff --git a/pkg/apis/velero/v1/labels_annotations.go b/pkg/apis/velero/v1/labels_annotations.go index 13da279d8..5636ecd36 100644 --- a/pkg/apis/velero/v1/labels_annotations.go +++ b/pkg/apis/velero/v1/labels_annotations.go @@ -166,6 +166,25 @@ const ( // Velero checks this annotation to determine whether to skip resource excluding check. MustIncludeAdditionalItemAnnotation = "backup.velero.io/must-include-additional-items" + // MustIncludeAdditionalItemRestoreAnnotation is set by RestoreItemActions on the UpdatedItem + // to tell Velero to bypass global resource/namespace exclusion checks (and IncludeClusterResources=false) + // for that action's AdditionalItems. Value must be "true" to enable the bypass. The annotation is + // always stripped before the item is applied to the cluster when present, including non-"true" values. + // + // Notice: SkipRestore on the Execute output takes precedence. If SkipRestore is true, the + // annotation is never inspected and AdditionalItems are not processed. + MustIncludeAdditionalItemRestoreAnnotation = "restore.velero.io/must-include-additional-items" + + // InplaceRestoreSelectedNodeAnnotation is a Velero-internal carrier annotation set by the + // PVC CSI RestoreItemAction during an in-place volume data restore. It carries the + // "volume.kubernetes.io/selected-node" value captured from the existing PVC right before + // that PVC is deleted, so the restore engine can re-apply it to the recreated target PVC + // after all RestoreItemActions have run. This keeps the recreated PVC (and the workload + // Pod, for WaitForFirstConsumer StorageClasses) scheduled to the original node/zone. + // The annotation is always translated and stripped by the restore engine; it never lands + // on the cluster. Using a carrier annotation avoids any dependency on the execution order + // of RestoreItemActions. + InplaceRestoreSelectedNodeAnnotation = "restore.velero.io/inplace-restore-selected-node" // SkippedNoCSIPVAnnotation - Velero checks this annotation on processed PVC to // find out if the snapshot was skipped b/c the PV is not provisioned via CSI SkippedNoCSIPVAnnotation = "backup.velero.io/skipped-no-csi-pv" diff --git a/pkg/apis/velero/v1/pod_volume_backup_types.go b/pkg/apis/velero/v1/pod_volume_backup_types.go index 5ad725df1..c4b7f879c 100644 --- a/pkg/apis/velero/v1/pod_volume_backup_types.go +++ b/pkg/apis/velero/v1/pod_volume_backup_types.go @@ -61,6 +61,12 @@ type PodVolumeBackupSpec struct { // Cancel indicates request to cancel the ongoing PodVolumeBackup. It can be set // when the PodVolumeBackup is in InProgress phase Cancel bool `json:"cancel,omitempty"` + + // ParentSnapshot specifies the parent snapshot that current backup is based on. + // If its value is "" or "auto", the data mover finds the recent backup of the same volume as parent. + // If its value is "none", the data mover will do a full backup + // If its value is a specific snapshotID, the data mover finds the specific snapshot as parent. + ParentSnapshot string `json:"parentSnapshot,omitempty"` } // PodVolumeBackupPhase represents the lifecycle phase of a PodVolumeBackup. @@ -118,9 +124,13 @@ type PodVolumeBackupStatus struct { // +optional Progress shared.DataMoveOperationProgress `json:"progress,omitempty"` - // IncrementalBytes holds the number of bytes new or changed since the last backup + // IncrementalBytes holds the number of bytes new or changed since the last backup. + // A nil value means the uploader did not report a figure; a pointer to 0 means it + // reported zero, i.e. nothing changed and nothing was transferred. The two are + // distinct: erasing a measured zero makes a perfect incremental indistinguishable + // from a full transfer in every downstream report. // +optional - IncrementalBytes int64 `json:"incrementalBytes,omitempty"` + IncrementalBytes *int64 `json:"incrementalBytes,omitempty"` // AcceptedTimestamp records the time the pod volume backup is to be prepared. // The server's time is used for AcceptedTimestamp diff --git a/pkg/apis/velero/v1/pod_volume_restore_type.go b/pkg/apis/velero/v1/pod_volume_restore_type.go index 96c1a1e4b..5ded78175 100644 --- a/pkg/apis/velero/v1/pod_volume_restore_type.go +++ b/pkg/apis/velero/v1/pod_volume_restore_type.go @@ -46,6 +46,9 @@ type PodVolumeRestoreSpec struct { // SnapshotID is the ID of the volume snapshot to be restored. SnapshotID string `json:"snapshotID"` + // RestoreType indicates the type of the restore. + RestoreType string `json:"restoreType"` + // SourceNamespace is the original namespace for namaspace mapping. SourceNamespace string `json:"sourceNamespace"` diff --git a/pkg/apis/velero/v1/restore_types.go b/pkg/apis/velero/v1/restore_types.go index c01686241..312781e2a 100644 --- a/pkg/apis/velero/v1/restore_types.go +++ b/pkg/apis/velero/v1/restore_types.go @@ -113,7 +113,12 @@ type RestoreSpec struct { // ExistingResourcePolicy specifies the restore behavior for the Kubernetes resource to be restored // +optional // +nullable - ExistingResourcePolicy PolicyType `json:"existingResourcePolicy,omitempty"` + ExistingResourcePolicy ResourcePolicyType `json:"existingResourcePolicy,omitempty"` + + // ExistingVolumeDataPolicy specifies the restore behavior for the volume data to be restored + // +optional + // +nullable + ExistingVolumeDataPolicy VolumeDataPolicyType `json:"existingVolumeDataPolicy,omitempty"` // ItemOperationTimeout specifies the time used to wait for RestoreItemAction operations // The default value is 4 hour. @@ -135,6 +140,14 @@ type RestoreSpec struct { // +nullable ResourcePolicy *corev1api.TypedLocalObjectReference `json:"resourcePolicy,omitempty"` + // SkipDefaultResourceModifier controls whether the server-configured default + // resource modifier is applied to this restore. + // When true, the default modifier is skipped even if configured on the server. + // Has no effect when a per-restore ResourceModifier is specified. + // +optional + // +nullable + SkipDefaultResourceModifier *bool `json:"skipDefaultResourceModifier,omitempty"` + // UploaderConfig specifies the configuration for the restore. // +optional // +nullable @@ -150,6 +163,11 @@ type UploaderConfigForRestore struct { // ParallelFilesDownload is the concurrency number setting for restore. // +optional ParallelFilesDownload int `json:"parallelFilesDownload,omitempty"` + // DeleteExtraFiles specifies whether to delete the extra files in the target volume that do not exist in the backup. + // This setting is only applicable to File System restores (PodVolumeBackup or CSI File System Data Move) and has no effect on Block Data Move restores. + // +optional + // +nullable + DeleteExtraFiles *bool `json:"deleteExtraFiles,omitempty"` } // RestoreHooks contains custom behaviors that should be executed during or post restore. @@ -316,13 +334,22 @@ const ( // The failing error is recorded in status.FailureReason. RestorePhaseFailed RestorePhase = "Failed" - // PolicyTypeNone means velero will not overwrite the resource + // ResourcePolicyTypeNone means velero will not overwrite the resource // in cluster with the one in backup whether changed/unchanged. - PolicyTypeNone PolicyType = "none" + ResourcePolicyTypeNone ResourcePolicyType = "none" - // PolicyTypeUpdate means velero will try to attempt a patch on + // ResourcePolicyTypeUpdate means velero will try to attempt a patch on // the changed resources. - PolicyTypeUpdate PolicyType = "update" + ResourcePolicyTypeUpdate ResourcePolicyType = "update" + + // VolumeDataPolicyTypeNone means velero will skip and not overwrite the volume data if the volume already exists + VolumeDataPolicyTypeNone VolumeDataPolicyType = "none" + + // VolumeDataPolicyTypeFull means velero will try to restore the volume data fully if the volume already exists. + VolumeDataPolicyTypeFull VolumeDataPolicyType = "full" + + // VolumeDataPolicyTypeIncremental means velero will try to restore the volume data incrementally if the volume already exists. + VolumeDataPolicyTypeIncremental VolumeDataPolicyType = "incremental" ) // RestoreStatus captures the current status of a Velero restore @@ -412,6 +439,11 @@ type RestoreProgress struct { // +kubebuilder:rbac:groups=velero.io,resources=restores,verbs=create;delete;get;list;patch;update;watch // +kubebuilder:rbac:groups=velero.io,resources=restores/status,verbs=get;update;patch // +kubebuilder:resource:shortName=rst +// +kubebuilder:printcolumn:name="Backup",type="string",JSONPath=".spec.backupName",description="The name of the backup this restore is from" +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="Restore status such as New/InProgress" +// +kubebuilder:printcolumn:name="Errors",type="integer",JSONPath=".status.errors",description="Total number of errors logged during the restore" +// +kubebuilder:printcolumn:name="Warnings",type="integer",JSONPath=".status.warnings",description="Total number of warnings logged during the restore" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // Restore is a Velero resource that represents the application of // resources from a Velero backup to a target Kubernetes cluster. @@ -428,6 +460,18 @@ type Restore struct { Status RestoreStatus `json:"status,omitempty"` } +func (r *Restore) IsVolumeDataInplaceRestore() bool { + return r.Spec.ExistingVolumeDataPolicy == VolumeDataPolicyTypeFull || r.Spec.ExistingVolumeDataPolicy == VolumeDataPolicyTypeIncremental +} + +func (r *Restore) IsVolumeDataInplaceFullRestore() bool { + return r.Spec.ExistingVolumeDataPolicy == VolumeDataPolicyTypeFull +} + +func (r *Restore) IsVolumeDataInplaceIncrementalRestore() bool { + return r.Spec.ExistingVolumeDataPolicy == VolumeDataPolicyTypeIncremental +} + // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // RestoreList is a list of Restores. @@ -440,5 +484,8 @@ type RestoreList struct { Items []Restore `json:"items"` } -// PolicyType helps specify the ExistingResourcePolicy -type PolicyType string +// ResourcePolicyType helps specify the ExistingResourcePolicy +type ResourcePolicyType string + +// VolumeDataPolicyType helps specify the ExistingVolumeDataPolicy +type VolumeDataPolicyType string diff --git a/pkg/apis/velero/v1/restore_types_test.go b/pkg/apis/velero/v1/restore_types_test.go new file mode 100644 index 000000000..72063d6f2 --- /dev/null +++ b/pkg/apis/velero/v1/restore_types_test.go @@ -0,0 +1,69 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "testing" +) + +func TestIsVolumeDataInplaceRestore(t *testing.T) { + tests := []struct { + name string + existingVolumeDataPolicy VolumeDataPolicyType + expected bool + }{ + { + name: "empty policy", + existingVolumeDataPolicy: "", + expected: false, + }, + { + name: "none policy", + existingVolumeDataPolicy: VolumeDataPolicyTypeNone, + expected: false, + }, + { + name: "full policy", + existingVolumeDataPolicy: VolumeDataPolicyTypeFull, + expected: true, + }, + { + name: "incremental policy", + existingVolumeDataPolicy: VolumeDataPolicyTypeIncremental, + expected: true, + }, + { + name: "unknown policy", + existingVolumeDataPolicy: VolumeDataPolicyType("unknown"), + expected: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + restore := &Restore{ + Spec: RestoreSpec{ + ExistingVolumeDataPolicy: tc.existingVolumeDataPolicy, + }, + } + actual := restore.IsVolumeDataInplaceRestore() + if actual != tc.expected { + t.Errorf("expected %v, got %v", tc.expected, actual) + } + }) + } +} diff --git a/pkg/apis/velero/v1/server_status_request_types.go b/pkg/apis/velero/v1/server_status_request_types.go index 98e15a0b5..26742e0a1 100644 --- a/pkg/apis/velero/v1/server_status_request_types.go +++ b/pkg/apis/velero/v1/server_status_request_types.go @@ -28,6 +28,10 @@ import ( // +kubebuilder:resource:shortName=ssr // +kubebuilder:object:generate=true // +kubebuilder:storageversion +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="The status of the server status request" +// +kubebuilder:printcolumn:name="Server Version",type="string",JSONPath=".status.serverVersion",description="The Velero server version" +// +kubebuilder:printcolumn:name="Processed",type="date",JSONPath=".status.processedTimestamp",description="The time the request was processed by the controller" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // ServerStatusRequest is a request to access current status information about // the Velero server. diff --git a/pkg/apis/velero/v1/volume_snapshot_location_type.go b/pkg/apis/velero/v1/volume_snapshot_location_type.go index 836701b77..1ff363a46 100644 --- a/pkg/apis/velero/v1/volume_snapshot_location_type.go +++ b/pkg/apis/velero/v1/volume_snapshot_location_type.go @@ -27,6 +27,9 @@ import ( // +kubebuilder:resource:shortName=vsl // +kubebuilder:object:generate=true // +kubebuilder:storageversion +// +kubebuilder:printcolumn:name="Provider",type="string",JSONPath=".spec.provider",description="Provider is the provider of the volume storage" +// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase",description="Volume Snapshot Location status such as Available/Unavailable" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // VolumeSnapshotLocation is a location where Velero stores volume snapshots. type VolumeSnapshotLocation struct { diff --git a/pkg/apis/velero/v1/zz_generated.deepcopy.go b/pkg/apis/velero/v1/zz_generated.deepcopy.go index c40fbb806..f4dc8a79a 100644 --- a/pkg/apis/velero/v1/zz_generated.deepcopy.go +++ b/pkg/apis/velero/v1/zz_generated.deepcopy.go @@ -895,6 +895,13 @@ func (in *Metadata) DeepCopyInto(out *Metadata) { (*out)[key] = val } } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Metadata. @@ -1048,6 +1055,11 @@ func (in *PodVolumeBackupStatus) DeepCopyInto(out *PodVolumeBackupStatus) { *out = (*in).DeepCopy() } out.Progress = in.Progress + if in.IncrementalBytes != nil { + in, out := &in.IncrementalBytes, &out.IncrementalBytes + *out = new(int64) + **out = **in + } if in.AcceptedTimestamp != nil { in, out := &in.AcceptedTimestamp, &out.AcceptedTimestamp *out = (*in).DeepCopy() @@ -1420,6 +1432,11 @@ func (in *RestoreSpec) DeepCopyInto(out *RestoreSpec) { *out = new(corev1.TypedLocalObjectReference) (*in).DeepCopyInto(*out) } + if in.SkipDefaultResourceModifier != nil { + in, out := &in.SkipDefaultResourceModifier, &out.SkipDefaultResourceModifier + *out = new(bool) + **out = **in + } if in.UploaderConfig != nil { in, out := &in.UploaderConfig, &out.UploaderConfig *out = new(UploaderConfigForRestore) @@ -1754,6 +1771,11 @@ func (in *UploaderConfigForRestore) DeepCopyInto(out *UploaderConfigForRestore) *out = new(bool) **out = **in } + if in.DeleteExtraFiles != nil { + in, out := &in.DeleteExtraFiles, &out.DeleteExtraFiles + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UploaderConfigForRestore. diff --git a/pkg/apis/velero/v2alpha1/data_download_types.go b/pkg/apis/velero/v2alpha1/data_download_types.go index 220bd382b..57827b97d 100644 --- a/pkg/apis/velero/v2alpha1/data_download_types.go +++ b/pkg/apis/velero/v2alpha1/data_download_types.go @@ -32,13 +32,21 @@ type DataDownloadSpec struct { BackupStorageLocation string `json:"backupStorageLocation"` // DataMover specifies the data mover to be used by the backup. - // If DataMover is "" or "velero", the built-in data mover will be used. + // If DataMover is "" or "velero", the built-in fs data mover will be used. // +optional DataMover string `json:"datamover,omitempty"` // SnapshotID is the ID of the Velero backup snapshot to be restored from. SnapshotID string `json:"snapshotID"` + // RestoreType indicates the type of the restore. + RestoreType string `json:"restoreType"` + + // CSISnapshot provides the information of the CSI snapshot used to do the incremental restore. + // +optional + // +nullable + CSISnapshot *CSISnapshotSpec `json:"csiSnapshot"` + // SourceNamespace is the original namespace where the volume is backed up from. // It may be different from SourcePVC's namespace if namespace is remapped during restore. SourceNamespace string `json:"sourceNamespace"` diff --git a/pkg/apis/velero/v2alpha1/data_upload_types.go b/pkg/apis/velero/v2alpha1/data_upload_types.go index ac57ad89d..db4c8d3a8 100644 --- a/pkg/apis/velero/v2alpha1/data_upload_types.go +++ b/pkg/apis/velero/v2alpha1/data_upload_types.go @@ -36,7 +36,7 @@ type DataUploadSpec struct { SourcePVC string `json:"sourcePVC"` // DataMover specifies the data mover to be used by the backup. - // If DataMover is "" or "velero", the built-in data mover will be used. + // If DataMover is "" or "velero", the built-in fs data mover will be used. // +optional DataMover string `json:"datamover,omitempty"` @@ -64,6 +64,12 @@ type DataUploadSpec struct { // SourceFSType is the file system type of the source volume. // +optional SourceFSType string `json:"sourceFSType,omitempty"` + + // ParentSnapshot specifies the parent snapshot that current backup is based on. + // If its value is "" or "auto", the data mover finds the recent backup of the same volume as parent. + // If its value is "none", the data mover will do a full backup + // If its value is a specific snapshotID, the data mover finds the specific snapshot as parent. + ParentSnapshot string `json:"parentSnapshot,omitempty"` } type SnapshotType string @@ -74,6 +80,10 @@ const ( // CSISnapshotSpec is the specification for a CSI snapshot. type CSISnapshotSpec struct { + // VolumeSnapshotNamespace is the namespece of the volume snapshot to be backed up + // +optional + VolumeSnapshotNamespace string `json:"volumeSnapshotNamespace"` + // VolumeSnapshot is the name of the volume snapshot to be backed up VolumeSnapshot string `json:"volumeSnapshot"` @@ -159,9 +169,13 @@ type DataUploadStatus struct { // +optional Progress shared.DataMoveOperationProgress `json:"progress,omitempty"` - // IncrementalBytes holds the number of bytes new or changed since the last backup + // IncrementalBytes holds the number of bytes new or changed since the last backup. + // A nil value means the uploader did not report a figure; a pointer to 0 means it + // reported zero, i.e. nothing changed and nothing was transferred. The two are + // distinct: erasing a measured zero makes a perfect incremental indistinguishable + // from a full transfer in every downstream report. // +optional - IncrementalBytes int64 `json:"incrementalBytes,omitempty"` + IncrementalBytes *int64 `json:"incrementalBytes,omitempty"` // Node is name of the node where the DataUpload is processed. // +optional @@ -262,4 +276,8 @@ type DataUploadResult struct { // FSType is the file system type of the volume. // +optional FSType string `json:"fsType,omitempty"` + + // SnapshotClass is the name of the snapshot class that the volume snapshot is created with + // +optional + SnapshotClass string `json:"snapshotClass,omitempty"` } diff --git a/pkg/apis/velero/v2alpha1/zz_generated.deepcopy.go b/pkg/apis/velero/v2alpha1/zz_generated.deepcopy.go index b86c573d3..927dc531c 100644 --- a/pkg/apis/velero/v2alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/velero/v2alpha1/zz_generated.deepcopy.go @@ -86,6 +86,11 @@ func (in *DataDownloadList) DeepCopyObject() runtime.Object { func (in *DataDownloadSpec) DeepCopyInto(out *DataDownloadSpec) { *out = *in out.TargetVolume = in.TargetVolume + if in.CSISnapshot != nil { + in, out := &in.CSISnapshot, &out.CSISnapshot + *out = new(CSISnapshotSpec) + **out = **in + } if in.DataMoverConfig != nil { in, out := &in.DataMoverConfig, &out.DataMoverConfig *out = make(map[string]string, len(*in)) @@ -270,6 +275,11 @@ func (in *DataUploadStatus) DeepCopyInto(out *DataUploadStatus) { *out = (*in).DeepCopy() } out.Progress = in.Progress + if in.IncrementalBytes != nil { + in, out := &in.IncrementalBytes, &out.IncrementalBytes + *out = new(int64) + **out = **in + } if in.AcceptedTimestamp != nil { in, out := &in.AcceptedTimestamp, &out.AcceptedTimestamp *out = (*in).DeepCopy() diff --git a/pkg/archive/extractor.go b/pkg/archive/extractor.go index aae9a7a85..15cacde22 100644 --- a/pkg/archive/extractor.go +++ b/pkg/archive/extractor.go @@ -32,14 +32,27 @@ import ( // Extractor unzips/extracts a backup tarball to a local // temp directory. type Extractor struct { - log logrus.FieldLogger - fs filesystem.Interface + log logrus.FieldLogger + fs filesystem.Interface + maxExtractionSize int64 + totalExtractedSize int64 +} + +var maxExtractionSize = int64(16) << 30 + +// SetMaxExtractionSize sets the maximum extraction size. It is normally called at server startup. +func SetMaxExtractionSize(size int64) { + if size > 0 { + maxExtractionSize = size + } } func NewExtractor(log logrus.FieldLogger, fs filesystem.Interface) *Extractor { return &Extractor{ - log: log, - fs: fs, + log: log, + fs: fs, + maxExtractionSize: maxExtractionSize, + totalExtractedSize: 0, } } @@ -96,6 +109,15 @@ func (e *Extractor) readBackup(tarRdr *tar.Reader) (string, error) { return "", err } + // Enforce maximum extraction size to prevent memory/storage exhaustion and zip bombs. + maxSize := e.maxExtractionSize + e.totalExtractedSize += header.Size + if e.totalExtractedSize > maxSize { + err := fmt.Errorf("decompressed backup exceeds maximum allowed size of %d bytes", maxSize) + e.log.Infof("error checking extracted size: %v", err) + return "", err + } + target, err := sanitizeArchivePath(dir, header.Name) if err != nil { e.log.Infof("error sanitizing archive path: %s", err.Error()) diff --git a/pkg/archive/extractor_test.go b/pkg/archive/extractor_test.go index a4daf02ca..d87f787c3 100644 --- a/pkg/archive/extractor_test.go +++ b/pkg/archive/extractor_test.go @@ -20,6 +20,7 @@ import ( "archive/tar" "bytes" "compress/gzip" + "fmt" "io" "os" "testing" @@ -113,6 +114,66 @@ func TestUnzipAndExtractBackupRejectsPathTraversal(t *testing.T) { require.Contains(t, err.Error(), "invalid archive path") } +func TestUnzipAndExtractBackupRejectsLargeFile(t *testing.T) { + SetMaxExtractionSize(1024) + defer SetMaxExtractionSize(16 * 1024 * 1024 * 1024) + ext := NewExtractor(test.NewLogger(), test.NewFakeFileSystem()) + + var buf bytes.Buffer + gzw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gzw) + + data := make([]byte, 2048) // 2KB data + err := tw.WriteHeader(&tar.Header{ + Name: "large.txt", + Mode: 0600, + Typeflag: tar.TypeReg, + Size: int64(len(data)), + }) + require.NoError(t, err) + + _, err = tw.Write(data) + require.NoError(t, err) + require.NoError(t, tw.Close()) + require.NoError(t, gzw.Close()) + + _, err = ext.UnzipAndExtractBackup(&buf) + require.Error(t, err) + require.Contains(t, err.Error(), "decompressed backup exceeds maximum allowed size") +} + +func TestUnzipAndExtractBackupRejectsManySmallFiles(t *testing.T) { + SetMaxExtractionSize(1024) + defer SetMaxExtractionSize(16 * 1024 * 1024 * 1024) + ext := NewExtractor(test.NewLogger(), test.NewFakeFileSystem()) + + var buf bytes.Buffer + gzw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gzw) + + // Create 100 files of 20 bytes each (total 2000 bytes, exceeding the 1024 byte limit) + for i := 0; i < 100; i++ { + data := make([]byte, 20) + err := tw.WriteHeader(&tar.Header{ + Name: fmt.Sprintf("small_%d.txt", i), + Mode: 0600, + Typeflag: tar.TypeReg, + Size: int64(len(data)), + }) + require.NoError(t, err) + + _, err = tw.Write(data) + require.NoError(t, err) + } + + require.NoError(t, tw.Close()) + require.NoError(t, gzw.Close()) + + _, err := ext.UnzipAndExtractBackup(&buf) + require.Error(t, err) + require.Contains(t, err.Error(), "decompressed backup exceeds maximum allowed size") +} + func createArchive(files []string, fs filesystem.Interface) (string, error) { outName := "output.tar.gz" out, err := fs.Create(outName) diff --git a/pkg/archive/filesystem.go b/pkg/archive/filesystem.go index 73b0d1dcf..310ab64dc 100644 --- a/pkg/archive/filesystem.go +++ b/pkg/archive/filesystem.go @@ -19,7 +19,9 @@ package archive import ( "encoding/json" "path/filepath" + "strings" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -27,13 +29,37 @@ import ( ) // GetItemFilePath returns an item's file path once extracted from a Velero backup archive. -func GetItemFilePath(rootDir, groupResource, namespace, name string) string { +func GetItemFilePath(rootDir, groupResource, namespace, name string) (string, error) { return GetVersionedItemFilePath(rootDir, groupResource, namespace, name, "") } // GetVersionedItemFilePath returns an item's file path once extracted from a Velero backup archive, with version included. -func GetVersionedItemFilePath(rootDir, groupResource, namespace, name, versionPath string) string { - return filepath.Join(rootDir, velerov1api.ResourcesDir, groupResource, versionPath, GetScopeDir(namespace), namespace, name+".json") +// +// The namespace and name components can originate from backup contents - for example the +// additional items a RestoreItemAction returns are built from annotations on a backed up +// object - so the joined path is verified to stay within rootDir. Without that check a +// component containing ".." escapes the extracted backup directory and addresses an +// arbitrary file on the Velero pod's filesystem. +func GetVersionedItemFilePath(rootDir, groupResource, namespace, name, versionPath string) (string, error) { + path := filepath.Join(rootDir, velerov1api.ResourcesDir, groupResource, versionPath, GetScopeDir(namespace), namespace, name+".json") + + // rootDir is empty when building the path of an entry inside the backup tarball rather + // than of an extracted file on disk; "." is the containment base for that relative form. + base := rootDir + if base == "" { + base = "." + } + + rel, err := filepath.Rel(base, path) + if err != nil { + return "", errors.Wrapf(err, "error resolving item path for %q/%q", namespace, name) + } + + if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", errors.Errorf("invalid item path for %q/%q: escapes the backup directory", namespace, name) + } + + return path, nil } // GetScopeDir returns NamespaceScopedDir if namespace is present, or ClusterScopedDir if empty diff --git a/pkg/archive/filesystem_test.go b/pkg/archive/filesystem_test.go index bf7f16c76..c6225ff85 100644 --- a/pkg/archive/filesystem_test.go +++ b/pkg/archive/filesystem_test.go @@ -27,31 +27,104 @@ import ( ) func TestGetItemFilePath(t *testing.T) { - res := GetItemFilePath("root", "resource", "", "item") + res, err := GetItemFilePath("root", "resource", "", "item") + require.NoError(t, err) assert.Equal(t, "root/resources/resource/cluster/item.json", res) - res = GetItemFilePath("root", "resource", "namespace", "item") + res, err = GetItemFilePath("root", "resource", "namespace", "item") + require.NoError(t, err) assert.Equal(t, "root/resources/resource/namespaces/namespace/item.json", res) - res = GetItemFilePath("", "resource", "", "item") + res, err = GetItemFilePath("", "resource", "", "item") + require.NoError(t, err) assert.Equal(t, "resources/resource/cluster/item.json", res) - res = GetVersionedItemFilePath("root", "resource", "", "item", "") + res, err = GetVersionedItemFilePath("root", "resource", "", "item", "") + require.NoError(t, err) assert.Equal(t, "root/resources/resource/cluster/item.json", res) - res = GetVersionedItemFilePath("root", "resource", "namespace", "item", "") + res, err = GetVersionedItemFilePath("root", "resource", "namespace", "item", "") + require.NoError(t, err) assert.Equal(t, "root/resources/resource/namespaces/namespace/item.json", res) - res = GetVersionedItemFilePath("root", "resource", "namespace", "item", "v1") + res, err = GetVersionedItemFilePath("root", "resource", "namespace", "item", "v1") + require.NoError(t, err) assert.Equal(t, "root/resources/resource/v1/namespaces/namespace/item.json", res) - res = GetVersionedItemFilePath("root", "resource", "", "item", "v1") + res, err = GetVersionedItemFilePath("root", "resource", "", "item", "v1") + require.NoError(t, err) assert.Equal(t, "root/resources/resource/v1/cluster/item.json", res) - res = GetVersionedItemFilePath("", "resource", "", "item", "") + res, err = GetVersionedItemFilePath("", "resource", "", "item", "") + require.NoError(t, err) assert.Equal(t, "resources/resource/cluster/item.json", res) } +// TestGetItemFilePathRejectsPathTraversal verifies that a name or namespace containing +// ".." cannot address a file outside the extracted backup directory. These components can +// come from backup contents, for example the additional items a RestoreItemAction builds +// from annotations on a backed up object. +func TestGetItemFilePathRejectsPathTraversal(t *testing.T) { + tests := []struct { + name string + rootDir string + groupResource string + namespace string + itemName string + }{ + { + name: "traversal in name escapes root", + rootDir: "/tmp/restore-dir", + groupResource: "secrets", + namespace: "x", + itemName: "../../../../../../root/.docker/config", + }, + { + name: "traversal in namespace escapes root", + rootDir: "/tmp/restore-dir", + groupResource: "secrets", + namespace: "../../../../../../etc", + itemName: "passwd", + }, + { + name: "traversal in group resource escapes root", + rootDir: "/tmp/restore-dir", + groupResource: "../../../../../../etc", + namespace: "", + itemName: "passwd", + }, + { + name: "traversal escapes archive-relative root", + rootDir: "", + groupResource: "secrets", + namespace: "x", + itemName: "../../../../../../escape", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + res, err := GetItemFilePath(tc.rootDir, tc.groupResource, tc.namespace, tc.itemName) + require.Error(t, err) + assert.Contains(t, err.Error(), "escapes the backup directory") + assert.Empty(t, res) + + res, err = GetVersionedItemFilePath(tc.rootDir, tc.groupResource, tc.namespace, tc.itemName, "v1") + require.Error(t, err) + assert.Contains(t, err.Error(), "escapes the backup directory") + assert.Empty(t, res) + }) + } +} + +// TestGetItemFilePathAllowsInnerDotDot verifies the containment check does not reject a +// path whose ".." segments resolve back inside the root directory. +func TestGetItemFilePathAllowsInnerDotDot(t *testing.T) { + res, err := GetItemFilePath("root", "resource", "namespaces/..", "item") + require.NoError(t, err) + assert.Equal(t, "root/resources/resource/namespaces/item.json", res) +} + func TestGetScopeDir(t *testing.T) { res := GetScopeDir("") assert.Equal(t, velerov1api.ClusterScopedDir, res) diff --git a/pkg/backup/actions/csi/pvc_action.go b/pkg/backup/actions/csi/pvc_action.go index 66c14b820..01f4e3d1a 100644 --- a/pkg/backup/actions/csi/pvc_action.go +++ b/pkg/backup/actions/csi/pvc_action.go @@ -42,11 +42,13 @@ import ( crclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + veleroshared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" veleroclient "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/label" + "github.com/vmware-tanzu/velero/pkg/nodeagent" plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/utils/volumehelper" "github.com/vmware-tanzu/velero/pkg/plugin/velero" @@ -54,6 +56,7 @@ import ( uploaderUtil "github.com/vmware-tanzu/velero/pkg/uploader/util" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/csi" + datamover "github.com/vmware-tanzu/velero/pkg/util/datamover" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" podvolumeutil "github.com/vmware-tanzu/velero/pkg/util/podvolume" vhutil "github.com/vmware-tanzu/velero/pkg/util/volumehelper" @@ -209,8 +212,10 @@ func (p *pvcBackupItemAction) validatePVCAndPV( } func (p *pvcBackupItemAction) createVolumeSnapshot( + ctx context.Context, pvc corev1api.PersistentVolumeClaim, backup *velerov1api.Backup, + policySnapshotClass string, ) ( vs *snapshotv1api.VolumeSnapshot, err error, @@ -218,7 +223,7 @@ func (p *pvcBackupItemAction) createVolumeSnapshot( p.log.Debugf("Fetching storage class for PV %s", *pvc.Spec.StorageClassName) storageClass := new(storagev1api.StorageClass) if err := p.crClient.Get( - context.TODO(), crclient.ObjectKey{Name: *pvc.Spec.StorageClassName}, + ctx, crclient.ObjectKey{Name: *pvc.Spec.StorageClassName}, storageClass, ); err != nil { return nil, errors.Wrap(err, "error getting storage class") @@ -226,11 +231,13 @@ func (p *pvcBackupItemAction) createVolumeSnapshot( p.log.Debugf("Fetching VolumeSnapshotClass for %s", storageClass.Provisioner) vsClass, err := csi.GetVolumeSnapshotClass( + ctx, storageClass.Provisioner, backup, &pvc, p.log, p.crClient, + policySnapshotClass, ) if err != nil { return nil, errors.Wrapf( @@ -261,7 +268,7 @@ func (p *pvcBackupItemAction) createVolumeSnapshot( }, } - if err := p.crClient.Create(context.TODO(), vs); err != nil { + if err := p.crClient.Create(ctx, vs); err != nil { return nil, errors.Wrapf( err, "error creating volume snapshot", ) @@ -290,6 +297,8 @@ func (p *pvcBackupItemAction) Execute( ) { p.log.Info("Starting PVCBackupItemAction") + ctx := context.Background() + if valid := p.validateBackup(*backup); !valid { return item, nil, "", nil, nil } @@ -314,7 +323,7 @@ func (p *pvcBackupItemAction) Execute( } // Ensure PVC-to-Pod cache is built for this namespace (lazy per-namespace caching) - if err := p.ensurePVCPodCacheForNamespace(context.TODO(), pvc.Namespace); err != nil { + if err := p.ensurePVCPodCacheForNamespace(ctx, pvc.Namespace); err != nil { return nil, nil, "", nil, err } @@ -337,7 +346,25 @@ func (p *pvcBackupItemAction) Execute( return nil, nil, "", nil, err } - vs, err := p.getVolumeSnapshotReference(context.TODO(), pvc, backup) + // validate that the node-agent daemonset is ready when snapshot data movement with + // the built-in data mover is requested. Without this, the DataUpload CR will be + // created but never processed (the DataUpload controller runs inside node-agent), + // causing the backup to hang until itemOperationTimeout expires. + if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) && datamover.IsBuiltInDataMover(backup.Spec.DataMover) { + if err := nodeagent.IsReady(ctx, backup.Namespace, p.crClient); err != nil { + p.log.WithError(err).Error("cannot perform snapshot data movement without running node-agent pods") + return nil, nil, "", nil, errors.Wrap(err, "CSI PVC BIA cannot proceed: node-agent is not ready for snapshot data movement") + } + } + + policySnapshotClass, scErr := vh.GetSnapshotClass(item, kuberesource.PersistentVolumeClaims) + if scErr != nil { + p.log.WithError(scErr).Warn("failed to get snapshotClass from volume policy, proceeding without it") + } else if policySnapshotClass != "" { + p.log.Infof("Volume policy specifies snapshotClass=%s for PVC %s/%s", policySnapshotClass, pvc.Namespace, pvc.Name) + } + + vs, err := p.getVolumeSnapshotReference(ctx, pvc, backup, policySnapshotClass) if err != nil { return nil, nil, "", nil, err } @@ -353,7 +380,7 @@ func (p *pvcBackupItemAction) Execute( if err != nil { p.log.Errorf("Failed to wait for VolumeSnapshot %s/%s to become ReadyToUse within timeout %v: %s", vs.Namespace, vs.Name, backup.Spec.CSISnapshotTimeout.Duration, err.Error()) - csi.CleanupVolumeSnapshot(vs, p.crClient, p.log) + csi.CleanupVolumeSnapshot(ctx, vs, p.crClient, p.log) return nil, nil, "", nil, errors.WithStack(err) } @@ -384,6 +411,8 @@ func (p *pvcBackupItemAction) Execute( "Backup": backup.Name, }) + dataMoverFromVolumePolicy := vh.GetDataMoverFromActionParameters(item, kuberesource.PersistentVolumeClaims) + dataUploadLog.Info("Starting data upload of backup") dataUpload, err := createDataUpload( @@ -395,13 +424,14 @@ func (p *pvcBackupItemAction) Execute( operationID, vsc, fsType, + dataMoverFromVolumePolicy, ) if err != nil { dataUploadLog.WithError(err).Error("failed to submit DataUpload") // TODO: need to use DeleteVolumeSnapshotIfAny, after data mover // adopting the controller-runtime client. - if deleteErr := p.crClient.Delete(context.TODO(), vs); deleteErr != nil { + if deleteErr := p.crClient.Delete(ctx, vs); deleteErr != nil { if !apierrors.IsNotFound(deleteErr) { dataUploadLog.WithError(deleteErr).Error("fail to delete VolumeSnapshot") } @@ -534,7 +564,19 @@ func newDataUpload( operationID string, vsc *snapshotv1api.VolumeSnapshotContent, fsType string, + dataMoverFromVolumePolicy string, ) *velerov2alpha1.DataUpload { + parentSnapshot := "" + + if backup.Spec.BackupType == velerov1api.BackupTypeFull { + parentSnapshot = veleroshared.ParentSnapshotNone + } + + dataMover := backup.Spec.DataMover + if dataMoverFromVolumePolicy != "" { + dataMover = dataMoverFromVolumePolicy + } + dataUpload := &velerov2alpha1.DataUpload{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov2alpha1.SchemeGroupVersion.String(), @@ -567,11 +609,12 @@ func newDataUpload( Driver: vsc.Spec.Driver, }, SourcePVC: pvc.Name, - DataMover: backup.Spec.DataMover, + DataMover: dataMover, BackupStorageLocation: backup.Spec.StorageLocation, SourceNamespace: pvc.Namespace, OperationTimeout: backup.Spec.CSISnapshotTimeout, SourceFSType: fsType, + ParentSnapshot: parentSnapshot, }, } @@ -597,8 +640,9 @@ func createDataUpload( operationID string, vsc *snapshotv1api.VolumeSnapshotContent, fsType string, + dataMoverFromVolumePolicy string, ) (*velerov2alpha1.DataUpload, error) { - dataUpload := newDataUpload(backup, vs, pvc, operationID, vsc, fsType) + dataUpload := newDataUpload(backup, vs, pvc, operationID, vsc, fsType, dataMoverFromVolumePolicy) err := crClient.Create(ctx, dataUpload) if err != nil { @@ -670,6 +714,7 @@ func (p *pvcBackupItemAction) getVolumeSnapshotReference( ctx context.Context, pvc corev1api.PersistentVolumeClaim, backup *velerov1api.Backup, + policySnapshotClass string, ) (*snapshotv1api.VolumeSnapshot, error) { vgsLabelKey := backup.Spec.VolumeGroupSnapshotLabelKey group, hasLabel := pvc.Labels[vgsLabelKey] @@ -800,7 +845,7 @@ func (p *pvcBackupItemAction) getVolumeSnapshotReference( } // Legacy fallback: create individual VS - return p.createVolumeSnapshot(pvc, backup) + return p.createVolumeSnapshot(ctx, pvc, backup, policySnapshotClass) } func (p *pvcBackupItemAction) findExistingVSForBackup( @@ -1189,7 +1234,7 @@ func setPVCRequestSizeToVSRestoreSize( logger logrus.FieldLogger, ) { if vsc.Status.RestoreSize != nil { - logger.Debugf("Patching PVC request size to fit the volumesnapshot restore size %d", vsc.Status.RestoreSize) + logger.Debugf("Patching PVC request size to fit the volumesnapshot restore size %d", *vsc.Status.RestoreSize) restoreSize := *resource.NewQuantity(*vsc.Status.RestoreSize, resource.BinarySI) // It is possible that the volume provider allocated a larger diff --git a/pkg/backup/actions/csi/pvc_action_test.go b/pkg/backup/actions/csi/pvc_action_test.go index e7320cd1a..4e021c7d1 100644 --- a/pkg/backup/actions/csi/pvc_action_test.go +++ b/pkg/backup/actions/csi/pvc_action_test.go @@ -23,31 +23,26 @@ import ( "testing" "time" - "github.com/vmware-tanzu/velero/pkg/kuberesource" - - volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2" - "github.com/stretchr/testify/assert" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/ptr" - - "github.com/vmware-tanzu/velero/pkg/label" - + "github.com/cockroachdb/errors" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - - "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" storagev1api "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/utils/ptr" crclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" @@ -55,8 +50,12 @@ import ( velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/builder" factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" + "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" + uploaderUtil "github.com/vmware-tanzu/velero/pkg/uploader/util" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" ) const testDriver = "csi.example.com" @@ -81,21 +80,23 @@ func (c *errorInjectingClient) Create(ctx context.Context, obj crclient.Object, func TestExecute(t *testing.T) { boolTrue := true tests := []struct { - name string - backup *velerov1api.Backup - pvc *corev1api.PersistentVolumeClaim - pv *corev1api.PersistentVolume - sc *storagev1api.StorageClass - vsClass *snapshotv1api.VolumeSnapshotClass - operationID string - expectedErr error - expectErr bool // Use bool for cases where we just need to check for any error - expectedBackup *velerov1api.Backup - expectedDataUpload *velerov2alpha1.DataUpload - expectedPVC *corev1api.PersistentVolumeClaim - resourcePolicy *corev1api.ConfigMap - failVSCreate bool - skipVSReadyUpdate bool // New flag to control VS readiness + name string + backup *velerov1api.Backup + pvc *corev1api.PersistentVolumeClaim + pv *corev1api.PersistentVolume + sc *storagev1api.StorageClass + vsClass *snapshotv1api.VolumeSnapshotClass + operationID string + expectedErr error + expectErr bool // Use bool for cases where we just need to check for any error + expectedBackup *velerov1api.Backup + expectedDataUpload *velerov2alpha1.DataUpload + expectedPVC *corev1api.PersistentVolumeClaim + resourcePolicy *corev1api.ConfigMap + extraObjects []runtime.Object + failVSCreate bool + skipVSReadyUpdate bool // New flag to control VS readiness + expectedVSClassName string }{ { name: "Skip PVC BIA when backup is in finalizing phase", @@ -122,12 +123,21 @@ func TestExecute(t *testing.T) { expectErr: true, // Expect an error, but the exact message can vary }, { - name: "Test SnapshotMoveData", - backup: builder.ForBackup("velero", "test").SnapshotMoveData(true).CSISnapshotTimeout(1 * time.Minute).Result(), - pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1api.ClaimBound).Result(), - pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), - sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), - vsClass: builder.ForVolumeSnapshotClass("testVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), + name: "Test SnapshotMoveData", + backup: builder.ForBackup("velero", "test").SnapshotMoveData(true).CSISnapshotTimeout(1 * time.Minute).Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1api.ClaimBound).Result(), + pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), + sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), + vsClass: builder.ForVolumeSnapshotClass("testVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), + extraObjects: []runtime.Object{ + &corev1api.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "linux-node", Labels: map[string]string{corev1api.LabelOSStable: "linux"}}, + }, + &appsv1api.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Namespace: "velero", Name: "node-agent"}, + Status: appsv1api.DaemonSetStatus{NumberReady: 3}, + }, + }, operationID: ".", expectedDataUpload: &velerov2alpha1.DataUpload{ TypeMeta: metav1.TypeMeta{ @@ -163,22 +173,42 @@ func TestExecute(t *testing.T) { SourcePVC: "testPVC", SourceNamespace: "velero", OperationTimeout: metav1.Duration{Duration: 1 * time.Minute}, + ParentSnapshot: "", }, }, }, { - name: "Verify PVC is modified as expected", - backup: builder.ForBackup("velero", "test").SnapshotMoveData(true).CSISnapshotTimeout(1 * time.Minute).Result(), - pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1api.ClaimBound).Result(), - pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), - sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), - vsClass: builder.ForVolumeSnapshotClass("tescVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), + name: "Verify PVC is modified as expected", + backup: builder.ForBackup("velero", "test").SnapshotMoveData(true).CSISnapshotTimeout(1 * time.Minute).Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1api.ClaimBound).Result(), + pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), + sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), + vsClass: builder.ForVolumeSnapshotClass("tescVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), + extraObjects: []runtime.Object{ + &corev1api.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "linux-node", Labels: map[string]string{corev1api.LabelOSStable: "linux"}}, + }, + &appsv1api.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Namespace: "velero", Name: "node-agent"}, + Status: appsv1api.DaemonSetStatus{NumberReady: 3}, + }, + }, operationID: ".", expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC"). ObjectMeta(builder.WithAnnotations(velerov1api.MustIncludeAdditionalItemAnnotation, "true", velerov1api.DataUploadNameAnnotation, "velero/"), builder.WithLabels(velerov1api.BackupNameLabel, "test")). VolumeName("testPV").StorageClass("testSC").Phase(corev1api.ClaimBound).Result(), }, + { + name: "Test SnapshotMoveData without node-agent", + backup: builder.ForBackup("velero", "test").SnapshotMoveData(true).CSISnapshotTimeout(1 * time.Minute).Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1api.ClaimBound).Result(), + pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), + sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), + vsClass: builder.ForVolumeSnapshotClass("testVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), + expectErr: true, + skipVSReadyUpdate: true, + }, { name: "Test ResourcePolicy", backup: builder.ForBackup("velero", "test").ResourcePolicies("resourcePolicy").SnapshotVolumes(false).CSISnapshotTimeout(time.Duration(3600) * time.Second).Result(), @@ -188,6 +218,16 @@ func TestExecute(t *testing.T) { sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), vsClass: builder.ForVolumeSnapshotClass("tescVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), }, + { + name: "Volume policy with snapshotClass selects correct VolumeSnapshotClass", + backup: builder.ForBackup("velero", "test").ResourcePolicies("resourcePolicy").CSISnapshotTimeout(time.Duration(3600) * time.Second).Result(), + resourcePolicy: builder.ForConfigMap("velero", "resourcePolicy").Data("policy", `{"version":"v1","volumePolicies":[{"conditions":{"csi":{}},"action":{"type":"snapshot","parameters":{"snapshotClass":"policy-selected-vsclass"}}}]}`).Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1api.ClaimBound).Result(), + pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), + sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), + vsClass: builder.ForVolumeSnapshotClass("policy-selected-vsclass").Driver("hostpath").Result(), + expectedVSClassName: "policy-selected-vsclass", + }, } for _, tc := range tests { @@ -210,6 +250,7 @@ func TestExecute(t *testing.T) { if tc.resourcePolicy != nil { objects = append(objects, tc.resourcePolicy) } + objects = append(objects, tc.extraObjects...) var crClient crclient.Client if tc.failVSCreate { @@ -301,6 +342,15 @@ func TestExecute(t *testing.T) { runtime.DefaultUnstructuredConverter.FromUnstructured(resultUnstructed.UnstructuredContent(), resultPVC) require.True(t, cmp.Equal(tc.expectedPVC, resultPVC, cmpopts.IgnoreFields(corev1api.PersistentVolumeClaim{}, "ResourceVersion", "Annotations", "Labels"))) } + + if tc.expectedVSClassName != "" { + vsList := new(snapshotv1api.VolumeSnapshotList) + require.NoError(t, crClient.List(t.Context(), vsList, &crclient.ListOptions{Namespace: tc.pvc.Namespace})) + require.NotEmpty(t, vsList.Items, "expected VolumeSnapshot to be created") + require.NotNil(t, vsList.Items[0].Spec.VolumeSnapshotClassName) + assert.Equal(t, tc.expectedVSClassName, *vsList.Items[0].Spec.VolumeSnapshotClassName, + "VolumeSnapshot should use the VolumeSnapshotClass specified by volume policy") + } }) } } @@ -2176,3 +2226,146 @@ func TestGetOrCreateVolumeHelper(t *testing.T) { // The pvcPodCache should be the same instance require.Same(t, cache1, action.pvcPodCache, "Expected same pvcPodCache instance on repeated calls") } + +func TestNewDataUpload(t *testing.T) { + tests := []struct { + name string + backupType velerov1api.BackupType + vsClassName *string + uploaderConfig *velerov1api.UploaderConfigForBackup + dataMoverFromVolumePolicy string + expectedParentSnap string + expectedDataMoverCfg map[string]string + }{ + { + name: "Full backup type, no uploader config, no vs class name", + backupType: velerov1api.BackupTypeFull, + vsClassName: nil, + uploaderConfig: nil, + expectedParentSnap: "none", + expectedDataMoverCfg: nil, + }, + { + name: "Incremental backup type, with uploader config, with vs class name", + backupType: velerov1api.BackupTypeIncremental, + vsClassName: ptr.To("test-vs-class"), + uploaderConfig: &velerov1api.UploaderConfigForBackup{ParallelFilesUpload: 10}, + expectedParentSnap: "", + expectedDataMoverCfg: map[string]string{ + uploaderUtil.ParallelFilesUpload: "10", + }, + }, + { + name: "Default backup type, uploader config with 0 parallel files", + backupType: "", + vsClassName: ptr.To("test-vs-class"), + uploaderConfig: &velerov1api.UploaderConfigForBackup{ParallelFilesUpload: 0}, + expectedParentSnap: "", + expectedDataMoverCfg: nil, + }, + { + name: "Default backup type, uploader config with 0 parallel files", + backupType: "", + vsClassName: ptr.To("test-vs-class"), + uploaderConfig: &velerov1api.UploaderConfigForBackup{ParallelFilesUpload: 0}, + dataMoverFromVolumePolicy: "velero-block", + expectedParentSnap: "", + expectedDataMoverCfg: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + backup := &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-backup", + Namespace: "velero", + UID: types.UID("backup-uid"), + }, + Spec: velerov1api.BackupSpec{ + BackupType: tc.backupType, + DataMover: "velero", + StorageLocation: "default", + CSISnapshotTimeout: metav1.Duration{Duration: 10 * time.Minute}, + UploaderConfig: tc.uploaderConfig, + }, + } + + vs := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vs", + }, + Spec: snapshotv1api.VolumeSnapshotSpec{ + VolumeSnapshotClassName: tc.vsClassName, + }, + } + + pvc := &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pvc", + Namespace: "test-ns", + UID: types.UID("pvc-uid"), + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + StorageClassName: ptr.To("test-storage-class"), + }, + } + + vsc := &snapshotv1api.VolumeSnapshotContent{ + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + Driver: "test-driver", + }, + } + + operationID := "test-op-id" + fsType := "ext4" + + du := newDataUpload(backup, vs, pvc, operationID, vsc, fsType, tc.dataMoverFromVolumePolicy) + + require.NotNil(t, du) + assert.Equal(t, velerov2alpha1.SchemeGroupVersion.String(), du.APIVersion) + assert.Equal(t, "DataUpload", du.Kind) + assert.Equal(t, backup.Namespace, du.Namespace) + assert.Equal(t, backup.Name+"-", du.GenerateName) + + require.Len(t, du.OwnerReferences, 1) + assert.Equal(t, velerov1api.SchemeGroupVersion.String(), du.OwnerReferences[0].APIVersion) + assert.Equal(t, "Backup", du.OwnerReferences[0].Kind) + assert.Equal(t, backup.Name, du.OwnerReferences[0].Name) + assert.Equal(t, backup.UID, du.OwnerReferences[0].UID) + assert.Equal(t, boolptr.True(), du.OwnerReferences[0].Controller) + + expectedLabels := map[string]string{ + velerov1api.BackupNameLabel: label.GetValidName(backup.Name), + velerov1api.BackupUIDLabel: string(backup.UID), + velerov1api.PVCUIDLabel: string(pvc.UID), + velerov1api.AsyncOperationIDLabel: operationID, + } + assert.Equal(t, expectedLabels, du.Labels) + + assert.Equal(t, velerov2alpha1.SnapshotTypeCSI, du.Spec.SnapshotType) + assert.Equal(t, vs.Name, du.Spec.CSISnapshot.VolumeSnapshot) + assert.Equal(t, *pvc.Spec.StorageClassName, du.Spec.CSISnapshot.StorageClass) + assert.Equal(t, vsc.Spec.Driver, du.Spec.CSISnapshot.Driver) + if tc.vsClassName != nil { + assert.Equal(t, *tc.vsClassName, du.Spec.CSISnapshot.SnapshotClass) + } else { + assert.Empty(t, du.Spec.CSISnapshot.SnapshotClass) + } + + assert.Equal(t, pvc.Name, du.Spec.SourcePVC) + if tc.dataMoverFromVolumePolicy != "" { + assert.Equal(t, tc.dataMoverFromVolumePolicy, du.Spec.DataMover) + } else { + assert.Equal(t, backup.Spec.DataMover, du.Spec.DataMover) + } + + assert.Equal(t, backup.Spec.StorageLocation, du.Spec.BackupStorageLocation) + assert.Equal(t, pvc.Namespace, du.Spec.SourceNamespace) + assert.Equal(t, backup.Spec.CSISnapshotTimeout, du.Spec.OperationTimeout) + assert.Equal(t, fsType, du.Spec.SourceFSType) + assert.Equal(t, tc.expectedParentSnap, du.Spec.ParentSnapshot) + assert.Equal(t, tc.expectedDataMoverCfg, du.Spec.DataMoverConfig) + }) + } +} diff --git a/pkg/backup/actions/csi/volumesnapshot_action.go b/pkg/backup/actions/csi/volumesnapshot_action.go index 49e690e93..eb302f2f2 100644 --- a/pkg/backup/actions/csi/volumesnapshot_action.go +++ b/pkg/backup/actions/csi/volumesnapshot_action.go @@ -78,6 +78,8 @@ func (p *volumeSnapshotBackupItemAction) Execute( ) { p.log.Infof("Executing VolumeSnapshotBackupItemAction") + ctx := context.Background() + vs := new(snapshotv1api.VolumeSnapshot) if err := runtime.DefaultUnstructuredConverter.FromUnstructured( item.UnstructuredContent(), vs); err != nil { @@ -90,7 +92,7 @@ func (p *volumeSnapshotBackupItemAction) Execute( WithField("Backup", fmt.Sprintf("%s/%s", backup.Namespace, backup.Name)). WithField("BackupPhase", backup.Status.Phase).Debugf("Cleaning VolumeSnapshots.") - csi.DeleteReadyVolumeSnapshot(*vs, p.crClient, p.log) + csi.DeleteReadyVolumeSnapshot(ctx, *vs, p.crClient, p.log) return item, nil, "", nil, nil } @@ -115,11 +117,9 @@ func (p *volumeSnapshotBackupItemAction) Execute( p.log.Infof("Getting VolumesnapshotContent for Volumesnapshot %s/%s", vs.Namespace, vs.Name) - ctx := context.TODO() - vsc, err := csi.GetVSCForVS(ctx, vs, p.crClient) if err != nil { - csi.CleanupVolumeSnapshot(vs, p.crClient, p.log) + csi.CleanupVolumeSnapshot(ctx, vs, p.crClient, p.log) return nil, nil, "", nil, errors.WithStack(err) } @@ -187,7 +187,7 @@ func (p *volumeSnapshotBackupItemAction) Execute( ) if vscPatchError := p.crClient.Patch( - context.TODO(), + ctx, vsc, crclient.MergeFrom(originVSC), ); vscPatchError != nil { @@ -203,7 +203,7 @@ func (p *volumeSnapshotBackupItemAction) Execute( originVS := vs.DeepCopy() kubeutil.AddAnnotations(&vs.ObjectMeta, annotations) if err := p.crClient.Patch( - context.TODO(), + ctx, vs, crclient.MergeFrom(originVS), ); err != nil { @@ -269,8 +269,8 @@ func (p *volumeSnapshotBackupItemAction) Progress( } var err error if progress.Started, err = time.Parse(time.RFC3339, operationIDParts[2]); err != nil { - p.log.Errorf("error parsing operation ID's StartedTime", - "part into time %s: %s", operationID, err.Error()) + p.log.Errorf("error parsing operation ID's StartedTime part into time %s: %s", + operationID, err.Error()) return progress, errors.WithStack(err) } diff --git a/pkg/backup/actions/csi/volumesnapshotcontent_action.go b/pkg/backup/actions/csi/volumesnapshotcontent_action.go index f184230d1..fc93cfb87 100644 --- a/pkg/backup/actions/csi/volumesnapshotcontent_action.go +++ b/pkg/backup/actions/csi/volumesnapshotcontent_action.go @@ -107,8 +107,7 @@ func (p *volumeSnapshotContentBackupItemAction) Execute( } p.log.Infof( - "Returning from VolumeSnapshotContentBackupItemAction", - "with %d additionalItems to backup", + "Returning from VolumeSnapshotContentBackupItemAction with %d additionalItems to backup", len(additionalItems), ) return &unstructured.Unstructured{Object: snapContMap}, additionalItems, "", nil, nil diff --git a/pkg/backup/backup.go b/pkg/backup/backup.go index dc60bba8c..038b85cc1 100644 --- a/pkg/backup/backup.go +++ b/pkg/backup/backup.go @@ -1263,21 +1263,12 @@ func buildFinalTarball(tr *tar.Reader, tw tarWriter, updateFiles map[string]File return errors.WithStack(err) } delete(updateFiles, header.Name) - // skip over file contents from old tarball - _, err := io.ReadAll(tr) - if err != nil { - return errors.WithStack(err) - } } else { // Add original content to new tarball, as item wasn't updated - oldContents, err := io.ReadAll(tr) - if err != nil { - return errors.WithStack(err) - } if err := tw.WriteHeader(header); err != nil { return errors.WithStack(err) } - if _, err := tw.Write(oldContents); err != nil { + if _, err := io.Copy(tw, tr); err != nil { return errors.WithStack(err) } } @@ -1428,22 +1419,20 @@ func resolveClusterScopedFilterPolicy( } func resolveResourceFilter(rf resourcepolicies.ResourceFilter) (*ResolvedResourceFilter, error) { - var selector labels.Selector - if len(rf.LabelSelector) > 0 { - var err error - selector, err = labels.ValidatedSelectorFromSet(labels.Set(rf.LabelSelector)) - if err != nil { - return nil, fmt.Errorf("invalid label selector in resource filter: %w", err) - } + selector, err := resourcepolicies.SelectorFromPolicyLabelSelector(rf.LabelSelector) + if err != nil { + return nil, fmt.Errorf("invalid label selector in resource filter: %w", err) } var orSelectors []labels.Selector for _, ols := range rf.OrLabelSelectors { - s, err := labels.ValidatedSelectorFromSet(labels.Set(ols)) + s, err := resourcepolicies.SelectorFromPolicyLabelSelector(ols) if err != nil { return nil, fmt.Errorf("invalid OR label selector in resource filter: %w", err) } - orSelectors = append(orSelectors, s) + if s != nil { + orSelectors = append(orSelectors, s) + } } var nameIE *collections.IncludesExcludes diff --git a/pkg/backup/backup_test.go b/pkg/backup/backup_test.go index 56f4aaf33..5d1ed1da2 100644 --- a/pkg/backup/backup_test.go +++ b/pkg/backup/backup_test.go @@ -43,6 +43,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/utils/ptr" "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/volume" @@ -2931,7 +2932,6 @@ func (*fakeVolumeSnapshotter) DeleteSnapshot(snapshotID string) error { // looking at the backup request's VolumeSnapshots field. This test uses the fakeVolumeSnapshotter // struct in place of real volume snapshotters. func TestBackupWithSnapshots(t *testing.T) { - // TODO: add more verification for skippedPVTracker itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) defer itemBlockPool.Stop() tests := []struct { @@ -2941,6 +2941,7 @@ func TestBackupWithSnapshots(t *testing.T) { apiResources []*test.APIResource snapshotterGetter volumeSnapshotterGetter want []*volume.Snapshot + wantSkippedPVs []SkippedPV }{ { name: "persistent volume with no zone annotation creates a snapshot", @@ -2977,6 +2978,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, }, }, + wantSkippedPVs: []SkippedPV{}, }, { name: "persistent volume with deprecated zone annotation creates a snapshot", @@ -2991,7 +2993,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, apiResources: []*test.APIResource{ test.PVs( - builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabels("failure-domain.beta.kubernetes.io/zone", "zone-1")).Result(), + builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabels(corev1api.LabelFailureDomainBetaZone, "zone-1")).Result(), ), }, snapshotterGetter: map[string]vsv1.VolumeSnapshotter{ @@ -3014,6 +3016,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, }, }, + wantSkippedPVs: []SkippedPV{}, }, { name: "persistent volume with GA zone annotation creates a snapshot", @@ -3028,7 +3031,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, apiResources: []*test.APIResource{ test.PVs( - builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabels("topology.kubernetes.io/zone", "zone-1")).Result(), + builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabels(corev1api.LabelTopologyZone, "zone-1")).Result(), ), }, snapshotterGetter: map[string]vsv1.VolumeSnapshotter{ @@ -3051,6 +3054,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, }, }, + wantSkippedPVs: []SkippedPV{}, }, { name: "persistent volume with both GA and deprecated zone annotation creates a snapshot and should use the GA", @@ -3065,7 +3069,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, apiResources: []*test.APIResource{ test.PVs( - builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabelsMap(map[string]string{"failure-domain.beta.kubernetes.io/zone": "zone-1-deprecated", "topology.kubernetes.io/zone": "zone-1-ga"})).Result(), + builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabelsMap(map[string]string{corev1api.LabelFailureDomainBetaZone: "zone-1-deprecated", corev1api.LabelTopologyZone: "zone-1-ga"})).Result(), ), }, snapshotterGetter: map[string]vsv1.VolumeSnapshotter{ @@ -3088,6 +3092,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, }, }, + wantSkippedPVs: []SkippedPV{}, }, { name: "error returned from CreateSnapshot results in a failed snapshot", @@ -3123,6 +3128,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, }, }, + wantSkippedPVs: []SkippedPV{}, }, { name: "backup with SnapshotVolumes=false does not create any snapshots", @@ -3144,6 +3150,17 @@ func TestBackupWithSnapshots(t *testing.T) { "default": new(fakeVolumeSnapshotter).WithVolume("pv-1", "vol-1", "", "type-1", 100, false), }, want: nil, + wantSkippedPVs: []SkippedPV{ + { + Name: "pv-1", + Reasons: []PVSkipReason{ + { + Approach: volumeSnapshotApproach, + Reason: "not satisfy the criteria for VolumePolicy or the legacy snapshot way", + }, + }, + }, + }, }, { name: "backup with no volume snapshot locations does not create any snapshots", @@ -3162,6 +3179,17 @@ func TestBackupWithSnapshots(t *testing.T) { "default": new(fakeVolumeSnapshotter).WithVolume("pv-1", "vol-1", "", "type-1", 100, false), }, want: nil, + wantSkippedPVs: []SkippedPV{ + { + Name: "pv-1", + Reasons: []PVSkipReason{ + { + Approach: volumeSnapshotApproach, + Reason: "no applicable volumesnapshotter found", + }, + }, + }, + }, }, { name: "backup with no volume snapshotters does not create any snapshots", @@ -3181,6 +3209,17 @@ func TestBackupWithSnapshots(t *testing.T) { }, snapshotterGetter: map[string]vsv1.VolumeSnapshotter{}, want: nil, + wantSkippedPVs: []SkippedPV{ + { + Name: "pv-1", + Reasons: []PVSkipReason{ + { + Approach: volumeSnapshotApproach, + Reason: "no applicable volumesnapshotter found", + }, + }, + }, + }, }, { name: "unsupported persistent volume type does not create any snapshots", @@ -3202,6 +3241,17 @@ func TestBackupWithSnapshots(t *testing.T) { "default": new(fakeVolumeSnapshotter), }, want: nil, + wantSkippedPVs: []SkippedPV{ + { + Name: "pv-1", + Reasons: []PVSkipReason{ + { + Approach: volumeSnapshotApproach, + Reason: "no applicable volumesnapshotter found", + }, + }, + }, + }, }, { name: "when there are multiple volumes, snapshot locations, and snapshotters, volumes are matched to the right snapshotters", @@ -3255,6 +3305,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, }, }, + wantSkippedPVs: []SkippedPV{}, }, } @@ -3273,6 +3324,7 @@ func TestBackupWithSnapshots(t *testing.T) { require.NoError(t, err) assert.Equal(t, tc.want, tc.req.VolumeSnapshots.Get()) + assert.Equal(t, tc.wantSkippedPVs, tc.req.SkippedPVTracker.Summary()) }) } } @@ -5429,6 +5481,29 @@ func TestBackupNamespaces(t *testing.T) { "resources/namespaces/v1-preferredversion/cluster/ns-3.json", }, }, + { + name: "Wildcard star with excluded namespaces test", + backup: defaultBackup().IncludedNamespaces("*").ExcludedNamespaces("ns-2").Result(), + apiResources: []*test.APIResource{ + test.Namespaces( + builder.ForNamespace("ns-1").Phase(corev1api.NamespaceActive).Result(), + builder.ForNamespace("ns-2").Phase(corev1api.NamespaceActive).Result(), + builder.ForNamespace("ns-3").Phase(corev1api.NamespaceActive).Result(), + ), + test.Deployments( + builder.ForDeployment("ns-1", "deploy-1").Result(), + builder.ForDeployment("ns-2", "deploy-2").Result(), + ), + }, + want: []string{ + "resources/namespaces/cluster/ns-1.json", + "resources/namespaces/v1-preferredversion/cluster/ns-1.json", + "resources/namespaces/cluster/ns-3.json", + "resources/namespaces/v1-preferredversion/cluster/ns-3.json", + "resources/deployments.apps/namespaces/ns-1/deploy-1.json", + "resources/deployments.apps/v1-preferredversion/namespaces/ns-1/deploy-1.json", + }, + }, { name: "Empty namespace test", backup: defaultBackup().IncludedNamespaces("invalid*").Result(), @@ -5607,7 +5682,7 @@ func TestUpdateVolumeInfos(t *testing.T) { RetainedSnapshot: "vs-1", SnapshotHandle: "snapshot-id", Size: 1000, - IncrementalSize: 500, + IncrementalSize: ptr.To(int64(500)), Phase: velerov2alpha1.DataUploadPhaseFailed, }, }, @@ -5647,7 +5722,7 @@ func TestUpdateVolumeInfos(t *testing.T) { RetainedSnapshot: "vs-1", SnapshotHandle: "snapshot-id", Size: 1000, - IncrementalSize: 500, + IncrementalSize: ptr.To(int64(500)), Phase: velerov2alpha1.DataUploadPhaseCompleted, }, }, @@ -5741,7 +5816,7 @@ func TestResolveResourceFilter(t *testing.T) { { name: "valid label selector", rf: resourcepolicies.ResourceFilter{ - LabelSelector: map[string]string{"app": "foo"}, + LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"app": "foo"}}, }, expectErr: false, checkResult: func(t *testing.T, r *ResolvedResourceFilter) { @@ -5754,16 +5829,16 @@ func TestResolveResourceFilter(t *testing.T) { { name: "invalid label selector", rf: resourcepolicies.ResourceFilter{ - LabelSelector: map[string]string{"invalid/label/key": "value"}, + LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"invalid/label/key": "value"}}, }, expectErr: true, }, { name: "valid or label selectors", rf: resourcepolicies.ResourceFilter{ - OrLabelSelectors: []map[string]string{ - {"app": "foo"}, - {"app": "bar"}, + OrLabelSelectors: []*resourcepolicies.PolicyLabelSelector{ + {MatchLabels: map[string]string{"app": "foo"}}, + {MatchLabels: map[string]string{"app": "bar"}}, }, }, expectErr: false, @@ -5776,8 +5851,8 @@ func TestResolveResourceFilter(t *testing.T) { { name: "invalid or label selectors", rf: resourcepolicies.ResourceFilter{ - OrLabelSelectors: []map[string]string{ - {"invalid/label/key": "value"}, + OrLabelSelectors: []*resourcepolicies.PolicyLabelSelector{ + {MatchLabels: map[string]string{"invalid/label/key": "value"}}, }, }, expectErr: true, @@ -5797,6 +5872,68 @@ func TestResolveResourceFilter(t *testing.T) { assert.False(t, r.NameIE.ShouldInclude("exc1")) }, }, + { + name: "empty labelSelector is no filter", + rf: resourcepolicies.ResourceFilter{ + LabelSelector: &resourcepolicies.PolicyLabelSelector{}, + }, + expectErr: false, + checkResult: func(t *testing.T, r *ResolvedResourceFilter) { + t.Helper() + require.NotNil(t, r) + assert.Nil(t, r.LabelSelector) + }, + }, + { + name: "set-based In and DoesNotExist", + rf: resourcepolicies.ResourceFilter{ + LabelSelector: &resourcepolicies.PolicyLabelSelector{ + MatchExpressions: []resourcepolicies.PolicyLabelSelectorRequirement{ + {Key: "environment", Operator: "In", Values: []string{"prod", "staging"}}, + {Key: "do-not-backup", Operator: "DoesNotExist"}, + }, + }, + }, + expectErr: false, + checkResult: func(t *testing.T, r *ResolvedResourceFilter) { + t.Helper() + require.NotNil(t, r.LabelSelector) + assert.True(t, r.LabelSelector.Matches(labels.Set{"environment": "prod"})) + assert.True(t, r.LabelSelector.Matches(labels.Set{"environment": "staging"})) + assert.False(t, r.LabelSelector.Matches(labels.Set{"environment": "dev"})) + assert.False(t, r.LabelSelector.Matches(labels.Set{"environment": "prod", "do-not-backup": "true"})) + }, + }, + { + name: "set-based NotIn and Exists", + rf: resourcepolicies.ResourceFilter{ + LabelSelector: &resourcepolicies.PolicyLabelSelector{ + MatchExpressions: []resourcepolicies.PolicyLabelSelectorRequirement{ + {Key: "tier", Operator: "NotIn", Values: []string{"debug"}}, + {Key: "app", Operator: "Exists"}, + }, + }, + }, + expectErr: false, + checkResult: func(t *testing.T, r *ResolvedResourceFilter) { + t.Helper() + require.NotNil(t, r.LabelSelector) + assert.True(t, r.LabelSelector.Matches(labels.Set{"app": "web", "tier": "frontend"})) + assert.False(t, r.LabelSelector.Matches(labels.Set{"app": "web", "tier": "debug"})) + assert.False(t, r.LabelSelector.Matches(labels.Set{"tier": "frontend"})) + }, + }, + { + name: "invalid operator", + rf: resourcepolicies.ResourceFilter{ + LabelSelector: &resourcepolicies.PolicyLabelSelector{ + MatchExpressions: []resourcepolicies.PolicyLabelSelectorRequirement{ + {Key: "env", Operator: "Equals", Values: []string{"prod"}}, + }, + }, + }, + expectErr: true, + }, } for _, tc := range tests { @@ -5834,11 +5971,11 @@ func TestResolveClusterScopedFilterPolicy(t *testing.T) { ResourceFilters: []resourcepolicies.ResourceFilter{ { Kinds: []string{"pods", "secrets"}, - LabelSelector: map[string]string{"app": "foo"}, + LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"app": "foo"}}, }, { Kinds: []string{"invalid-kind"}, - LabelSelector: map[string]string{"invalid/label/key": "value"}, + LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"invalid/label/key": "value"}}, }, }, } @@ -5852,7 +5989,7 @@ func TestResolveClusterScopedFilterPolicy(t *testing.T) { ResourceFilters: []resourcepolicies.ResourceFilter{ { Kinds: []string{"pods", "secrets"}, - LabelSelector: map[string]string{"app": "foo"}, + LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"app": "foo"}}, }, }, } @@ -5900,11 +6037,11 @@ func TestResolveNamespacedFilterPolicies(t *testing.T) { ResourceFilters: []resourcepolicies.ResourceFilter{ { Kinds: []string{"pods"}, - LabelSelector: map[string]string{"app": "foo"}, + LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"app": "foo"}}, }, { Kinds: []string{"*"}, - LabelSelector: map[string]string{"catch": "all"}, + LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"catch": "all"}}, }, }, }, @@ -5932,7 +6069,7 @@ func TestResolveNamespacedFilterPolicies(t *testing.T) { ResourceFilters: []resourcepolicies.ResourceFilter{ { Kinds: []string{"pods"}, - LabelSelector: map[string]string{"invalid/label/key": "value"}, + LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"invalid/label/key": "value"}}, }, }, }, @@ -6016,7 +6153,7 @@ func TestBackupWithResPoliciesLogs(t *testing.T) { ResourceFilters: []resourcepolicies.ResourceFilter{ { Kinds: []string{"pods"}, - LabelSelector: map[string]string{"invalid/label/key": "value"}, + LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"invalid/label/key": "value"}}, }, }, } @@ -6035,7 +6172,7 @@ func TestBackupWithResPoliciesLogs(t *testing.T) { ResourceFilters: []resourcepolicies.ResourceFilter{ { Kinds: []string{"pods"}, - LabelSelector: map[string]string{"invalid/label/key": "value"}, + LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"invalid/label/key": "value"}}, }, }, }, diff --git a/pkg/backup/item_backupper.go b/pkg/backup/item_backupper.go index f43888252..16ba0fe9b 100644 --- a/pkg/backup/item_backupper.go +++ b/pkg/backup/item_backupper.go @@ -351,16 +351,28 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti if versionPath == preferredGVR.Version { // backing up preferred version backup without API Group version - for backward compatibility log.Debugf("Resource %s/%s, version= %s, preferredVersion=%s", groupResource.String(), name, versionPath, preferredGVR.Version) - itemFiles = append(itemFiles, getFileForArchive(namespace, name, groupResource.String(), "", itemBytes)) + fileForArchive, err := getFileForArchive(namespace, name, groupResource.String(), "", itemBytes) + if err != nil { + return false, itemFiles, err + } + itemFiles = append(itemFiles, fileForArchive) versionPath = versionPath + velerov1api.PreferredVersionDir } - itemFiles = append(itemFiles, getFileForArchive(namespace, name, groupResource.String(), versionPath, itemBytes)) + fileForArchive, err := getFileForArchive(namespace, name, groupResource.String(), versionPath, itemBytes) + if err != nil { + return false, itemFiles, err + } + itemFiles = append(itemFiles, fileForArchive) return true, itemFiles, nil } -func getFileForArchive(namespace, name, groupResource, versionPath string, itemBytes []byte) FileForArchive { - filePath := archive.GetVersionedItemFilePath("", groupResource, namespace, name, versionPath) +func getFileForArchive(namespace, name, groupResource, versionPath string, itemBytes []byte) (FileForArchive, error) { + filePath, err := archive.GetVersionedItemFilePath("", groupResource, namespace, name, versionPath) + if err != nil { + return FileForArchive{}, err + } + hdr := &tar.Header{ Name: filePath, Size: int64(len(itemBytes)), @@ -368,7 +380,7 @@ func getFileForArchive(namespace, name, groupResource, versionPath string, itemB Mode: 0755, ModTime: time.Now(), } - return FileForArchive{FilePath: filePath, Header: hdr, FileBytes: itemBytes} + return FileForArchive{FilePath: filePath, Header: hdr, FileBytes: itemBytes}, nil } // backupPodVolumes triggers pod volume backups of the specified pod volumes, and returns a list of PodVolumeBackups @@ -557,9 +569,9 @@ func (ib *itemBackupper) executeActions( // zoneLabel is the label that stores availability-zone info // on PVs const ( - zoneLabelDeprecated = "failure-domain.beta.kubernetes.io/zone" + zoneLabelDeprecated = corev1api.LabelFailureDomainBetaZone // this is reused for nodeAffinity requirements - zoneLabel = "topology.kubernetes.io/zone" + zoneLabel = corev1api.LabelTopologyZone awsEbsCsiZoneKey = "topology.ebs.csi.aws.com/zone" azureCsiZoneKey = "topology.disk.csi.azure.com/zone" diff --git a/pkg/backup/item_collector.go b/pkg/backup/item_collector.go index f4c712921..1733e9ac9 100644 --- a/pkg/backup/item_collector.go +++ b/pkg/backup/item_collector.go @@ -346,7 +346,15 @@ func getOrderedResourcesForType( if !ok || len(orderStr) == 0 { return nil } - orders := strings.Split(orderStr, ",") + parts := strings.Split(orderStr, ",") + orders := make([]string, 0, len(parts)) + for _, part := range parts { + name := strings.TrimSpace(part) + if name == "" { + continue + } + orders = append(orders, name) + } return orders } @@ -508,7 +516,8 @@ func (r *itemCollector) getResourceItems( kind: resource.Kind, }) - if item.GetNamespace() != "" { + if item.GetNamespace() != "" && + r.backupRequest.NamespaceIncludesExcludes.ShouldInclude(item.GetNamespace()) { log.Debugf("Track namespace %s in nsTracker", item.GetNamespace()) r.nsTracker.track(item.GetNamespace()) } diff --git a/pkg/backup/item_collector_test.go b/pkg/backup/item_collector_test.go index 084d5b5ff..47a1d7be5 100644 --- a/pkg/backup/item_collector_test.go +++ b/pkg/backup/item_collector_test.go @@ -445,3 +445,24 @@ func TestGetResourceItems(t *testing.T) { }) } } + +func TestGetOrderedResourcesForTypeTrimsSpaces(t *testing.T) { + // Spaces after commas are common in CLI input and should not break ordering. + orders := getOrderedResourcesForType(map[string]string{ + "pods": "ns1/pod2, ns1/pod1", + }, "pods") + require.Equal(t, []string{"ns1/pod2", "ns1/pod1"}, orders) + + log := logrus.StandardLogger() + podResources := []*kubernetesResource{ + {namespace: "ns1", name: "pod3"}, + {namespace: "ns1", name: "pod1"}, + {namespace: "ns1", name: "pod2"}, + } + sorted := sortResourcesByOrder(log, podResources, orders) + require.Equal(t, []*kubernetesResource{ + {namespace: "ns1", name: "pod2", orderedResource: true}, + {namespace: "ns1", name: "pod1", orderedResource: true}, + {namespace: "ns1", name: "pod3"}, + }, sorted) +} diff --git a/pkg/backup/snapshots_test.go b/pkg/backup/snapshots_test.go new file mode 100644 index 000000000..e3ad65833 --- /dev/null +++ b/pkg/backup/snapshots_test.go @@ -0,0 +1,241 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package backup + +import ( + "testing" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/features" + velerotest "github.com/vmware-tanzu/velero/pkg/test" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" +) + +func TestGetBackupCSIResources(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, snapshotv1api.AddToScheme(scheme)) + require.NoError(t, velerov1api.AddToScheme(scheme)) + + tests := []struct { + name string + backup *velerov1api.Backup + csiFeatureEnabled bool + existingObjects []kbclient.Object + wantSnapshots int + wantSnapshotContents int + wantSnapshotClasses int + }{ + { + name: "SnapshotMoveData is true, skip CSI resources", + backup: &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "test-backup"}, + Spec: velerov1api.BackupSpec{ + SnapshotMoveData: boolptr.True(), + }, + }, + csiFeatureEnabled: true, + existingObjects: []kbclient.Object{ + &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Namespace: "ns-1", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, + }, + &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-1", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + VolumeSnapshotClassName: func(s string) *string { return &s }("vsc-class-1"), + }, + }, + &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-class-1", + }, + }, + }, + wantSnapshots: 0, + wantSnapshotContents: 0, + wantSnapshotClasses: 0, + }, + { + name: "CSIFeatureFlag is false, skip CSI resources", + backup: &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "test-backup"}, + Spec: velerov1api.BackupSpec{ + SnapshotMoveData: boolptr.False(), + }, + }, + csiFeatureEnabled: false, + existingObjects: []kbclient.Object{ + &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Namespace: "ns-1", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, + }, + &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-1", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + VolumeSnapshotClassName: func(s string) *string { return &s }("vsc-class-1"), + }, + }, + &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-class-1", + }, + }, + }, + wantSnapshots: 0, + wantSnapshotContents: 0, + wantSnapshotClasses: 0, + }, + { + name: "CSIFeatureFlag enabled, retrieve CSI resources", + backup: &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "test-backup"}, + Spec: velerov1api.BackupSpec{ + SnapshotMoveData: boolptr.False(), + }, + }, + csiFeatureEnabled: true, + existingObjects: []kbclient.Object{ + &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Namespace: "ns-1", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, + }, + &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-1", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + VolumeSnapshotClassName: func(s string) *string { return &s }("vsc-class-1"), + }, + }, + &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-class-1", + }, + }, + }, + wantSnapshots: 1, + wantSnapshotContents: 1, + wantSnapshotClasses: 1, + }, + { + name: "CSIFeatureFlag enabled, multiple contents referencing same class", + backup: &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "test-backup"}, + Spec: velerov1api.BackupSpec{ + SnapshotMoveData: boolptr.False(), + }, + }, + csiFeatureEnabled: true, + existingObjects: []kbclient.Object{ + &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-1", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + VolumeSnapshotClassName: func(s string) *string { return &s }("vsc-class-1"), + }, + }, + &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-2", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test-backup", + }, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + VolumeSnapshotClassName: func(s string) *string { return &s }("vsc-class-1"), + }, + }, + &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-class-1", + }, + }, + }, + wantSnapshots: 0, + wantSnapshotContents: 2, + wantSnapshotClasses: 1, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + defer features.NewFeatureFlagSet() + + if tc.csiFeatureEnabled { + features.Enable(velerov1api.CSIFeatureFlag) + } else { + features.Disable(velerov1api.CSIFeatureFlag) + } + + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tc.existingObjects...).Build() + logger := velerotest.NewLogger() + + snaps, contents, classes := GetBackupCSIResources(client, client, tc.backup, logger) + + assert.Len(t, snaps, tc.wantSnapshots) + assert.Len(t, contents, tc.wantSnapshotContents) + assert.Len(t, classes, tc.wantSnapshotClasses) + + // If we expect CSI resources to be pulled, ensure the attempts count was updated on the backup object + if tc.csiFeatureEnabled && !boolptr.IsSetToTrue(tc.backup.Spec.SnapshotMoveData) { + assert.Equal(t, tc.wantSnapshots, tc.backup.Status.CSIVolumeSnapshotsAttempted) + } else { + assert.Equal(t, 0, tc.backup.Status.CSIVolumeSnapshotsAttempted) + } + }) + } +} diff --git a/pkg/builder/backup_builder.go b/pkg/builder/backup_builder.go index 0553116a4..056f198c2 100644 --- a/pkg/builder/backup_builder.go +++ b/pkg/builder/backup_builder.go @@ -109,8 +109,23 @@ func (b *BackupBuilder) FromSchedule(schedule *velerov1api.Schedule) *BackupBuil b.object.Spec = schedule.Spec.Template b.ObjectMeta(WithLabelsMap(labels)) - if schedule.Annotations != nil { - b.ObjectMeta(WithAnnotationsMap(schedule.Annotations)) + var annotations map[string]string + + // Check if there's explicit Annotations defined in the Schedule object template + // and if present then copy it to the backup object. + if schedule.Spec.Template.Metadata.Annotations != nil { + logger := logging.DefaultLogger(logging.LogLevelFlag(logrus.InfoLevel).Parse(), logging.NewFormatFlag().Parse()) + annotations = schedule.Spec.Template.Metadata.Annotations + logger.WithFields(logrus.Fields{ + "backup": fmt.Sprintf("%s/%s", b.object.GetNamespace(), b.object.GetName()), + "annotations": schedule.Spec.Template.Metadata.Annotations, + }).Info("Schedule.template.metadata.annotations set - using those annotations instead of schedule.annotations for backup object") + } else { + annotations = schedule.Annotations + } + + if annotations != nil { + b.ObjectMeta(WithAnnotationsMap(annotations)) } if boolptr.IsSetToTrue(schedule.Spec.UseOwnerReferencesInBackup) { diff --git a/pkg/builder/backup_builder_test.go b/pkg/builder/backup_builder_test.go new file mode 100644 index 000000000..c7f3ef000 --- /dev/null +++ b/pkg/builder/backup_builder_test.go @@ -0,0 +1,84 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" +) + +func TestBackupFromSchedule(t *testing.T) { + tests := []struct { + name string + schedule *velerov1api.Schedule + expectedLabels map[string]string + expectedAnnotations map[string]string + }{ + { + name: "no schedule labels/annotations and no template overrides", + schedule: ForSchedule("velero", "test"). + Result(), + expectedLabels: map[string]string{velerov1api.ScheduleNameLabel: "test"}, + expectedAnnotations: nil, + }, + { + name: "schedule labels/annotations are copied when no template override is set", + schedule: ForSchedule("velero", "test"). + ObjectMeta( + WithLabels("schedule-label", "schedule-value"), + WithAnnotations("schedule-annotation", "schedule-value"), + ). + Result(), + expectedLabels: map[string]string{ + "schedule-label": "schedule-value", + velerov1api.ScheduleNameLabel: "test", + }, + expectedAnnotations: map[string]string{"schedule-annotation": "schedule-value"}, + }, + { + name: "template.metadata.labels/annotations override schedule labels/annotations", + schedule: ForSchedule("velero", "test"). + ObjectMeta( + WithLabels("schedule-label", "schedule-value"), + WithAnnotations("schedule-annotation", "schedule-value"), + ). + Template(velerov1api.BackupSpec{ + Metadata: velerov1api.Metadata{ + Labels: map[string]string{"template-label": "template-value"}, + Annotations: map[string]string{"template-annotation": "template-value"}, + }, + }). + Result(), + expectedLabels: map[string]string{ + "template-label": "template-value", + velerov1api.ScheduleNameLabel: "test", + }, + expectedAnnotations: map[string]string{"template-annotation": "template-value"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + backup := ForBackup("velero", "test-backup").FromSchedule(test.schedule).Result() + assert.Equal(t, test.expectedLabels, backup.GetLabels()) + assert.Equal(t, test.expectedAnnotations, backup.GetAnnotations()) + }) + } +} diff --git a/pkg/builder/container_builder.go b/pkg/builder/container_builder.go index 762462c86..e25002629 100644 --- a/pkg/builder/container_builder.go +++ b/pkg/builder/container_builder.go @@ -18,10 +18,12 @@ package builder import ( "encoding/json" + "fmt" "strings" corev1api "k8s.io/api/core/v1" apimachineryRuntime "k8s.io/apimachinery/pkg/runtime" + utilrand "k8s.io/apimachinery/pkg/util/rand" "github.com/vmware-tanzu/velero/pkg/label" ) @@ -42,15 +44,22 @@ func ForContainer(name, image string) *ContainerBuilder { } // ForPluginContainer is a helper builder specifically for plugin init containers -func ForPluginContainer(image string, pullPolicy corev1api.PullPolicy) *ContainerBuilder { +func ForPluginContainer(image string, pullPolicy corev1api.PullPolicy, existingContainers []corev1api.Container) *ContainerBuilder { volumeMount := ForVolumeMount("plugins", "/target").Result() - return ForContainer(getName(image), image).PullPolicy(pullPolicy).VolumeMounts(volumeMount) + return ForContainer(getName(image, existingContainers), image).PullPolicy(pullPolicy).VolumeMounts(volumeMount) } // getName returns the 'name' component of a docker image that includes the entire string // except the registry name, and transforms the combined string into a DNS-1123 compatible name // that fits within the 63-character limit for Kubernetes container names. -func getName(image string) string { +// It appends a random string if there is a collision with existing container names. +func getName(image string, existingContainers []corev1api.Container) string { + // Convert existingContainers to a map for O(1) collision lookups + existingNames := make(map[string]bool, len(existingContainers)) + for _, c := range existingContainers { + existingNames[c.Name] = true + } + slashIndex := strings.Index(image, "/") slashCount := 0 if slashIndex >= 0 { @@ -88,7 +97,20 @@ func getName(image string) string { name := re.Replace(image[start:end]) // Ensure the name doesn't exceed Kubernetes container name length limit - return label.GetValidName(name) + name = label.GetValidName(name) + + for existingNames[name] { + name = re.Replace(image[start:end]) + if len(name) > 57 { + // Leave 6 characters for "-xxxxx" random string + name = name[:57] + name = strings.TrimSuffix(name, "-") + } + name = fmt.Sprintf("%s-%s", name, utilrand.String(5)) + name = label.GetValidName(name) + } + + return name } // Result returns the built Container. diff --git a/pkg/builder/container_builder_test.go b/pkg/builder/container_builder_test.go index b23cbddfd..e0af71f75 100644 --- a/pkg/builder/container_builder_test.go +++ b/pkg/builder/container_builder_test.go @@ -16,16 +16,19 @@ limitations under the License. package builder import ( + "strings" "testing" "github.com/stretchr/testify/assert" + corev1api "k8s.io/api/core/v1" ) func TestGetName(t *testing.T) { tests := []struct { - name string - image string - expected string + name string + image string + existingContainers []corev1api.Container + expected string }{ { name: "image name with registry hostname and tag", @@ -92,11 +95,25 @@ func TestGetName(t *testing.T) { image: "quay.io/vmware-tanzu/velero@sha256:a75f9e8c3ced3943515f249597be389f8233e1258d289b11184796edceaa7dab", expected: "vmware-tanzu-velero", }, + { + name: "duplicate plugin name", + image: "gcr.io/my-repo/my-image:latest", + existingContainers: []corev1api.Container{ + {Name: "my-repo-my-image"}, + }, + expected: "my-repo-my-image-", // we will check it has the prefix + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.expected, getName(test.image)) + if test.name == "duplicate plugin name" { + result := getName(test.image, test.existingContainers) + assert.True(t, strings.HasPrefix(result, test.expected), "expected prefix %s in %s", test.expected, result) + assert.Len(t, result, len(test.expected)+5) + } else { + assert.Equal(t, test.expected, getName(test.image, test.existingContainers)) + } }) } } @@ -117,7 +134,7 @@ func TestGetNameWithLongPaths(t *testing.T) { // Should be exactly 63 characters (truncated with hash) assert.Len(t, result, 63) // Should be deterministic - result2 := getName("arohcpsvcdev.azurecr.io/redhat-user-workloads/ocp-art-tenant/oadp-hypershift-oadp-plugin-main@sha256:adb840bf3890b4904a8cdda1a74c82cf8d96c52eba9944ac10e795335d6fd450") + result2 := getName("arohcpsvcdev.azurecr.io/redhat-user-workloads/ocp-art-tenant/oadp-hypershift-oadp-plugin-main@sha256:adb840bf3890b4904a8cdda1a74c82cf8d96c52eba9944ac10e795335d6fd450", nil) assert.Equal(t, result, result2) }, }, @@ -142,7 +159,7 @@ func TestGetNameWithLongPaths(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - result := getName(test.image) + result := getName(test.image, nil) test.validate(t, result) }) } diff --git a/pkg/builder/data_upload_builder.go b/pkg/builder/data_upload_builder.go index c8fa34956..9805e71a3 100644 --- a/pkg/builder/data_upload_builder.go +++ b/pkg/builder/data_upload_builder.go @@ -147,7 +147,7 @@ func (d *DataUploadBuilder) Progress(progress shared.DataMoveOperationProgress) // IncrementalBytes sets the DataUpload's IncrementalBytes. func (d *DataUploadBuilder) IncrementalBytes(incrementalBytes int64) *DataUploadBuilder { - d.object.Status.IncrementalBytes = incrementalBytes + d.object.Status.IncrementalBytes = &incrementalBytes return d } diff --git a/pkg/builder/restore_builder.go b/pkg/builder/restore_builder.go index 22e880a98..5ef993617 100644 --- a/pkg/builder/restore_builder.go +++ b/pkg/builder/restore_builder.go @@ -98,7 +98,13 @@ func (b *RestoreBuilder) ExcludedResources(resources ...string) *RestoreBuilder // ExistingResourcePolicy sets the Restore's resource policy. func (b *RestoreBuilder) ExistingResourcePolicy(policy string) *RestoreBuilder { - b.object.Spec.ExistingResourcePolicy = velerov1api.PolicyType(policy) + b.object.Spec.ExistingResourcePolicy = velerov1api.ResourcePolicyType(policy) + return b +} + +// ExistingVolumeDataPolicy sets the Restore's volume data policy. +func (b *RestoreBuilder) ExistingVolumeDataPolicy(policy string) *RestoreBuilder { + b.object.Spec.ExistingVolumeDataPolicy = velerov1api.VolumeDataPolicyType(policy) return b } @@ -181,3 +187,9 @@ func (b *RestoreBuilder) ResourcePoliciesConfigmap(name string) *RestoreBuilder } return b } + +// SkipDefaultResourceModifier sets whether to skip the server default resource modifier. +func (b *RestoreBuilder) SkipDefaultResourceModifier(val bool) *RestoreBuilder { + b.object.Spec.SkipDefaultResourceModifier = &val + return b +} diff --git a/pkg/cbtservice/csi_service_impl.go b/pkg/cbtservice/csi_service_impl.go index 4d0ea3fca..235477bd4 100644 --- a/pkg/cbtservice/csi_service_impl.go +++ b/pkg/cbtservice/csi_service_impl.go @@ -86,6 +86,12 @@ func (s *ServiceImpl) GetAllocatedBlocks(ctx context.Context, snapshot string, r return err } + saNamespace := "" + if s.SAName != "" { + // The SA is created in the same namespace as Velero server. vsNamespace is the namespace of Velero server. + saNamespace = s.vsNamespace + } + args := iterator.Args{ SnapshotName: snapshot, Emitter: &emitterImpl{ @@ -95,7 +101,7 @@ func (s *ServiceImpl) GetAllocatedBlocks(ctx context.Context, snapshot string, r Clients: clients, Namespace: s.vsNamespace, // DataUpload is created in the same namespace as Velero server. vsNamespace is the namespace of the Velero server. - SANamespace: s.vsNamespace, // The SA is created in the same namespace as Velero server. vsNamespace is the namespace of Velero server. + SANamespace: saNamespace, SAName: s.SAName, TokenExpirySecs: iterator.DefaultTokenExpirySeconds, MaxResults: 0, // If 0 then the CSI driver decides the value. @@ -110,6 +116,12 @@ func (s *ServiceImpl) GetChangedBlocks(ctx context.Context, snapshot string, cha return err } + saNamespace := "" + if s.SAName != "" { + // The SA is created in the same namespace as Velero server. vsNamespace is the namespace of Velero server. + saNamespace = s.vsNamespace + } + args := iterator.Args{ SnapshotName: snapshot, PrevSnapshotID: changeID, @@ -120,7 +132,7 @@ func (s *ServiceImpl) GetChangedBlocks(ctx context.Context, snapshot string, cha Clients: clients, Namespace: s.vsNamespace, - SANamespace: s.vsNamespace, + SANamespace: saNamespace, SAName: s.SAName, TokenExpirySecs: iterator.DefaultTokenExpirySeconds, MaxResults: 0, // If 0 then the CSI driver decides the value. diff --git a/pkg/client/client.go b/pkg/client/client.go index 39cdc9141..e49fbd0cc 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -58,6 +58,23 @@ func Config(kubeconfig, kubecontext, baseName string, qps float32, burst int) (* return clientConfig, nil } +// NamespaceFromKubeContext returns the namespace associated with the given kubeconfig context +// (or the current context if kubecontext is empty), using the given kubeconfig file (or the +// default loading rules if kubeconfig is empty). +func NamespaceFromKubeContext(kubeconfig, kubecontext string) (string, error) { + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + loadingRules.ExplicitPath = kubeconfig + configOverrides := &clientcmd.ConfigOverrides{CurrentContext: kubecontext} + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + + namespace, _, err := kubeConfig.Namespace() + if err != nil { + return "", errors.Wrap(err, "error finding namespace in --kubeconfig, $KUBECONFIG, or in-cluster configuration") + } + + return namespace, nil +} + // buildUserAgent builds a User-Agent string from given args. func buildUserAgent(command, version, formattedSha, os, arch string) string { return fmt.Sprintf( diff --git a/pkg/client/config.go b/pkg/client/config.go index 2a96e3467..793c4b9a0 100644 --- a/pkg/client/config.go +++ b/pkg/client/config.go @@ -27,10 +27,16 @@ import ( ) const ( - ConfigKeyNamespace = "namespace" - ConfigKeyFeatures = "features" - ConfigKeyCACert = "cacert" - ConfigKeyColorized = "colorized" + ConfigKeyNamespace = "namespace" + ConfigKeyNamespaceMode = "namespace-mode" + ConfigKeyFeatures = "features" + ConfigKeyCACert = "cacert" + ConfigKeyColorized = "colorized" + + // NamespaceModeAuto is the ConfigKeyNamespaceMode value that makes Velero resolve the + // namespace for operational commands from the current kubeconfig context on every + // invocation, instead of the static ConfigKeyNamespace value. + NamespaceModeAuto = "auto" ) // VeleroConfig is a map of strings to any for deserializing Velero client config options. @@ -99,6 +105,20 @@ func (c VeleroConfig) Namespace() string { return ns } +func (c VeleroConfig) NamespaceMode() string { + val, ok := c[ConfigKeyNamespaceMode] + if !ok { + return "" + } + + mode, ok := val.(string) + if !ok { + return "" + } + + return mode +} + func (c VeleroConfig) Features() []string { val, ok := c[ConfigKeyFeatures] if !ok { diff --git a/pkg/client/factory.go b/pkg/client/factory.go index 17e2a243a..01df4ed7b 100644 --- a/pkg/client/factory.go +++ b/pkg/client/factory.go @@ -77,20 +77,22 @@ type Factory interface { } type factory struct { - flags *pflag.FlagSet - kubeconfig string - kubecontext string - baseName string - namespace string - clientQPS float32 - clientBurst int + flags *pflag.FlagSet + kubeconfig string + kubecontext string + baseName string + namespace string + namespaceMode string + clientQPS float32 + clientBurst int } // NewFactory returns a Factory. func NewFactory(baseName string, config VeleroConfig) Factory { f := &factory{ - flags: pflag.NewFlagSet("", pflag.ContinueOnError), - baseName: baseName, + flags: pflag.NewFlagSet("", pflag.ContinueOnError), + baseName: baseName, + namespaceMode: config.NamespaceMode(), } f.namespace = os.Getenv("VELERO_NAMESPACE") @@ -242,5 +244,15 @@ func (f *factory) SetClientBurst(burst int) { } func (f *factory) Namespace() string { + // In auto mode, the namespace is resolved from the current kubeconfig context on every + // call, unless the caller explicitly overrode it with --namespace or VELERO_NAMESPACE. + if f.namespaceMode == NamespaceModeAuto && + !f.flags.Changed("namespace") && + os.Getenv("VELERO_NAMESPACE") == "" { + if namespace, err := NamespaceFromKubeContext(f.kubeconfig, f.kubecontext); err == nil && namespace != "" { + return namespace + } + } + return f.namespace } diff --git a/pkg/client/factory_test.go b/pkg/client/factory_test.go index 5b9db37f1..2f63d3a3c 100644 --- a/pkg/client/factory_test.go +++ b/pkg/client/factory_test.go @@ -64,6 +64,45 @@ func TestFactory(t *testing.T) { os.Unsetenv("VELERO_NAMESPACE") + // namespace-mode=auto should resolve the namespace from the current kubeconfig context. + f = NewFactory("velero", VeleroConfig{ConfigKeyNamespaceMode: NamespaceModeAuto}) + flags = new(flag.FlagSet) + f.BindFlags(flags) + require.NoError(t, flags.Parse([]string{"--kubeconfig", "kubeconfig", "--kubecontext", "federal-context"})) + assert.Equal(t, "chisel-ns", f.Namespace()) + + // namespace-mode=auto should track kubecontext changes dynamically. + f = NewFactory("velero", VeleroConfig{ConfigKeyNamespaceMode: NamespaceModeAuto}) + flags = new(flag.FlagSet) + f.BindFlags(flags) + require.NoError(t, flags.Parse([]string{"--kubeconfig", "kubeconfig", "--kubecontext", "queen-anne-context"})) + assert.Equal(t, "saw-ns", f.Namespace()) + + // An explicit --namespace flag overrides namespace-mode=auto. + f = NewFactory("velero", VeleroConfig{ConfigKeyNamespaceMode: NamespaceModeAuto}) + flags = new(flag.FlagSet) + f.BindFlags(flags) + require.NoError(t, flags.Parse([]string{"--kubeconfig", "kubeconfig", "--kubecontext", "federal-context", "--namespace", s})) + assert.Equal(t, s, f.Namespace()) + + // VELERO_NAMESPACE overrides namespace-mode=auto. + t.Run("VELERO_NAMESPACE overrides namespace-mode=auto", func(t *testing.T) { + t.Setenv("VELERO_NAMESPACE", "env-velero") + f := NewFactory("velero", VeleroConfig{ConfigKeyNamespaceMode: NamespaceModeAuto}) + flags := new(flag.FlagSet) + f.BindFlags(flags) + require.NoError(t, flags.Parse([]string{"--kubeconfig", "kubeconfig", "--kubecontext", "federal-context"})) + assert.Equal(t, "env-velero", f.Namespace()) + }) + + // namespace-mode=auto falls back to the stored/default namespace when the kubeconfig + // namespace can't be resolved (e.g. the kubeconfig file doesn't exist). + f = NewFactory("velero", VeleroConfig{ConfigKeyNamespace: "stored-ns", ConfigKeyNamespaceMode: NamespaceModeAuto}) + flags = new(flag.FlagSet) + f.BindFlags(flags) + require.NoError(t, flags.Parse([]string{"--kubeconfig", "nonexistent-kubeconfig"})) + assert.Equal(t, "stored-ns", f.Namespace()) + tests := []struct { name string kubeconfig string diff --git a/pkg/cmd/cli/backup/create.go b/pkg/cmd/cli/backup/create.go index 5e18f468f..8cdec3d99 100644 --- a/pkg/cmd/cli/backup/create.go +++ b/pkg/cmd/cli/backup/create.go @@ -32,6 +32,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" "github.com/vmware-tanzu/velero/pkg/util/collections" @@ -75,6 +76,10 @@ func NewCreateCommand(f client.Factory, use string) *cobra.Command { output.BindFlags(c.Flags()) output.ClearOutputFlagDefault(c) + _ = c.RegisterFlagCompletionFunc("from-schedule", cli.CompleteScheduleNames(f)) + _ = c.RegisterFlagCompletionFunc("storage-location", cli.CompleteBackupStorageLocationNames(f)) + _ = c.RegisterFlagCompletionFunc("volume-snapshot-locations", cli.CompleteVolumeSnapshotLocationNames(f)) + return c } @@ -237,11 +242,17 @@ func (o *CreateOptions) validateFromScheduleFlag(c *cobra.Command) error { return nil } +// validateBackupType check the backupType value and return the valid value. func (o *CreateOptions) validateBackupType() error { - backupType := strings.TrimSpace(o.BackupType) + // Allow full, and incremental from the CLI, and ignore case of the input string's case. + backupType := strings.ToLower(strings.TrimSpace(o.BackupType)) switch backupType { - case "", "Incremental", "Full": + case "": + case "incremental": + o.BackupType = string(velerov1api.BackupTypeIncremental) + case "full": + o.BackupType = string(velerov1api.BackupTypeFull) default: return fmt.Errorf("invalid backup type %s - valid values are 'Incremental', and 'Full'", backupType) } @@ -374,8 +385,19 @@ func ParseOrderedResources(orderMapStr string) (map[string]string, error) { return nil, fmt.Errorf("invalid OrderedResources '%s'", entry) } kind := strings.TrimSpace(kv[0]) - order := strings.TrimSpace(kv[1]) - orderedResources[kind] = order + orderParts := strings.Split(kv[1], ",") + cleaned := make([]string, 0, len(orderParts)) + for _, part := range orderParts { + name := strings.TrimSpace(part) + if name == "" { + continue + } + cleaned = append(cleaned, name) + } + if kind == "" || len(cleaned) == 0 { + return nil, fmt.Errorf("invalid OrderedResources '%s'", entry) + } + orderedResources[kind] = strings.Join(cleaned, ",") } return orderedResources, nil } diff --git a/pkg/cmd/cli/backup/create_test.go b/pkg/cmd/cli/backup/create_test.go index 718ab0e96..07d5bb493 100644 --- a/pkg/cmd/cli/backup/create_test.go +++ b/pkg/cmd/cli/backup/create_test.go @@ -129,30 +129,34 @@ func TestCreateOptions_ValidateBackupType(t *testing.T) { o.BackupType = "" err := o.validateBackupType() require.NoError(t, err) + require.Empty(t, o.BackupType) o.BackupType = "Incremental" err = o.validateBackupType() require.NoError(t, err) + require.EqualValues(t, velerov1api.BackupTypeIncremental, o.BackupType) o.BackupType = "Full" err = o.validateBackupType() require.NoError(t, err) + require.EqualValues(t, velerov1api.BackupTypeFull, o.BackupType) o.BackupType = " Incremental " err = o.validateBackupType() require.NoError(t, err) + require.EqualValues(t, velerov1api.BackupTypeIncremental, o.BackupType) + + o.BackupType = "iNcReMeNtAl" + err = o.validateBackupType() + require.NoError(t, err) + require.EqualValues(t, velerov1api.BackupTypeIncremental, o.BackupType) }) t.Run("invalid backup type", func(t *testing.T) { o := NewCreateOptions() - o.BackupType = "incremental" - err := o.validateBackupType() - require.Error(t, err) - require.Equal(t, "invalid backup type incremental - valid values are 'Incremental', and 'Full'", err.Error()) - o.BackupType = "invalid" - err = o.validateBackupType() + err := o.validateBackupType() require.Error(t, err) require.Equal(t, "invalid backup type invalid - valid values are 'Incremental', and 'Full'", err.Error()) }) @@ -230,6 +234,14 @@ func TestCreateOptions_OrderedResources(t *testing.T) { "persistentvolumes": "pv1,pv2", } assert.Equal(t, expectedMixedResources, orderedResources) + + // Spaces after commas in the resource list must be trimmed. + orderedResources, err = ParseOrderedResources("pods=ns1/p1, ns1/p2 ; persistentvolumeclaims= ns2/pvc1, ns2/pvc2") + require.NoError(t, err) + assert.Equal(t, map[string]string{ + "pods": "ns1/p1,ns1/p2", + "persistentvolumeclaims": "ns2/pvc1,ns2/pvc2", + }, orderedResources) } func TestCreateCommand(t *testing.T) { diff --git a/pkg/cmd/cli/backup/delete.go b/pkg/cmd/cli/backup/delete.go index f4eaf1b83..ba5a4954b 100644 --- a/pkg/cmd/cli/backup/delete.go +++ b/pkg/cmd/cli/backup/delete.go @@ -64,6 +64,7 @@ func NewDeleteCommand(f client.Factory, use string) *cobra.Command { }, } + c.ValidArgsFunction = cli.CompleteBackupNames(f) o.BindFlags(c.Flags()) return c diff --git a/pkg/cmd/cli/backup/describe.go b/pkg/cmd/cli/backup/describe.go index b0ef4a93e..dd819edd1 100644 --- a/pkg/cmd/cli/backup/describe.go +++ b/pkg/cmd/cli/backup/describe.go @@ -29,6 +29,7 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" "github.com/vmware-tanzu/velero/pkg/label" ) @@ -112,6 +113,7 @@ func NewDescribeCommand(f client.Factory, use string) *cobra.Command { }, } + c.ValidArgsFunction = cli.CompleteBackupNames(f) c.Flags().StringVarP(&listOptions.LabelSelector, "selector", "l", listOptions.LabelSelector, "Only show items matching this label selector.") c.Flags().BoolVar(&details, "details", details, "Display additional detail in the command output.") c.Flags().BoolVar(&insecureSkipTLSVerify, "insecure-skip-tls-verify", insecureSkipTLSVerify, "If true, the object store's TLS certificate will not be checked for validity. This is insecure and susceptible to man-in-the-middle attacks. Not recommended for production.") diff --git a/pkg/cmd/cli/backup/download.go b/pkg/cmd/cli/backup/download.go index e4afd216c..a8d692520 100644 --- a/pkg/cmd/cli/backup/download.go +++ b/pkg/cmd/cli/backup/download.go @@ -31,6 +31,7 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" ) @@ -55,6 +56,7 @@ func NewDownloadCommand(f client.Factory) *cobra.Command { }, } + c.ValidArgsFunction = cli.CompleteBackupNames(f) o.BindFlags(c.Flags()) return c diff --git a/pkg/cmd/cli/backup/get.go b/pkg/cmd/cli/backup/get.go index 159fac30d..1af80399b 100644 --- a/pkg/cmd/cli/backup/get.go +++ b/pkg/cmd/cli/backup/get.go @@ -27,6 +27,7 @@ import ( api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" ) @@ -66,6 +67,7 @@ func NewGetCommand(f client.Factory, use string) *cobra.Command { }, } + c.ValidArgsFunction = cli.CompleteBackupNames(f) c.Flags().StringVarP(&listOptions.LabelSelector, "selector", "l", listOptions.LabelSelector, "Only show items matching this label selector") output.BindFlags(c.Flags()) diff --git a/pkg/cmd/cli/backup/logs.go b/pkg/cmd/cli/backup/logs.go index a0149acf1..6e60c30f1 100644 --- a/pkg/cmd/cli/backup/logs.go +++ b/pkg/cmd/cli/backup/logs.go @@ -30,6 +30,7 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" ) @@ -119,6 +120,7 @@ func NewLogsCommand(f client.Factory) *cobra.Command { }, } + c.ValidArgsFunction = cli.CompleteBackupNames(f) l.BindFlags(c.Flags()) return c diff --git a/pkg/cmd/cli/backuplocation/delete.go b/pkg/cmd/cli/backuplocation/delete.go index 9c1e60507..eabadef97 100644 --- a/pkg/cmd/cli/backuplocation/delete.go +++ b/pkg/cmd/cli/backuplocation/delete.go @@ -62,6 +62,7 @@ func NewDeleteCommand(f client.Factory, use string) *cobra.Command { }, } + c.ValidArgsFunction = cli.CompleteBackupStorageLocationNames(f) o.BindFlags(c.Flags()) return c } diff --git a/pkg/cmd/cli/backuplocation/get.go b/pkg/cmd/cli/backuplocation/get.go index fd7c057c2..964ae5a7e 100644 --- a/pkg/cmd/cli/backuplocation/get.go +++ b/pkg/cmd/cli/backuplocation/get.go @@ -27,6 +27,7 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" ) @@ -89,6 +90,7 @@ func NewGetCommand(f client.Factory, use string) *cobra.Command { }, } + c.ValidArgsFunction = cli.CompleteBackupStorageLocationNames(f) c.Flags().BoolVar(&showDefaultOnly, "default", false, "Displays the current default backup storage location.") c.Flags().StringVarP(&listOptions.LabelSelector, "selector", "l", listOptions.LabelSelector, "Only show items matching this label selector.") diff --git a/pkg/cmd/cli/backuplocation/set.go b/pkg/cmd/cli/backuplocation/set.go index c1b52e536..2024f0761 100644 --- a/pkg/cmd/cli/backuplocation/set.go +++ b/pkg/cmd/cli/backuplocation/set.go @@ -33,6 +33,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" "github.com/vmware-tanzu/velero/pkg/util/boolptr" ) @@ -51,6 +52,7 @@ func NewSetCommand(f client.Factory, use string) *cobra.Command { }, } + c.ValidArgsFunction = cli.CompleteBackupStorageLocationNames(f) o.BindFlags(c.Flags()) return c diff --git a/pkg/cmd/cli/completion_functions.go b/pkg/cmd/cli/completion_functions.go new file mode 100644 index 000000000..c2ef20d04 --- /dev/null +++ b/pkg/cmd/cli/completion_functions.go @@ -0,0 +1,105 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cli + +import ( + "context" + "strings" + "time" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/api/meta" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/client" +) + +// completionFunc is the function signature for cobra's ValidArgsFunction. +type completionFunc = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) + +// completeNames builds a completion function for any Velero list type. +// It extracts resource names via apimachinery's meta helpers. +func completeNames(f client.Factory, list kbclient.ObjectList) completionFunc { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + kbClient, err := f.KubebuilderClient() + if err != nil { + return nil, cobra.ShellCompDirectiveNoFileComp + } + parentCtx := context.Background() + if cmd != nil && cmd.Context() != nil { + parentCtx = cmd.Context() + } + ctx, cancel := context.WithTimeout(parentCtx, 3*time.Second) + defer cancel() + freshObject := list.DeepCopyObject() + freshList, ok := freshObject.(kbclient.ObjectList) + if !ok { + return nil, cobra.ShellCompDirectiveNoFileComp + } + if err := kbClient.List(ctx, freshList, &kbclient.ListOptions{Namespace: f.Namespace()}); err != nil { + return nil, cobra.ShellCompDirectiveNoFileComp + } + items, err := meta.ExtractList(freshList) + if err != nil { + return nil, cobra.ShellCompDirectiveNoFileComp + } + seen := make(map[string]bool, len(args)) + for _, a := range args { + seen[a] = true + } + var filtered []string + for _, item := range items { + accessor, err := meta.Accessor(item) + if err != nil { + continue + } + name := accessor.GetName() + if seen[name] { + continue + } + if strings.HasPrefix(name, toComplete) { + filtered = append(filtered, name) + } + } + return filtered, cobra.ShellCompDirectiveNoFileComp + } +} + +func CompleteBackupNames(f client.Factory) completionFunc { + return completeNames(f, &velerov1api.BackupList{}) +} + +func CompleteRestoreNames(f client.Factory) completionFunc { + return completeNames(f, &velerov1api.RestoreList{}) +} + +func CompleteScheduleNames(f client.Factory) completionFunc { + return completeNames(f, &velerov1api.ScheduleList{}) +} + +func CompleteBackupStorageLocationNames(f client.Factory) completionFunc { + return completeNames(f, &velerov1api.BackupStorageLocationList{}) +} + +func CompleteVolumeSnapshotLocationNames(f client.Factory) completionFunc { + return completeNames(f, &velerov1api.VolumeSnapshotLocationList{}) +} + +func CompleteBackupRepositoryNames(f client.Factory) completionFunc { + return completeNames(f, &velerov1api.BackupRepositoryList{}) +} diff --git a/pkg/cmd/cli/completion_functions_test.go b/pkg/cmd/cli/completion_functions_test.go new file mode 100644 index 000000000..3bc33402d --- /dev/null +++ b/pkg/cmd/cli/completion_functions_test.go @@ -0,0 +1,212 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cli + +import ( + "fmt" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" + velerotest "github.com/vmware-tanzu/velero/pkg/test" +) + +// TestCompleteNames exercises the core completeNames helper with various list +// types, prefix filters, and edge cases (empty cluster, no match). +func TestCompleteNames(t *testing.T) { + tests := []struct { + name string + objects []runtime.Object + list kbclient.ObjectList + args []string + toComplete string + want []string + }{ + { + name: "no resources returns nil", + objects: nil, + list: &velerov1api.BackupList{}, + toComplete: "", + want: nil, + }, + { + name: "returns all matching names", + objects: []runtime.Object{ + &velerov1api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "daily", Namespace: "velero"}}, + &velerov1api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "weekly", Namespace: "velero"}}, + }, + list: &velerov1api.BackupList{}, + toComplete: "", + want: []string{"daily", "weekly"}, + }, + { + name: "filters by prefix", + objects: []runtime.Object{ + &velerov1api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "daily", Namespace: "velero"}}, + &velerov1api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "weekly", Namespace: "velero"}}, + &velerov1api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "daily-full", Namespace: "velero"}}, + }, + list: &velerov1api.BackupList{}, + toComplete: "dai", + want: []string{"daily", "daily-full"}, + }, + { + name: "no prefix match returns nil", + objects: []runtime.Object{ + &velerov1api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "daily", Namespace: "velero"}}, + }, + list: &velerov1api.BackupList{}, + toComplete: "xyz", + want: nil, + }, + { + name: "works with RestoreList", + objects: []runtime.Object{ + &velerov1api.Restore{ObjectMeta: metav1.ObjectMeta{Name: "restore-1", Namespace: "velero"}}, + &velerov1api.Restore{ObjectMeta: metav1.ObjectMeta{Name: "restore-2", Namespace: "velero"}}, + }, + list: &velerov1api.RestoreList{}, + toComplete: "restore-", + want: []string{"restore-1", "restore-2"}, + }, + { + name: "works with ScheduleList", + objects: []runtime.Object{ + &velerov1api.Schedule{ObjectMeta: metav1.ObjectMeta{Name: "nightly", Namespace: "velero"}}, + }, + list: &velerov1api.ScheduleList{}, + toComplete: "", + want: []string{"nightly"}, + }, + { + name: "works with BackupStorageLocationList", + objects: []runtime.Object{ + &velerov1api.BackupStorageLocation{ObjectMeta: metav1.ObjectMeta{Name: "default", Namespace: "velero"}}, + &velerov1api.BackupStorageLocation{ObjectMeta: metav1.ObjectMeta{Name: "secondary", Namespace: "velero"}}, + }, + list: &velerov1api.BackupStorageLocationList{}, + toComplete: "s", + want: []string{"secondary"}, + }, + { + name: "works with VolumeSnapshotLocationList", + objects: []runtime.Object{ + &velerov1api.VolumeSnapshotLocation{ObjectMeta: metav1.ObjectMeta{Name: "aws-snap", Namespace: "velero"}}, + }, + list: &velerov1api.VolumeSnapshotLocationList{}, + toComplete: "", + want: []string{"aws-snap"}, + }, + { + name: "works with BackupRepositoryList", + objects: []runtime.Object{ + &velerov1api.BackupRepository{ObjectMeta: metav1.ObjectMeta{Name: "repo-1", Namespace: "velero"}}, + }, + list: &velerov1api.BackupRepositoryList{}, + toComplete: "", + want: []string{"repo-1"}, + }, + { + name: "excludes already-typed args", + objects: []runtime.Object{ + &velerov1api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "daily", Namespace: "velero"}}, + &velerov1api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "weekly", Namespace: "velero"}}, + &velerov1api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "monthly", Namespace: "velero"}}, + }, + list: &velerov1api.BackupList{}, + args: []string{"daily", "monthly"}, + toComplete: "", + want: []string{"weekly"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + kbClient := velerotest.NewFakeControllerRuntimeClient(t, tc.objects...) + + f := new(factorymocks.Factory) + f.On("KubebuilderClient").Return(kbClient, nil) + f.On("Namespace").Return("velero") + + completionFn := completeNames(f, tc.list) + got, directive := completionFn(&cobra.Command{}, tc.args, tc.toComplete) + + assert.Equal(t, cobra.ShellCompDirectiveNoFileComp, directive) + assert.ElementsMatch(t, tc.want, got) + }) + } +} + +// TestCompleteNames_KubebuilderClientError verifies that a factory error +// (e.g. no kubeconfig) returns nil completions instead of panicking. +func TestCompleteNames_KubebuilderClientError(t *testing.T) { + f := new(factorymocks.Factory) + f.On("KubebuilderClient").Return(nil, fmt.Errorf("connection refused")) + + completionFn := completeNames(f, &velerov1api.BackupList{}) + got, directive := completionFn(&cobra.Command{}, nil, "") + + assert.Equal(t, cobra.ShellCompDirectiveNoFileComp, directive) + assert.Nil(t, got) +} + +// TestCompleteWrappers verifies each exported Complete*Names wrapper returns +// only its own resource type. A single fake client holds one object of every +// type, so each wrapper must filter correctly and not leak other kinds. +func TestCompleteWrappers(t *testing.T) { + objects := []runtime.Object{ + &velerov1api.Backup{ObjectMeta: metav1.ObjectMeta{Name: "b1", Namespace: "velero"}}, + &velerov1api.Restore{ObjectMeta: metav1.ObjectMeta{Name: "r1", Namespace: "velero"}}, + &velerov1api.Schedule{ObjectMeta: metav1.ObjectMeta{Name: "s1", Namespace: "velero"}}, + &velerov1api.BackupStorageLocation{ObjectMeta: metav1.ObjectMeta{Name: "bsl1", Namespace: "velero"}}, + &velerov1api.VolumeSnapshotLocation{ObjectMeta: metav1.ObjectMeta{Name: "vsl1", Namespace: "velero"}}, + &velerov1api.BackupRepository{ObjectMeta: metav1.ObjectMeta{Name: "br1", Namespace: "velero"}}, + } + kbClient := velerotest.NewFakeControllerRuntimeClient(t, objects...) + + f := new(factorymocks.Factory) + f.On("KubebuilderClient").Return(kbClient, nil) + f.On("Namespace").Return("velero") + + tests := []struct { + name string + fn completionFunc + expected []string + }{ + {"CompleteBackupNames", CompleteBackupNames(f), []string{"b1"}}, + {"CompleteRestoreNames", CompleteRestoreNames(f), []string{"r1"}}, + {"CompleteScheduleNames", CompleteScheduleNames(f), []string{"s1"}}, + {"CompleteBackupStorageLocationNames", CompleteBackupStorageLocationNames(f), []string{"bsl1"}}, + {"CompleteVolumeSnapshotLocationNames", CompleteVolumeSnapshotLocationNames(f), []string{"vsl1"}}, + {"CompleteBackupRepositoryNames", CompleteBackupRepositoryNames(f), []string{"br1"}}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, directive := tc.fn(&cobra.Command{}, nil, "") + require.Equal(t, cobra.ShellCompDirectiveNoFileComp, directive) + assert.Equal(t, tc.expected, got) + }) + } +} diff --git a/pkg/cmd/cli/datamover/backup.go b/pkg/cmd/cli/datamover/backup.go index f352c0aad..aa0b2bcfb 100644 --- a/pkg/cmd/cli/datamover/backup.go +++ b/pkg/cmd/cli/datamover/backup.go @@ -15,6 +15,7 @@ package datamover import ( "context" + "crypto/fips140" "fmt" "os" "strings" @@ -87,7 +88,10 @@ func NewBackupCommand(f client.Factory) *cobra.Command { kube.ExitPodWithMessage(logger, false, "Failed to create data mover backup, %v", err) } - s.run() + // Disable FIPS-140 compliance check, because Kopia doesn't support FIPS-140 yet. + fips140.WithoutEnforcement(func() { + s.run() + }) }, } @@ -327,6 +331,7 @@ func (s *dataMoverBackup) createDataPathService() (dataPathService, error) { s.config.changeID, s.config.volumeID, s.config.snapshotID, + s.cbtService, s.logger, ), nil } diff --git a/pkg/cmd/cli/datamover/restore.go b/pkg/cmd/cli/datamover/restore.go index 1d3cf84f4..6b112a248 100644 --- a/pkg/cmd/cli/datamover/restore.go +++ b/pkg/cmd/cli/datamover/restore.go @@ -15,6 +15,7 @@ package datamover import ( "context" + "crypto/fips140" "fmt" "os" "strings" @@ -35,6 +36,7 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/buildinfo" + "github.com/vmware-tanzu/velero/pkg/cbtservice" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd/util/signals" "github.com/vmware-tanzu/velero/pkg/datamover" @@ -55,6 +57,9 @@ type dataMoverRestoreConfig struct { ddName string cacheDir string resourceTimeout time.Duration + cbtSAName string + vsNamespace string + volumeID string } func NewRestoreCommand(f client.Factory) *cobra.Command { @@ -81,7 +86,10 @@ func NewRestoreCommand(f client.Factory) *cobra.Command { kube.ExitPodWithMessage(logger, false, "Failed to create data mover restore, %v", err) } - s.run() + // Disable FIPS-140 compliance check, because Kopia doesn't support FIPS-140 yet. + fips140.WithoutEnforcement(func() { + s.run() + }) }, } @@ -92,6 +100,9 @@ func NewRestoreCommand(f client.Factory) *cobra.Command { command.Flags().StringVar(&config.ddName, "data-download", config.ddName, "The data download name") command.Flags().StringVar(&config.cacheDir, "cache-volume-path", config.cacheDir, "The full path of the cache volume") command.Flags().DurationVar(&config.resourceTimeout, "resource-timeout", config.resourceTimeout, "How long to wait for resource processes which are not covered by other specific timeout parameters.") + command.Flags().StringVar(&config.cbtSAName, "cbt-sa-name", config.cbtSAName, "The name of the service account used by CSI's CBT service") + command.Flags().StringVar(&config.vsNamespace, "vs-namespace", config.vsNamespace, "The namespace of the VolumeSnapshot") + command.Flags().StringVar(&config.volumeID, "volume-id", config.volumeID, "The volume ID of the snapshot") _ = command.MarkFlagRequired("volume-path") _ = command.MarkFlagRequired("volume-mode") @@ -112,6 +123,7 @@ type dataMoverRestore struct { config dataMoverRestoreConfig kubeClient kubernetes.Interface dataPathMgr *datapath.Manager + cbtService cbtservice.Service } func newdataMoverRestore(logger logrus.FieldLogger, factory client.Factory, config dataMoverRestoreConfig) (*dataMoverRestore, error) { @@ -197,6 +209,12 @@ func newdataMoverRestore(logger logrus.FieldLogger, factory client.Factory, conf config: config, namespace: factory.Namespace(), nodeName: nodeName, + cbtService: cbtservice.NewService( + logger, + config.vsNamespace, + config.cbtSAName, + clientConfig, + ), } s.kubeClient, err = factory.KubeClient() @@ -290,5 +308,5 @@ func (s *dataMoverRestore) createDataPathService() (dataPathService, error) { return datamover.NewRestoreMicroService(s.ctx, s.client, s.kubeClient, s.config.ddName, s.namespace, s.nodeName, datapath.AccessPoint{ ByPath: s.config.volumePath, VolMode: uploader.PersistentVolumeMode(s.config.volumeMode), - }, s.dataPathMgr, repoEnsurer, credGetter, duInformer, s.config.cacheDir, s.logger), nil + }, s.dataPathMgr, repoEnsurer, credGetter, duInformer, s.config.cacheDir, s.config.volumeID, s.cbtService, s.logger), nil } diff --git a/pkg/cmd/cli/debug/debug.go b/pkg/cmd/cli/debug/debug.go index fac49d622..62f1d0823 100644 --- a/pkg/cmd/cli/debug/debug.go +++ b/pkg/cmd/cli/debug/debug.go @@ -38,6 +38,7 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/cli" ) //go:embed cshd-scripts/velero.cshd @@ -171,6 +172,10 @@ specs of resources created by velero server, and optionally the logs of backup a }, } o.bindFlags(c.Flags()) + + _ = c.RegisterFlagCompletionFunc("backup", cli.CompleteBackupNames(f)) + _ = c.RegisterFlagCompletionFunc("restore", cli.CompleteRestoreNames(f)) + return c } diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 0df53eb32..67c9517da 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -42,60 +42,61 @@ import ( // Options collects all the options for installing Velero into a Kubernetes cluster. type Options struct { - Namespace string - Image string - BucketName string - Prefix string - ProviderName string - PodAnnotations flag.Map - PodLabels flag.Map - ServiceAccountAnnotations flag.Map - ServiceAccountName string - VeleroPodCPURequest string - VeleroPodMemRequest string - VeleroPodCPULimit string - VeleroPodMemLimit string - NodeAgentPodCPURequest string - NodeAgentPodMemRequest string - NodeAgentPodCPULimit string - NodeAgentPodMemLimit string - RestoreOnly bool - SecretFile string - NoSecret bool - DryRun bool - BackupStorageConfig flag.Map - VolumeSnapshotConfig flag.Map - UseNodeAgent bool - UseNodeAgentWindows bool - PrivilegedNodeAgent bool - Wait bool - UseVolumeSnapshots bool - DefaultRepoMaintenanceFrequency time.Duration - GarbageCollectionFrequency time.Duration - PodVolumeOperationTimeout time.Duration - Plugins flag.StringArray - NoDefaultBackupLocation bool - CRDsOnly bool - CACertFile string - Features string - DefaultVolumesToFsBackup bool - UploaderType string - DefaultSnapshotMoveData bool - CSISnapshotEarlyFrequentPolling bool - DisableInformerCache bool - ScheduleSkipImmediately bool - PodResources kubeutil.PodResources - KeepLatestMaintenanceJobs int - BackupRepoConfigMap string - RepoMaintenanceJobConfigMap string - NodeAgentConfigMap string - ItemBlockWorkerCount int - ConcurrentBackups int - NodeAgentDisableHostPath bool - kubeletRootDir string - Apply bool - ServerPriorityClassName string - NodeAgentPriorityClassName string + Namespace string + Image string + BucketName string + Prefix string + ProviderName string + PodAnnotations flag.Map + PodLabels flag.Map + ServiceAccountAnnotations flag.Map + ServiceAccountName string + VeleroPodCPURequest string + VeleroPodMemRequest string + VeleroPodCPULimit string + VeleroPodMemLimit string + NodeAgentPodCPURequest string + NodeAgentPodMemRequest string + NodeAgentPodCPULimit string + NodeAgentPodMemLimit string + RestoreOnly bool + SecretFile string + NoSecret bool + DryRun bool + BackupStorageConfig flag.Map + VolumeSnapshotConfig flag.Map + UseNodeAgent bool + UseNodeAgentWindows bool + PrivilegedNodeAgent bool + Wait bool + UseVolumeSnapshots bool + DefaultRepoMaintenanceFrequency time.Duration + GarbageCollectionFrequency time.Duration + PodVolumeOperationTimeout time.Duration + Plugins flag.StringArray + NoDefaultBackupLocation bool + CRDsOnly bool + CACertFile string + Features string + DefaultVolumesToFsBackup bool + UploaderType string + DefaultSnapshotMoveData bool + CSISnapshotEarlyFrequentPolling bool + DisableInformerCache bool + ScheduleSkipImmediately bool + PodResources kubeutil.PodResources + KeepLatestMaintenanceJobs int + BackupRepoConfigMap string + RepoMaintenanceJobConfigMap string + DefaultResourceModifierConfigMap string + NodeAgentConfigMap string + ItemBlockWorkerCount int + ConcurrentBackups int + NodeAgentDisableHostPath bool + kubeletRootDir string + Apply bool + ServerPriorityClassName string + NodeAgentPriorityClassName string } // BindFlags adds command line values to the options struct. @@ -189,6 +190,12 @@ func (o *Options) BindFlags(flags *pflag.FlagSet) { o.RepoMaintenanceJobConfigMap, "The name of ConfigMap containing repository maintenance Job configurations.", ) + flags.StringVar( + &o.DefaultResourceModifierConfigMap, + "default-resource-modifier-configmap", + o.DefaultResourceModifierConfigMap, + "The name of a ConfigMap in the Velero namespace containing default resource modifier rules applied to all restores.", + ) flags.StringVar( &o.NodeAgentConfigMap, "node-agent-configmap", @@ -298,49 +305,50 @@ func (o *Options) AsVeleroOptions() (*install.VeleroOptions, error) { } return &install.VeleroOptions{ - Namespace: o.Namespace, - Image: o.Image, - ProviderName: o.ProviderName, - Bucket: o.BucketName, - Prefix: o.Prefix, - PodAnnotations: o.PodAnnotations.Data(), - PodLabels: o.PodLabels.Data(), - ServiceAccountAnnotations: o.ServiceAccountAnnotations.Data(), - ServiceAccountName: o.ServiceAccountName, - VeleroPodResources: veleroPodResources, - NodeAgentPodResources: nodeAgentPodResources, - SecretData: secretData, - RestoreOnly: o.RestoreOnly, - UseNodeAgent: o.UseNodeAgent, - UseNodeAgentWindows: o.UseNodeAgentWindows, - PrivilegedNodeAgent: o.PrivilegedNodeAgent, - UseVolumeSnapshots: o.UseVolumeSnapshots, - BSLConfig: o.BackupStorageConfig.Data(), - VSLConfig: o.VolumeSnapshotConfig.Data(), - DefaultRepoMaintenanceFrequency: o.DefaultRepoMaintenanceFrequency, - GarbageCollectionFrequency: o.GarbageCollectionFrequency, - PodVolumeOperationTimeout: o.PodVolumeOperationTimeout, - Plugins: o.Plugins, - NoDefaultBackupLocation: o.NoDefaultBackupLocation, - CACertData: caCertData, - Features: strings.Split(o.Features, ","), - DefaultVolumesToFsBackup: o.DefaultVolumesToFsBackup, - UploaderType: o.UploaderType, - DefaultSnapshotMoveData: o.DefaultSnapshotMoveData, - CSISnapshotEarlyFrequentPolling: o.CSISnapshotEarlyFrequentPolling, - DisableInformerCache: o.DisableInformerCache, - ScheduleSkipImmediately: o.ScheduleSkipImmediately, - PodResources: o.PodResources, - KeepLatestMaintenanceJobs: o.KeepLatestMaintenanceJobs, - BackupRepoConfigMap: o.BackupRepoConfigMap, - RepoMaintenanceJobConfigMap: o.RepoMaintenanceJobConfigMap, - NodeAgentConfigMap: o.NodeAgentConfigMap, - ItemBlockWorkerCount: o.ItemBlockWorkerCount, - ConcurrentBackups: o.ConcurrentBackups, - KubeletRootDir: o.kubeletRootDir, - NodeAgentDisableHostPath: o.NodeAgentDisableHostPath, - ServerPriorityClassName: o.ServerPriorityClassName, - NodeAgentPriorityClassName: o.NodeAgentPriorityClassName, + Namespace: o.Namespace, + Image: o.Image, + ProviderName: o.ProviderName, + Bucket: o.BucketName, + Prefix: o.Prefix, + PodAnnotations: o.PodAnnotations.Data(), + PodLabels: o.PodLabels.Data(), + ServiceAccountAnnotations: o.ServiceAccountAnnotations.Data(), + ServiceAccountName: o.ServiceAccountName, + VeleroPodResources: veleroPodResources, + NodeAgentPodResources: nodeAgentPodResources, + SecretData: secretData, + RestoreOnly: o.RestoreOnly, + UseNodeAgent: o.UseNodeAgent, + UseNodeAgentWindows: o.UseNodeAgentWindows, + PrivilegedNodeAgent: o.PrivilegedNodeAgent, + UseVolumeSnapshots: o.UseVolumeSnapshots, + BSLConfig: o.BackupStorageConfig.Data(), + VSLConfig: o.VolumeSnapshotConfig.Data(), + DefaultRepoMaintenanceFrequency: o.DefaultRepoMaintenanceFrequency, + GarbageCollectionFrequency: o.GarbageCollectionFrequency, + PodVolumeOperationTimeout: o.PodVolumeOperationTimeout, + Plugins: o.Plugins, + NoDefaultBackupLocation: o.NoDefaultBackupLocation, + CACertData: caCertData, + Features: strings.Split(o.Features, ","), + DefaultVolumesToFsBackup: o.DefaultVolumesToFsBackup, + UploaderType: o.UploaderType, + DefaultSnapshotMoveData: o.DefaultSnapshotMoveData, + CSISnapshotEarlyFrequentPolling: o.CSISnapshotEarlyFrequentPolling, + DisableInformerCache: o.DisableInformerCache, + ScheduleSkipImmediately: o.ScheduleSkipImmediately, + PodResources: o.PodResources, + KeepLatestMaintenanceJobs: o.KeepLatestMaintenanceJobs, + BackupRepoConfigMap: o.BackupRepoConfigMap, + RepoMaintenanceJobConfigMap: o.RepoMaintenanceJobConfigMap, + DefaultResourceModifierConfigMap: o.DefaultResourceModifierConfigMap, + NodeAgentConfigMap: o.NodeAgentConfigMap, + ItemBlockWorkerCount: o.ItemBlockWorkerCount, + ConcurrentBackups: o.ConcurrentBackups, + KubeletRootDir: o.kubeletRootDir, + NodeAgentDisableHostPath: o.NodeAgentDisableHostPath, + ServerPriorityClassName: o.ServerPriorityClassName, + NodeAgentPriorityClassName: o.NodeAgentPriorityClassName, }, nil } diff --git a/pkg/cmd/cli/nodeagent/server.go b/pkg/cmd/cli/nodeagent/server.go index 287e45591..c1442aab0 100644 --- a/pkg/cmd/cli/nodeagent/server.go +++ b/pkg/cmd/cli/nodeagent/server.go @@ -27,6 +27,7 @@ import ( "github.com/bombsimon/logrusr/v3" "github.com/cockroachdb/errors" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" snapshotv1client "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" @@ -175,6 +176,10 @@ func newNodeAgentServer(logger logrus.FieldLogger, factory client.Factory, confi cancelFunc() return nil, err } + if err := snapshotv1api.AddToScheme(scheme); err != nil { + cancelFunc() + return nil, err + } nodeName := os.Getenv("NODE_NAME") @@ -484,6 +489,7 @@ func (s *nodeAgentServer) run() { s.repoConfigMgr, podLabels, podAnnotations, + csiSnapshotMetadataServiceConfigs, ) if err := dataDownloadReconciler.SetupWithManager(s.mgr); err != nil { diff --git a/pkg/cmd/cli/plugin/add.go b/pkg/cmd/cli/plugin/add.go index 45a112a46..553a217dc 100644 --- a/pkg/cmd/cli/plugin/add.go +++ b/pkg/cmd/cli/plugin/add.go @@ -111,7 +111,7 @@ func NewAddCommand(f client.Factory) *cobra.Command { } // add the plugin as an init container - plugin := *builder.ForPluginContainer(args[0], corev1api.PullPolicy(imagePullPolicyFlag.String())).Result() + plugin := *builder.ForPluginContainer(args[0], corev1api.PullPolicy(imagePullPolicyFlag.String()), veleroDeploy.Spec.Template.Spec.InitContainers).Result() veleroDeploy.Spec.Template.Spec.InitContainers = append(veleroDeploy.Spec.Template.Spec.InitContainers, plugin) diff --git a/pkg/cmd/cli/podvolume/backup.go b/pkg/cmd/cli/podvolume/backup.go index 8bef9c574..93014a789 100644 --- a/pkg/cmd/cli/podvolume/backup.go +++ b/pkg/cmd/cli/podvolume/backup.go @@ -15,6 +15,7 @@ package podvolume import ( "context" + "crypto/fips140" "fmt" "os" "strings" @@ -80,7 +81,10 @@ func NewBackupCommand(f client.Factory) *cobra.Command { kube.ExitPodWithMessage(logger, false, "Failed to create pod volume backup, %v", err) } - s.run() + // Disable FIPS-140 compliance check, because Kopia doesn't support FIPS-140 yet. + fips140.WithoutEnforcement(func() { + s.run() + }) }, } diff --git a/pkg/cmd/cli/podvolume/restore.go b/pkg/cmd/cli/podvolume/restore.go index ab6554999..f982a5871 100644 --- a/pkg/cmd/cli/podvolume/restore.go +++ b/pkg/cmd/cli/podvolume/restore.go @@ -15,6 +15,7 @@ package podvolume import ( "context" + "crypto/fips140" "fmt" "os" "strings" @@ -79,7 +80,10 @@ func NewRestoreCommand(f client.Factory) *cobra.Command { kube.ExitPodWithMessage(logger, false, "Failed to create pod volume restore, %v", err) } - s.run() + // Disable FIPS-140 compliance check, because Kopia doesn't support FIPS-140 yet. + fips140.WithoutEnforcement(func() { + s.run() + }) }, } diff --git a/pkg/cmd/cli/repo/get.go b/pkg/cmd/cli/repo/get.go index ec57b9845..b3b914ae3 100644 --- a/pkg/cmd/cli/repo/get.go +++ b/pkg/cmd/cli/repo/get.go @@ -27,6 +27,7 @@ import ( api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" ) @@ -66,6 +67,7 @@ func NewGetCommand(f client.Factory, use string) *cobra.Command { }, } + c.ValidArgsFunction = cli.CompleteBackupRepositoryNames(f) c.Flags().StringVarP(&listOptions.LabelSelector, "selector", "l", listOptions.LabelSelector, "Only show items matching this label selector.") output.BindFlags(c.Flags()) diff --git a/pkg/cmd/cli/repomantenance/maintenance.go b/pkg/cmd/cli/repomantenance/maintenance.go index f89aba257..d541427a6 100644 --- a/pkg/cmd/cli/repomantenance/maintenance.go +++ b/pkg/cmd/cli/repomantenance/maintenance.go @@ -2,6 +2,7 @@ package repomantenance import ( "context" + "crypto/fips140" "fmt" "os" "strings" @@ -57,7 +58,10 @@ func NewCommand(f velerocli.Factory) *cobra.Command { Hidden: true, Short: "VELERO INTERNAL COMMAND ONLY - not intended to be run directly by users", Run: func(c *cobra.Command, args []string) { - o.Run(f) + // Disable FIPS-140 compliance check, because Kopia doesn't support FIPS-140 yet. + fips140.WithoutEnforcement(func() { + o.Run(f) + }) }, } diff --git a/pkg/cmd/cli/restore/create.go b/pkg/cmd/cli/restore/create.go index 3f59b6a6b..7b65407de 100644 --- a/pkg/cmd/cli/restore/create.go +++ b/pkg/cmd/cli/restore/create.go @@ -36,6 +36,7 @@ import ( api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" "github.com/vmware-tanzu/velero/pkg/util/boolptr" @@ -81,36 +82,42 @@ Notes: output.BindFlags(c.Flags()) output.ClearOutputFlagDefault(c) + _ = c.RegisterFlagCompletionFunc("from-backup", cli.CompleteBackupNames(f)) + _ = c.RegisterFlagCompletionFunc("from-schedule", cli.CompleteScheduleNames(f)) + return c } type CreateOptions struct { - BackupName string - ScheduleName string - RestoreName string - RestoreVolumes flag.OptionalBool - PreserveNodePorts flag.OptionalBool - Labels flag.Map - Annotations flag.Map - IncludeNamespaces flag.StringArray - ExcludeNamespaces flag.StringArray - ExistingResourcePolicy string - IncludeResources flag.StringArray - ExcludeResources flag.StringArray - StatusIncludeResources flag.StringArray - StatusExcludeResources flag.StringArray - NamespaceMappings flag.Map - Selector flag.LabelSelector - OrSelector flag.OrLabelSelector - IncludeClusterResources flag.OptionalBool - Wait bool - AllowPartiallyFailed flag.OptionalBool - ItemOperationTimeout time.Duration - ResourceModifierConfigMap string - ResourcePoliciesConfigMap string - WriteSparseFiles flag.OptionalBool - ParallelFilesDownload int - client kbclient.WithWatch + BackupName string + ScheduleName string + RestoreName string + RestoreVolumes flag.OptionalBool + PreserveNodePorts flag.OptionalBool + Labels flag.Map + Annotations flag.Map + IncludeNamespaces flag.StringArray + ExcludeNamespaces flag.StringArray + ExistingResourcePolicy string + ExistingVolumeDataPolicy string + IncludeResources flag.StringArray + ExcludeResources flag.StringArray + StatusIncludeResources flag.StringArray + StatusExcludeResources flag.StringArray + NamespaceMappings flag.Map + Selector flag.LabelSelector + OrSelector flag.OrLabelSelector + IncludeClusterResources flag.OptionalBool + Wait bool + AllowPartiallyFailed flag.OptionalBool + ItemOperationTimeout time.Duration + ResourceModifierConfigMap string + ResourcePoliciesConfigMap string + SkipDefaultResourceModifier bool + WriteSparseFiles flag.OptionalBool + ParallelFilesDownload int + DeleteExtraFiles flag.OptionalBool + client kbclient.WithWatch } func NewCreateOptions() *CreateOptions { @@ -123,6 +130,7 @@ func NewCreateOptions() *CreateOptions { PreserveNodePorts: flag.NewOptionalBool(nil), IncludeClusterResources: flag.NewOptionalBool(nil), WriteSparseFiles: flag.NewOptionalBool(nil), + DeleteExtraFiles: flag.NewOptionalBool(nil), } } @@ -136,7 +144,8 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) { flags.Var(&o.Annotations, "annotations", "Annotations to apply to the restore.") flags.Var(&o.IncludeResources, "include-resources", "Resources to include in the restore, formatted as resource.group, such as storageclasses.storage.k8s.io (use '*' for all resources).") flags.Var(&o.ExcludeResources, "exclude-resources", "Resources to exclude from the restore, formatted as resource.group, such as storageclasses.storage.k8s.io.") - flags.StringVar(&o.ExistingResourcePolicy, "existing-resource-policy", "", "Restore Policy to be used during the restore workflow, can be - none or update") + flags.StringVar(&o.ExistingResourcePolicy, "existing-resource-policy", "", "Restore Policy to be used during the restore workflow for Kubernetes resources, can be - none or update") + flags.StringVar(&o.ExistingVolumeDataPolicy, "existing-volume-data-policy", "", "Restore Policy to be used during the restore workflow for volume data, can be - none, full or incremental") flags.Var(&o.StatusIncludeResources, "status-include-resources", "Resources to include in the restore status, formatted as resource.group, such as storageclasses.storage.k8s.io.") flags.Var(&o.StatusExcludeResources, "status-exclude-resources", "Resources to exclude from the restore status, formatted as resource.group, such as storageclasses.storage.k8s.io.") flags.VarP(&o.Selector, "selector", "l", "Only restore resources matching this label selector.") @@ -164,10 +173,15 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) { flags.StringVar(&o.ResourcePoliciesConfigMap, "resource-policies-configmap", "", "Reference to the ConfigMap containing restore resource filter policies") + flags.BoolVar(&o.SkipDefaultResourceModifier, "skip-default-resource-modifier", false, "Skip applying the server-configured default resource modifier for this restore") + f = flags.VarPF(&o.WriteSparseFiles, "write-sparse-files", "", "Whether to write sparse files during restoring volumes") f.NoOptDefVal = cmd.TRUE flags.IntVar(&o.ParallelFilesDownload, "parallel-files-download", 0, "The number of restore operations to run in parallel. If set to 0, the default parallelism will be the number of CPUs for the node that node agent pod is running.") + + f = flags.VarPF(&o.DeleteExtraFiles, "delete-extra-files", "", "Whether to delete extra files in the target volume that do not exist in the backup during file system restore. This setting is only applicable to File System restores (PodVolumeBackup or CSI File System Data Move) and has no effect on Block Data Move restores.") + f.NoOptDefVal = cmd.TRUE } func (o *CreateOptions) Complete(args []string, f client.Factory) error { @@ -217,6 +231,10 @@ func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Facto return errors.New("existing-resource-policy has invalid value, it accepts only none, update as value") } + if len(o.ExistingVolumeDataPolicy) > 0 && !restore.IsVolumeDataPolicyValid(o.ExistingVolumeDataPolicy) { + return errors.New("existing-volume-data-policy has invalid value, it accepts only none, full, incremental as value") + } + if o.ParallelFilesDownload < 0 { return errors.New("parallel-files-download cannot be negative") } @@ -337,31 +355,37 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { Annotations: o.Annotations.Data(), }, Spec: api.RestoreSpec{ - BackupName: o.BackupName, - ScheduleName: o.ScheduleName, - IncludedNamespaces: o.IncludeNamespaces, - ExcludedNamespaces: o.ExcludeNamespaces, - IncludedResources: o.IncludeResources, - ExcludedResources: o.ExcludeResources, - ExistingResourcePolicy: api.PolicyType(o.ExistingResourcePolicy), - NamespaceMapping: o.NamespaceMappings.Data(), - LabelSelector: o.Selector.LabelSelector, - OrLabelSelectors: o.OrSelector.OrLabelSelectors, - RestorePVs: o.RestoreVolumes.Value, - PreserveNodePorts: o.PreserveNodePorts.Value, - IncludeClusterResources: o.IncludeClusterResources.Value, - ResourceModifier: resModifiers, - ResourcePolicy: resPolicies, + BackupName: o.BackupName, + ScheduleName: o.ScheduleName, + IncludedNamespaces: o.IncludeNamespaces, + ExcludedNamespaces: o.ExcludeNamespaces, + IncludedResources: o.IncludeResources, + ExcludedResources: o.ExcludeResources, + ExistingResourcePolicy: api.ResourcePolicyType(o.ExistingResourcePolicy), + ExistingVolumeDataPolicy: api.VolumeDataPolicyType(o.ExistingVolumeDataPolicy), + NamespaceMapping: o.NamespaceMappings.Data(), + LabelSelector: o.Selector.LabelSelector, + OrLabelSelectors: o.OrSelector.OrLabelSelectors, + RestorePVs: o.RestoreVolumes.Value, + PreserveNodePorts: o.PreserveNodePorts.Value, + IncludeClusterResources: o.IncludeClusterResources.Value, + ResourceModifier: resModifiers, + ResourcePolicy: resPolicies, ItemOperationTimeout: metav1.Duration{ Duration: o.ItemOperationTimeout, }, UploaderConfig: &api.UploaderConfigForRestore{ WriteSparseFiles: o.WriteSparseFiles.Value, ParallelFilesDownload: o.ParallelFilesDownload, + DeleteExtraFiles: o.DeleteExtraFiles.Value, }, }, } + if o.SkipDefaultResourceModifier { + restore.Spec.SkipDefaultResourceModifier = boolptr.True() + } + if len([]string(o.StatusIncludeResources)) > 0 { restore.Spec.RestoreStatus = &api.RestoreStatusSpec{ IncludedResources: o.StatusIncludeResources, diff --git a/pkg/cmd/cli/restore/create_test.go b/pkg/cmd/cli/restore/create_test.go index 9a6a92608..50d4aebde 100644 --- a/pkg/cmd/cli/restore/create_test.go +++ b/pkg/cmd/cli/restore/create_test.go @@ -68,6 +68,7 @@ func TestCreateCommand(t *testing.T) { includeNamespaces := "app1,app2" excludeNamespaces := "pod1,pod2,pod3" existingResourcePolicy := "none" + existingVolumeDataPolicy := "none" includeResources := "sc,sts" excludeResources := "job" statusIncludeResources := "sc,sts" @@ -80,6 +81,7 @@ func TestCreateCommand(t *testing.T) { resourceModifierConfigMap := "modifier-cm" ResourcePoliciesConfigMap := "policies-cm" writeSparseFiles := "true" + deleteExtraFiles := "true" parallel := 2 flags := new(pflag.FlagSet) o := NewCreateOptions() @@ -92,6 +94,7 @@ func TestCreateCommand(t *testing.T) { flags.Parse([]string{"--labels", labels}) flags.Parse([]string{"--annotations", annotations}) flags.Parse([]string{"--existing-resource-policy", existingResourcePolicy}) + flags.Parse([]string{"--existing-volume-data-policy", existingVolumeDataPolicy}) flags.Parse([]string{"--include-namespaces", includeNamespaces}) flags.Parse([]string{"--exclude-namespaces", excludeNamespaces}) flags.Parse([]string{"--include-resources", includeResources}) @@ -105,7 +108,9 @@ func TestCreateCommand(t *testing.T) { flags.Parse([]string{"--item-operation-timeout", itemOperationTimeout}) flags.Parse([]string{"--resource-modifier-configmap", resourceModifierConfigMap}) flags.Parse([]string{"--resource-policies-configmap", ResourcePoliciesConfigMap}) + flags.Parse([]string{"--skip-default-resource-modifier"}) flags.Parse([]string{"--write-sparse-files", writeSparseFiles}) + flags.Parse([]string{"--delete-extra-files", deleteExtraFiles}) flags.Parse([]string{"--parallel-files-download", "2"}) client := velerotest.NewFakeControllerRuntimeClient(t).(kbclient.WithWatch) @@ -133,6 +138,7 @@ func TestCreateCommand(t *testing.T) { require.Equal(t, includeNamespaces, o.IncludeNamespaces.String()) require.Equal(t, excludeNamespaces, o.ExcludeNamespaces.String()) require.Equal(t, existingResourcePolicy, o.ExistingResourcePolicy) + require.Equal(t, existingVolumeDataPolicy, o.ExistingVolumeDataPolicy) require.Equal(t, includeResources, o.IncludeResources.String()) require.Equal(t, excludeResources, o.ExcludeResources.String()) @@ -145,8 +151,10 @@ func TestCreateCommand(t *testing.T) { require.Equal(t, itemOperationTimeout, o.ItemOperationTimeout.String()) require.Equal(t, resourceModifierConfigMap, o.ResourceModifierConfigMap) require.Equal(t, ResourcePoliciesConfigMap, o.ResourcePoliciesConfigMap) + require.True(t, o.SkipDefaultResourceModifier) require.Equal(t, writeSparseFiles, o.WriteSparseFiles.String()) require.Equal(t, parallel, o.ParallelFilesDownload) + require.Equal(t, deleteExtraFiles, o.DeleteExtraFiles.String()) }) t.Run("create a restore from schedule", func(t *testing.T) { diff --git a/pkg/cmd/cli/restore/delete.go b/pkg/cmd/cli/restore/delete.go index 51c31e1da..b20186fb8 100644 --- a/pkg/cmd/cli/restore/delete.go +++ b/pkg/cmd/cli/restore/delete.go @@ -61,6 +61,7 @@ func NewDeleteCommand(f client.Factory, use string) *cobra.Command { cmd.CheckError(Run(o)) }, } + c.ValidArgsFunction = cli.CompleteRestoreNames(f) o.BindFlags(c.Flags()) return c } diff --git a/pkg/cmd/cli/restore/describe.go b/pkg/cmd/cli/restore/describe.go index 6404ef21d..7fc58ce22 100644 --- a/pkg/cmd/cli/restore/describe.go +++ b/pkg/cmd/cli/restore/describe.go @@ -29,6 +29,7 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" "github.com/vmware-tanzu/velero/pkg/label" ) @@ -92,6 +93,7 @@ func NewDescribeCommand(f client.Factory, use string) *cobra.Command { }, } + c.ValidArgsFunction = cli.CompleteRestoreNames(f) c.Flags().StringVarP(&listOptions.LabelSelector, "selector", "l", listOptions.LabelSelector, "Only show items matching this label selector.") c.Flags().BoolVar(&details, "details", details, "Display additional detail in the command output.") c.Flags().BoolVar(&insecureSkipTLSVerify, "insecure-skip-tls-verify", insecureSkipTLSVerify, "If true, the object store's TLS certificate will not be checked for validity. This is insecure and susceptible to man-in-the-middle attacks. Not recommended for production.") diff --git a/pkg/cmd/cli/restore/get.go b/pkg/cmd/cli/restore/get.go index 9a4014b25..568e31b8d 100644 --- a/pkg/cmd/cli/restore/get.go +++ b/pkg/cmd/cli/restore/get.go @@ -27,6 +27,7 @@ import ( api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" ) @@ -76,6 +77,7 @@ func NewGetCommand(f client.Factory, use string) *cobra.Command { }, } + c.ValidArgsFunction = cli.CompleteRestoreNames(f) c.Flags().StringVarP(&listOptions.LabelSelector, "selector", "l", listOptions.LabelSelector, "Only show items matching this label selector.") output.BindFlags(c.Flags()) diff --git a/pkg/cmd/cli/restore/logs.go b/pkg/cmd/cli/restore/logs.go index f4315c917..366fd511e 100644 --- a/pkg/cmd/cli/restore/logs.go +++ b/pkg/cmd/cli/restore/logs.go @@ -23,68 +23,106 @@ import ( "time" "github.com/spf13/cobra" + "github.com/spf13/pflag" apierrors "k8s.io/apimachinery/pkg/api/errors" - ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" ) -func NewLogsCommand(f client.Factory) *cobra.Command { +// LogsOptions holds the state for the restore logs command, mirroring +// pkg/cmd/cli/backup.LogsOptions so both commands are shaped the same way. +type LogsOptions struct { + Timeout time.Duration + InsecureSkipTLSVerify bool + CaCertFile string + Client kbclient.Client + RestoreName string +} + +func NewLogsOptions() LogsOptions { config, err := client.LoadConfig() if err != nil { fmt.Fprintf(os.Stderr, "WARNING: Error reading config file: %v\n", err) } - timeout := time.Minute - insecureSkipTLSVerify := false - caCertFile := config.CACertFile() + return LogsOptions{ + Timeout: time.Minute, + InsecureSkipTLSVerify: false, + CaCertFile: config.CACertFile(), + } +} + +func (l *LogsOptions) BindFlags(flags *pflag.FlagSet) { + flags.DurationVar(&l.Timeout, "timeout", l.Timeout, "How long to wait to receive logs.") + flags.BoolVar(&l.InsecureSkipTLSVerify, "insecure-skip-tls-verify", l.InsecureSkipTLSVerify, "If true, the object store's TLS certificate will not be checked for validity. This is insecure and susceptible to man-in-the-middle attacks. Not recommended for production.") + flags.StringVar(&l.CaCertFile, "cacert", l.CaCertFile, "Path to a certificate bundle to use when verifying TLS connections. If not specified, the CA certificate from the BackupStorageLocation will be used if available.") +} + +func (l *LogsOptions) Run(c *cobra.Command, f client.Factory) error { + restore := new(velerov1api.Restore) + err := l.Client.Get(context.Background(), kbclient.ObjectKey{Namespace: f.Namespace(), Name: l.RestoreName}, restore) + if apierrors.IsNotFound(err) { + return fmt.Errorf("restore %q does not exist", l.RestoreName) + } else if err != nil { + return fmt.Errorf("error checking for restore %q: %v", l.RestoreName, err) + } + + switch restore.Status.Phase { + case velerov1api.RestorePhaseCompleted, velerov1api.RestorePhaseFailed, velerov1api.RestorePhasePartiallyFailed, velerov1api.RestorePhaseWaitingForPluginOperations, velerov1api.RestorePhaseWaitingForPluginOperationsPartiallyFailed: + // terminal and waiting for plugin operations phases, do nothing. + default: + return fmt.Errorf("logs for restore %q are not available until it's finished processing, please wait "+ + "until the restore has a phase of Completed or Failed and try again", l.RestoreName) + } + + // Get BSL cacert if available + bslCACert, err := cacert.GetCACertFromRestore(context.Background(), l.Client, f.Namespace(), restore) + if err != nil { + // Log the error but don't fail - we can still try to download without the BSL cacert + fmt.Fprintf(os.Stderr, "WARNING: Error getting cacert from BSL: %v\n", err) + bslCACert = "" + } + + return downloadrequest.StreamWithBSLCACert(context.Background(), l.Client, f.Namespace(), l.RestoreName, velerov1api.DownloadTargetKindRestoreLog, os.Stdout, l.Timeout, l.InsecureSkipTLSVerify, l.CaCertFile, bslCACert) +} + +func (l *LogsOptions) Complete(args []string, f client.Factory) error { + if len(args) > 0 { + l.RestoreName = args[0] + } + + kbClient, err := f.KubebuilderClient() + if err != nil { + return err + } + l.Client = kbClient + return nil +} + +func NewLogsCommand(f client.Factory) *cobra.Command { + l := NewLogsOptions() c := &cobra.Command{ Use: "logs RESTORE", Short: "Get restore logs", Args: cobra.ExactArgs(1), Run: func(c *cobra.Command, args []string) { - restoreName := args[0] - - kbClient, err := f.KubebuilderClient() + err := l.Complete(args, f) cmd.CheckError(err) - restore := new(velerov1api.Restore) - err = kbClient.Get(context.Background(), ctrlclient.ObjectKey{Namespace: f.Namespace(), Name: restoreName}, restore) - if apierrors.IsNotFound(err) { - cmd.Exit("Restore %q does not exist.", restoreName) - } else if err != nil { - cmd.Exit("Error checking for restore %q: %v", restoreName, err) - } - - switch restore.Status.Phase { - case velerov1api.RestorePhaseCompleted, velerov1api.RestorePhaseFailed, velerov1api.RestorePhasePartiallyFailed, velerov1api.RestorePhaseWaitingForPluginOperations, velerov1api.RestorePhaseWaitingForPluginOperationsPartiallyFailed: - // terminal and waiting for plugin operations phases, don't exit. - default: - cmd.Exit("Logs for restore %q are not available until it's finished processing. Please wait "+ - "until the restore has a phase of Completed or Failed and try again.", restoreName) - } - - // Get BSL cacert if available - bslCACert, err := cacert.GetCACertFromRestore(context.Background(), kbClient, f.Namespace(), restore) - if err != nil { - // Log the error but don't fail - we can still try to download without the BSL cacert - fmt.Fprintf(os.Stderr, "WARNING: Error getting cacert from BSL: %v\n", err) - bslCACert = "" - } - - err = downloadrequest.StreamWithBSLCACert(context.Background(), kbClient, f.Namespace(), restoreName, velerov1api.DownloadTargetKindRestoreLog, os.Stdout, timeout, insecureSkipTLSVerify, caCertFile, bslCACert) + err = l.Run(c, f) cmd.CheckError(err) }, } - c.Flags().DurationVar(&timeout, "timeout", timeout, "How long to wait to receive logs.") - c.Flags().BoolVar(&insecureSkipTLSVerify, "insecure-skip-tls-verify", insecureSkipTLSVerify, "If true, the object store's TLS certificate will not be checked for validity. This is insecure and susceptible to man-in-the-middle attacks. Not recommended for production.") - c.Flags().StringVar(&caCertFile, "cacert", caCertFile, "Path to a certificate bundle to use when verifying TLS connections. If not specified, the CA certificate from the BackupStorageLocation will be used if available.") + c.ValidArgsFunction = cli.CompleteRestoreNames(f) + l.BindFlags(c.Flags()) return c } diff --git a/pkg/cmd/cli/restore/logs_test.go b/pkg/cmd/cli/restore/logs_test.go index 61c2392b6..5e020bf43 100644 --- a/pkg/cmd/cli/restore/logs_test.go +++ b/pkg/cmd/cli/restore/logs_test.go @@ -17,10 +17,12 @@ limitations under the License. package restore import ( + "fmt" "os" "testing" "time" + flag "github.com/spf13/pflag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" kbclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -77,13 +79,20 @@ func TestNewLogsCommand(t *testing.T) { c := NewLogsCommand(f) assert.Equal(t, "Get restore logs", c.Short) - // The restore command exits with an error message when restore is not complete - // We can't easily test this since it calls cmd.Exit, which exits the process - // So we'll skip this test case - t.Skip("Cannot test restore not complete case due to cmd.Exit() call") + l := NewLogsOptions() + flags := new(flag.FlagSet) + l.BindFlags(flags) + err = l.Complete([]string{restoreName}, f) + require.NoError(t, err) + + err = l.Run(c, f) + require.Error(t, err) + require.ErrorContains(t, err, fmt.Sprintf("logs for restore %q are not available until it's finished processing", restoreName)) }) t.Run("Restore not exist test", func(t *testing.T) { + restoreName := "not-exist" + // create a factory f := &factorymocks.Factory{} @@ -95,10 +104,15 @@ func TestNewLogsCommand(t *testing.T) { c := NewLogsCommand(f) assert.Equal(t, "Get restore logs", c.Short) - // The restore command exits with an error message when restore doesn't exist - // We can't easily test this since it calls cmd.Exit, which exits the process - // So we'll skip this test case - t.Skip("Cannot test restore not exist case due to cmd.Exit() call") + l := NewLogsOptions() + flags := new(flag.FlagSet) + l.BindFlags(flags) + err := l.Complete([]string{restoreName}, f) + require.NoError(t, err) + + err = l.Run(c, f) + require.Error(t, err) + require.Equal(t, fmt.Sprintf("restore %q does not exist", restoreName), err.Error()) }) t.Run("Restore with BSL cacert test", func(t *testing.T) { diff --git a/pkg/cmd/cli/schedule/create.go b/pkg/cmd/cli/schedule/create.go index 2e4a1e8e9..03f5626fd 100644 --- a/pkg/cmd/cli/schedule/create.go +++ b/pkg/cmd/cli/schedule/create.go @@ -30,6 +30,7 @@ import ( api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/cli/backup" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" ) @@ -77,6 +78,9 @@ example: "@every 2h30m".`, output.BindFlags(c.Flags()) output.ClearOutputFlagDefault(c) + _ = c.RegisterFlagCompletionFunc("storage-location", cli.CompleteBackupStorageLocationNames(f)) + _ = c.RegisterFlagCompletionFunc("volume-snapshot-locations", cli.CompleteVolumeSnapshotLocationNames(f)) + return c } diff --git a/pkg/cmd/cli/schedule/delete.go b/pkg/cmd/cli/schedule/delete.go index 78e8c9104..28418afbd 100644 --- a/pkg/cmd/cli/schedule/delete.go +++ b/pkg/cmd/cli/schedule/delete.go @@ -62,6 +62,7 @@ func NewDeleteCommand(f client.Factory, use string) *cobra.Command { }, } + c.ValidArgsFunction = cli.CompleteScheduleNames(f) o.BindFlags(c.Flags()) return c } diff --git a/pkg/cmd/cli/schedule/describe.go b/pkg/cmd/cli/schedule/describe.go index 82c88dac7..b657245e9 100644 --- a/pkg/cmd/cli/schedule/describe.go +++ b/pkg/cmd/cli/schedule/describe.go @@ -28,6 +28,7 @@ import ( v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" ) @@ -73,6 +74,7 @@ func NewDescribeCommand(f client.Factory, use string) *cobra.Command { }, } + c.ValidArgsFunction = cli.CompleteScheduleNames(f) c.Flags().StringVarP(&listOptions.LabelSelector, "selector", "l", listOptions.LabelSelector, "Only show items matching this label selector.") return c diff --git a/pkg/cmd/cli/schedule/get.go b/pkg/cmd/cli/schedule/get.go index 88bd49fe0..ba8ddb122 100644 --- a/pkg/cmd/cli/schedule/get.go +++ b/pkg/cmd/cli/schedule/get.go @@ -27,6 +27,7 @@ import ( api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" ) @@ -71,6 +72,7 @@ func NewGetCommand(f client.Factory, use string) *cobra.Command { }, } + c.ValidArgsFunction = cli.CompleteScheduleNames(f) c.Flags().StringVarP(&listOptions.LabelSelector, "selector", "l", listOptions.LabelSelector, "Only show items matching this label selector.") output.BindFlags(c.Flags()) diff --git a/pkg/cmd/cli/schedule/pause.go b/pkg/cmd/cli/schedule/pause.go index 41a17f384..06fc43f5c 100644 --- a/pkg/cmd/cli/schedule/pause.go +++ b/pkg/cmd/cli/schedule/pause.go @@ -60,6 +60,7 @@ func NewPauseCommand(f client.Factory, use string) *cobra.Command { }, } + c.ValidArgsFunction = cli.CompleteScheduleNames(f) o.BindFlags(c.Flags()) pauseOpts.BindFlags(c.Flags()) diff --git a/pkg/cmd/cli/schedule/unpause.go b/pkg/cmd/cli/schedule/unpause.go index 72197a934..15107ba38 100644 --- a/pkg/cmd/cli/schedule/unpause.go +++ b/pkg/cmd/cli/schedule/unpause.go @@ -49,6 +49,7 @@ func NewUnpauseCommand(f client.Factory, use string) *cobra.Command { }, } + c.ValidArgsFunction = cli.CompleteScheduleNames(f) o.BindFlags(c.Flags()) pauseOpts.BindFlags(c.Flags()) diff --git a/pkg/cmd/cli/snapshotlocation/get.go b/pkg/cmd/cli/snapshotlocation/get.go index 2acddbf7f..79da478bf 100644 --- a/pkg/cmd/cli/snapshotlocation/get.go +++ b/pkg/cmd/cli/snapshotlocation/get.go @@ -26,6 +26,7 @@ import ( api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" ) @@ -56,6 +57,7 @@ func NewGetCommand(f client.Factory, use string) *cobra.Command { cmd.CheckError(err) }, } + c.ValidArgsFunction = cli.CompleteVolumeSnapshotLocationNames(f) c.Flags().StringVarP(&listOptions.LabelSelector, "selector", "l", listOptions.LabelSelector, "Only show items matching this label selector") output.BindFlags(c.Flags()) return c diff --git a/pkg/cmd/cli/snapshotlocation/set.go b/pkg/cmd/cli/snapshotlocation/set.go index 0814bdfe7..c67ef4231 100644 --- a/pkg/cmd/cli/snapshotlocation/set.go +++ b/pkg/cmd/cli/snapshotlocation/set.go @@ -30,6 +30,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" ) @@ -48,6 +49,7 @@ func NewSetCommand(f client.Factory, use string) *cobra.Command { }, } + c.ValidArgsFunction = cli.CompleteVolumeSnapshotLocationNames(f) o.BindFlags(c.Flags()) return c } diff --git a/pkg/cmd/cli/uninstall/uninstall.go b/pkg/cmd/cli/uninstall/uninstall.go index 93e0118c6..033d0603c 100644 --- a/pkg/cmd/cli/uninstall/uninstall.go +++ b/pkg/cmd/cli/uninstall/uninstall.go @@ -57,13 +57,11 @@ var resToDelete = []kbclient.ObjectList{} // uninstallOptions collects all the options for uninstalling Velero from a Kubernetes cluster. type uninstallOptions struct { - wait bool // deprecated force bool } // BindFlags adds command line values to the options struct. func (o *uninstallOptions) BindFlags(flags *pflag.FlagSet) { - flags.BoolVar(&o.wait, "wait", o.wait, "Wait for Velero uninstall to be ready. Optional. Deprecated.") flags.BoolVar(&o.force, "force", o.force, "Forces the Velero uninstall. Optional.") } @@ -81,10 +79,6 @@ Use '--force' to skip the prompt confirming if you want to uninstall Velero. `, Example: ` # velero uninstall --namespace staging`, Run: func(c *cobra.Command, args []string) { - if o.wait { - fmt.Println("Warning: the \"--wait\" option is deprecated and will be removed in a future release. The uninstall command always waits for the uninstall to complete.") - } - // Confirm if not asked to force-skip confirmation if !o.force { fmt.Println("You are about to uninstall Velero.") diff --git a/pkg/cmd/server/config/config.go b/pkg/cmd/server/config/config.go index c8080da21..5198adcbc 100644 --- a/pkg/cmd/server/config/config.go +++ b/pkg/cmd/server/config/config.go @@ -28,6 +28,11 @@ const ( defaultPodVolumeOperationTimeout = 240 * time.Minute defaultResourceTerminatingTimeout = 10 * time.Minute + // DefaultResourceTimeout is the default for --resource-timeout. It matches + // defaultResourceTerminatingTimeout so controller fallbacks stay aligned with + // server defaults (see pkg/cmd/server/config/config.go). + DefaultResourceTimeout = defaultResourceTerminatingTimeout + // server's client default qps and burst defaultClientQPS float32 = 100.0 defaultClientBurst int = 100 @@ -41,7 +46,7 @@ const ( defaultCSISnapshotTimeout = 10 * time.Minute defaultItemOperationTimeout = 4 * time.Hour - resourceTimeout = 10 * time.Minute + resourceTimeout = defaultResourceTerminatingTimeout defaultMaxConcurrentK8SConnections = 30 defaultDisableInformerCache = false @@ -182,6 +187,8 @@ type Config struct { ItemBlockWorkerCount int ConcurrentBackups int GlobalBackupVolumePoliciesConfigMap string + DefaultResourceModifierConfigMap string + MaxBackupExtractionSize int } func GetDefaultConfig() *Config { @@ -282,4 +289,16 @@ func (c *Config) BindFlags(flags *pflag.FlagSet) { c.GlobalBackupVolumePoliciesConfigMap, "The name of a ConfigMap in the Velero install namespace holding global backup volume policies that are merged into every backup. Optional.", ) + flags.StringVar( + &c.DefaultResourceModifierConfigMap, + "default-resource-modifier-configmap", + c.DefaultResourceModifierConfigMap, + "The name of a ConfigMap in the Velero namespace containing default resource modifier rules applied to all restores. Ignored when a per-restore resource modifier is specified.", + ) + flags.IntVar( + &c.MaxBackupExtractionSize, + "max-backup-extraction-size", + c.MaxBackupExtractionSize, + "Maximum size of a backup extraction in megabytes. If not set, default value (16GB) will be used.", + ) } diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 83627f9d1..665577672 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -61,6 +61,7 @@ import ( "github.com/vmware-tanzu/velero/internal/storage" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + "github.com/vmware-tanzu/velero/pkg/archive" "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/buildinfo" "github.com/vmware-tanzu/velero/pkg/client" @@ -881,6 +882,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.config.DisableInformerCache, s.crClient, s.config.ResourceTimeout, + s.config.DefaultResourceModifierConfigMap, ) if err = r.SetupWithManager(s.mgr); err != nil { @@ -934,6 +936,11 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string } } + if s.config.MaxBackupExtractionSize > 0 { + s.logger.Infof("Setting backup data extraction cap as %v MB", s.config.MaxBackupExtractionSize) + archive.SetMaxExtractionSize(int64(s.config.MaxBackupExtractionSize) * 1024 * 1024) + } + s.logger.Info("Server starting...") if err := s.mgr.Start(s.ctx); err != nil { diff --git a/pkg/cmd/util/downloadrequest/downloadrequest.go b/pkg/cmd/util/downloadrequest/downloadrequest.go index f0956b1cb..2c90a2894 100644 --- a/pkg/cmd/util/downloadrequest/downloadrequest.go +++ b/pkg/cmd/util/downloadrequest/downloadrequest.go @@ -40,6 +40,12 @@ import ( // not found var ErrNotFound = errors.New("file not found") var ErrDownloadRequestDownloadURLTimeout = errors.New("download request download url timeout, check velero server logs for errors. backup storage location may not be available") +var unzipLimit int64 = 1024 * 1024 * 1024 // 1GB limit + +// ErrDownloadRequestFailed is returned when the server refused the request and gave no +// reason. The controller sets a message in every path that fails today, so this is a +// fallback rather than the usual case. +var ErrDownloadRequestFailed = errors.New("download request failed, check velero server logs for errors") func Stream( ctx context.Context, @@ -114,6 +120,16 @@ func getDownloadURL( if updated.Status.DownloadURL != "" { return updated.Status.DownloadURL, nil } + + // Failed is terminal. Waiting for a URL that will never be signed would end in + // ErrDownloadRequestDownloadURLTimeout, which blames the storage location for + // something the status already explains. + if updated.Status.Phase == veleroV1api.DownloadRequestPhaseFailed { + if updated.Status.Message != "" { + return "", errors.New(updated.Status.Message) + } + return "", ErrDownloadRequestFailed + } } } } @@ -202,17 +218,35 @@ func download( return errors.Errorf("request failed: %v", string(body)) } - reader := resp.Body + var r io.Reader = resp.Body + var gzipReader *gzip.Reader if kind != veleroV1api.DownloadTargetKindBackupContents { // need to decompress logs - gzipReader, err := gzip.NewReader(resp.Body) + var err error + gzipReader, err = gzip.NewReader(resp.Body) if err != nil { return err } defer gzipReader.Close() - reader = gzipReader + + r = io.LimitReader(gzipReader, unzipLimit) } - _, err = io.Copy(w, reader) - return err + _, err = io.Copy(w, r) + if err != nil { + return err + } + + if gzipReader != nil { + var buf [1]byte + n, err := gzipReader.Read(buf[:]) + if n > 0 || err == nil { + return errors.Errorf("decompressed data exceeds the limit") + } + if err != io.EOF { + return err + } + } + + return nil } diff --git a/pkg/cmd/util/downloadrequest/downloadrequest_test.go b/pkg/cmd/util/downloadrequest/downloadrequest_test.go index 995e83dc6..36a02413a 100644 --- a/pkg/cmd/util/downloadrequest/downloadrequest_test.go +++ b/pkg/cmd/util/downloadrequest/downloadrequest_test.go @@ -463,6 +463,7 @@ func TestDownload(t *testing.T) { expectedContent string expectedError bool errorType error + expectedErrMsg string }{ { name: "successful download with gzip for logs", @@ -474,6 +475,16 @@ func TestDownload(t *testing.T) { expectedContent: testContent, expectedError: false, }, + { + name: "error decompressed data exceeds the limit", + serverHandler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(compressedContent.Bytes()) + }, + target: velerov1api.DownloadTargetKindBackupLog, + expectedError: true, + expectedErrMsg: "decompressed data exceeds the limit", + }, { name: "successful download without gzip for backup contents", serverHandler: func(w http.ResponseWriter, r *http.Request) { @@ -506,6 +517,12 @@ func TestDownload(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + originalLimit := unzipLimit + if tc.expectedErrMsg == "decompressed data exceeds the limit" { + unzipLimit = 10 + } + defer func() { unzipLimit = originalLimit }() + server := httptest.NewServer(tc.serverHandler) defer server.Close() @@ -525,6 +542,9 @@ func TestDownload(t *testing.T) { if tc.errorType != nil { assert.Equal(t, tc.errorType, err) } + if tc.expectedErrMsg != "" { + assert.Contains(t, err.Error(), tc.expectedErrMsg) + } } else { require.NoError(t, err) assert.Equal(t, tc.expectedContent, buf.String()) diff --git a/pkg/cmd/util/output/backup_describer.go b/pkg/cmd/util/output/backup_describer.go index 4c8222f81..a8d43b89f 100644 --- a/pkg/cmd/util/output/backup_describer.go +++ b/pkg/cmd/util/output/backup_describer.go @@ -21,7 +21,6 @@ import ( "context" "encoding/json" "fmt" - "io" "sort" "strconv" "strings" @@ -31,7 +30,6 @@ import ( "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/sirupsen/logrus" "github.com/fatih/color" kbclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -94,9 +92,6 @@ func DescribeBackup( if backup.Spec.ResourcePolicy != nil { d.Println() DescribeResourcePolicies(d, backup.Spec.ResourcePolicy) - - // Display fine-grained filter policies if they exist - DescribeFineGrainedFilterPolicies(ctx, kbClient, d, backup) } DescribeGlobalVolumePolicy(d, backup) @@ -151,119 +146,6 @@ func DescribeGlobalVolumePolicy(d *Describer, backup *velerov1api.Backup) { d.Printf("\tName:\t%s\n", name) } -// DescribeFineGrainedFilterPolicies describes cluster-scoped and namespace-scoped filter policies if present -func DescribeFineGrainedFilterPolicies(ctx context.Context, kbClient kbclient.Client, d *Describer, backup *velerov1api.Backup) { - if backup.Spec.ResourcePolicy == nil { - return - } - - // Create a discard logger for the resource policies function since this is CLI output context - discardLogger := logrus.New() - discardLogger.Out = io.Discard - - resourcePolicies, err := resourcepolicies.GetResourcePoliciesFromBackup(*backup, kbClient, discardLogger) - if err != nil { - // Don't fail the describe if we can't read policies, just skip - return - } - - if resourcePolicies == nil { - return - } - - clusterScopedFilterPolicy := resourcePolicies.GetClusterScopedFilterPolicy() - if clusterScopedFilterPolicy != nil { - d.Printf("\nCluster Scoped Filter Policy:\n") - d.Printf(" Resource Filters:\n") - for _, rf := range clusterScopedFilterPolicy.ResourceFilters { - kindsStr := strings.Join(rf.Kinds, ", ") - d.Printf(" %s:\n", kindsStr) - - // Label selector - if len(rf.LabelSelector) > 0 { - selectorStr := formatLabelMap(rf.LabelSelector) - d.Printf(" Label selector: %s\n", selectorStr) - } else if len(rf.OrLabelSelectors) > 0 { - var orStrs []string - for _, ols := range rf.OrLabelSelectors { - orStrs = append(orStrs, formatLabelMap(ols)) - } - d.Printf(" OR label selectors: [%s]\n", strings.Join(orStrs, ", ")) - } else { - d.Printf(" Label selector: \n") - } - - // Name patterns - if len(rf.Names) > 0 { - d.Printf(" Included names: [%s]\n", strings.Join(rf.Names, ", ")) - } else { - d.Printf(" Included names: \n") - } - - if len(rf.ExcludedNames) > 0 { - d.Printf(" Excluded names: [%s]\n", strings.Join(rf.ExcludedNames, ", ")) - } else { - d.Printf(" Excluded names: \n") - } - } - } - - nfPolicies := resourcePolicies.GetNamespacedFilterPolicies() - if len(nfPolicies) > 0 { - d.Printf("\nNamespace-Scoped Filter Policies:\n") - for _, policy := range nfPolicies { - for _, ns := range policy.Namespaces { - d.Printf(" %s:\n", ns) - d.Printf(" Resource Filters:\n") - for _, rf := range policy.ResourceFilters { - var kindsStr string - if rf.IsCatchAll() { - kindsStr = " (all other kinds)" - } else { - kindsStr = strings.Join(rf.Kinds, ", ") - } - d.Printf(" %s:\n", kindsStr) - - // Label selector - if len(rf.LabelSelector) > 0 { - selectorStr := formatLabelMap(rf.LabelSelector) - d.Printf(" Label selector: %s\n", selectorStr) - } else if len(rf.OrLabelSelectors) > 0 { - var orStrs []string - for _, ols := range rf.OrLabelSelectors { - orStrs = append(orStrs, formatLabelMap(ols)) - } - d.Printf(" OR label selectors: [%s]\n", strings.Join(orStrs, ", ")) - } else { - d.Printf(" Label selector: \n") - } - - // Name patterns - if len(rf.Names) > 0 { - d.Printf(" Included names: [%s]\n", strings.Join(rf.Names, ", ")) - } else { - d.Printf(" Included names: \n") - } - - if len(rf.ExcludedNames) > 0 { - d.Printf(" Excluded names: [%s]\n", strings.Join(rf.ExcludedNames, ", ")) - } else { - d.Printf(" Excluded names: \n") - } - } - } - } - } -} - -func formatLabelMap(labelMap map[string]string) string { - var pairs []string - for k, v := range labelMap { - pairs = append(pairs, fmt.Sprintf("%s=%s", k, v)) - } - return strings.Join(pairs, ",") -} - // DescribeUploaderConfigForBackup describes uploader config in human-readable format func DescribeUploaderConfigForBackup(d *Describer, spec velerov1api.BackupSpec) { d.Printf("Uploader config:\n") @@ -857,8 +739,12 @@ func describeDataMovement(d *Describer, details bool, info *volume.BackupVolumeI d.Printf("\t\t\t\tData Mover: %s\n", dataMover) d.Printf("\t\t\t\tUploader Type: %s\n", info.SnapshotDataMovementInfo.UploaderType) d.Printf("\t\t\t\tMoved data Size (bytes): %d\n", info.SnapshotDataMovementInfo.Size) - if info.SnapshotDataMovementInfo.IncrementalSize > 0 { - d.Printf("\t\t\t\tIncremental data Size (bytes): %d\n", info.SnapshotDataMovementInfo.IncrementalSize) + // Print whenever the uploader measured a figure, including zero. A zero-delta + // incremental transfers nothing, which is the whole point of CBT; hiding it + // leaves only the volume size on display and makes the best possible result + // indistinguishable from a full transfer. + if info.SnapshotDataMovementInfo.IncrementalSize != nil { + d.Printf("\t\t\t\tIncremental data Size (bytes): %d\n", *info.SnapshotDataMovementInfo.IncrementalSize) } d.Printf("\t\t\t\tResult: %s\n", info.Result) } else { @@ -1033,7 +919,7 @@ type volumesByPod struct { // Add adds a pod volume with the specified pod namespace, name // and volume to the appropriate group. // Used for both backup and restore -func (v *volumesByPod) Add(namespace, name, volume, phase string, progress veleroapishared.DataMoveOperationProgress, incrementalBytes int64) { +func (v *volumesByPod) Add(namespace, name, volume, phase string, progress veleroapishared.DataMoveOperationProgress, incrementalBytes *int64) { if v.volumesByPodMap == nil { v.volumesByPodMap = make(map[string]*podVolumeGroup) } @@ -1043,8 +929,12 @@ func (v *volumesByPod) Add(namespace, name, volume, phase string, progress veler // append backup progress percentage if backup is in progress if phase == "In Progress" && progress.TotalBytes != 0 { volume = fmt.Sprintf("%s (%.2f%%)", volume, float64(progress.BytesDone)/float64(progress.TotalBytes)*100) - } else if phase == string(velerov1api.PodVolumeBackupPhaseCompleted) && incrementalBytes > 0 { - volume = fmt.Sprintf("%s (size: %v, incremental size: %v)", volume, progress.TotalBytes, incrementalBytes) + } else if phase == string(velerov1api.PodVolumeBackupPhaseCompleted) && incrementalBytes != nil { + // Report the incremental figure whenever it was measured, including zero. Zero is + // the best possible outcome - nothing changed, so nothing was transferred - and + // suppressing it leaves only the volume size on display, which reads as a full + // transfer. + volume = fmt.Sprintf("%s (size: %v, incremental size: %v)", volume, progress.TotalBytes, *incrementalBytes) } else if (phase == string(velerov1api.PodVolumeBackupPhaseCompleted) || phase == string(velerov1api.PodVolumeRestorePhaseCompleted)) && progress.TotalBytes > 0 { diff --git a/pkg/cmd/util/output/backup_describer_test.go b/pkg/cmd/util/output/backup_describer_test.go index 248b0a45b..c64ae04cb 100644 --- a/pkg/cmd/util/output/backup_describer_test.go +++ b/pkg/cmd/util/output/backup_describer_test.go @@ -18,7 +18,6 @@ package output import ( "bytes" - "context" "testing" "text/tabwriter" "time" @@ -26,8 +25,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client/fake" + "k8s.io/utils/ptr" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -632,7 +630,7 @@ func TestCSISnapshots(t *testing.T) { SnapshotHandle: "fake-repo-id-5", OperationID: "fake-operation-5", Size: 100, - IncrementalSize: 50, + IncrementalSize: ptr.To(int64(50)), Phase: velerov2alpha1.DataUploadPhaseFailed, }, }, @@ -897,85 +895,3 @@ func TestDescribeBackupItemOperation(t *testing.T) { d.out.Flush() assert.Equal(t, expected, d.buf.String()) } - -func TestDescribeFineGrainedFilterPolicies(t *testing.T) { - yamlData := ` -version: v1 -clusterScopedFilterPolicy: - resourceFilters: - - kinds: ["StorageClass"] - labelSelector: {"app": "velero"} - - kinds: ["ClusterRole"] - orLabelSelectors: - - {"app": "velero"} - - {"app": "test"} - names: ["role1"] - excludedNames: ["role2"] -namespacedFilterPolicies: -- namespaces: ["ns1", "ns2"] - resourceFilters: - - kinds: ["Pod", "ConfigMap"] - labelSelector: {"app": "velero"} - - kinds: ["*"] -` - cm := &corev1api.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-policy", - Namespace: "velero", - }, - Data: map[string]string{ - "policy.yaml": yamlData, - }, - } - - client := fake.NewClientBuilder().WithRuntimeObjects(cm).Build() - - backup := builder.ForBackup("velero", "test-backup"). - ResourcePolicies("test-policy").Result() - - d := &Describer{ - Prefix: "", - out: &tabwriter.Writer{}, - buf: &bytes.Buffer{}, - } - d.out.Init(d.buf, 0, 8, 2, ' ', 0) - - DescribeFineGrainedFilterPolicies(context.Background(), client, d, backup) - d.out.Flush() - - expected := ` -Cluster Scoped Filter Policy: - Resource Filters: - StorageClass: - Label selector: app=velero - Included names: - Excluded names: - ClusterRole: - OR label selectors: [app=velero, app=test] - Included names: [role1] - Excluded names: [role2] - -Namespace-Scoped Filter Policies: - ns1: - Resource Filters: - Pod, ConfigMap: - Label selector: app=velero - Included names: - Excluded names: - (all other kinds): - Label selector: - Included names: - Excluded names: - ns2: - Resource Filters: - Pod, ConfigMap: - Label selector: app=velero - Included names: - Excluded names: - (all other kinds): - Label selector: - Included names: - Excluded names: -` - assert.Equal(t, expected, d.buf.String()) -} diff --git a/pkg/cmd/util/output/backup_printer.go b/pkg/cmd/util/output/backup_printer.go index 873bc9fc3..420b73129 100644 --- a/pkg/cmd/util/output/backup_printer.go +++ b/pkg/cmd/util/output/backup_printer.go @@ -90,8 +90,12 @@ func printBackup(backup *velerov1api.Backup) []metav1.TableRow { if backup.Status.Expiration != nil { expiration = backup.Status.Expiration.Time } - if expiration.IsZero() && backup.Spec.TTL.Duration > 0 { - expiration = backup.CreationTimestamp.Add(backup.Spec.TTL.Duration) + // Only estimate expiration from TTL after the backup has started. Backups + // stalled in New have no Status.Expiration yet; using CreationTimestamp + // would incorrectly show them as already expired (issue #3555). + if expiration.IsZero() && backup.Spec.TTL.Duration > 0 && + backup.Status.StartTimestamp != nil && !backup.Status.StartTimestamp.Time.IsZero() { + expiration = backup.Status.StartTimestamp.Time.Add(backup.Spec.TTL.Duration) } status := string(backup.Status.Phase) @@ -107,7 +111,7 @@ func printBackup(backup *velerov1api.Backup) []metav1.TableRow { status, backup.Status.Errors, backup.Status.Warnings, - backup.Status.StartTimestamp, + formatTimestamp(backup.Status.StartTimestamp), humanReadableTimeFromNow(expiration), backup.Spec.StorageLocation, queuePosition(backup.Status.QueuePosition), diff --git a/pkg/cmd/util/output/backup_structured_describer.go b/pkg/cmd/util/output/backup_structured_describer.go index dfffcda06..1c0aefa34 100644 --- a/pkg/cmd/util/output/backup_structured_describer.go +++ b/pkg/cmd/util/output/backup_structured_describer.go @@ -21,10 +21,8 @@ import ( "context" "encoding/json" "fmt" - "io" "strings" - "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -57,7 +55,6 @@ func DescribeBackupInSF( if backup.Spec.ResourcePolicy != nil { DescribeResourcePoliciesInSF(d, backup.Spec.ResourcePolicy) - DescribeFineGrainedFilterPoliciesInSF(ctx, kbClient, d, backup) } DescribeGlobalVolumePolicyInSF(d, backup) @@ -228,88 +225,6 @@ func DescribeBackupSpecInSF(d *StructuredDescriber, spec velerov1api.BackupSpec) d.Describe("spec", backupSpecInfo) } -// DescribeFineGrainedFilterPoliciesInSF adds the clusterScopedFilterPolicy -// and namespacedFilterPolicies sections to the structured describer output when present -// in the ResourcePolicy ConfigMap referenced by the backup. -func DescribeFineGrainedFilterPoliciesInSF(ctx context.Context, kbClient kbclient.Client, d *StructuredDescriber, backup *velerov1api.Backup) { - if backup.Spec.ResourcePolicy == nil { - return - } - - discardLogger := logrus.New() - discardLogger.Out = io.Discard - - resPolicies, err := resourcepolicies.GetResourcePoliciesFromBackup(*backup, kbClient, discardLogger) - if err != nil || resPolicies == nil { - return - } - - clusterScopedFilterPolicy := resPolicies.GetClusterScopedFilterPolicy() - if clusterScopedFilterPolicy != nil { - var clusterScopedFilters []map[string]any - for _, rf := range clusterScopedFilterPolicy.ResourceFilters { - entry := map[string]any{ - "kinds": rf.Kinds, - } - if len(rf.LabelSelector) > 0 { - entry["labelSelector"] = rf.LabelSelector - } - if len(rf.OrLabelSelectors) > 0 { - entry["orLabelSelectors"] = rf.OrLabelSelectors - } - if len(rf.Names) > 0 { - entry["names"] = rf.Names - } - if len(rf.ExcludedNames) > 0 { - entry["excludedNames"] = rf.ExcludedNames - } - clusterScopedFilters = append(clusterScopedFilters, entry) - } - d.Describe("clusterScopedFilterPolicy", map[string]any{ - "resourceFilters": clusterScopedFilters, - }) - } - - nfPolicies := resPolicies.GetNamespacedFilterPolicies() - if len(nfPolicies) == 0 { - return - } - - var structuredPolicies []map[string]any - for _, policy := range nfPolicies { - for _, ns := range policy.Namespaces { - var rfEntries []map[string]any - for _, rf := range policy.ResourceFilters { - entry := map[string]any{} - if rf.IsCatchAll() { - entry["kinds"] = []string{} - entry["isCatchAll"] = true - } else { - entry["kinds"] = rf.Kinds - } - if len(rf.LabelSelector) > 0 { - entry["labelSelector"] = rf.LabelSelector - } - if len(rf.OrLabelSelectors) > 0 { - entry["orLabelSelectors"] = rf.OrLabelSelectors - } - if len(rf.Names) > 0 { - entry["names"] = rf.Names - } - if len(rf.ExcludedNames) > 0 { - entry["excludedNames"] = rf.ExcludedNames - } - rfEntries = append(rfEntries, entry) - } - structuredPolicies = append(structuredPolicies, map[string]any{ - "namespace": ns, - "resourceFilters": rfEntries, - }) - } - } - d.Describe("namespacedFilterPolicies", structuredPolicies) -} - // DescribeBackupStatusInSF describes a backup status in structured format. func DescribeBackupStatusInSF(ctx context.Context, kbClient kbclient.Client, d *StructuredDescriber, backup *velerov1api.Backup, details bool, insecureSkipTLSVerify bool, caCertPath string, podVolumeBackups []velerov1api.PodVolumeBackup) { @@ -552,9 +467,13 @@ func describeDataMovementInSF(details bool, info *volume.BackupVolumeInfo, snaps dataMovement["uploaderType"] = info.SnapshotDataMovementInfo.UploaderType dataMovement["result"] = string(info.Result) - if info.SnapshotDataMovementInfo.Size > 0 || info.SnapshotDataMovementInfo.IncrementalSize > 0 { + if info.SnapshotDataMovementInfo.Size > 0 { dataMovement["size"] = info.SnapshotDataMovementInfo.Size - dataMovement["incrementalSize"] = info.SnapshotDataMovementInfo.IncrementalSize + } + // Emit whenever measured, including zero - a zero-delta incremental transferred + // nothing, and that has to be reportable rather than absent. + if info.SnapshotDataMovementInfo.IncrementalSize != nil { + dataMovement["incrementalSize"] = *info.SnapshotDataMovementInfo.IncrementalSize } snapshotDetail["dataMovement"] = dataMovement diff --git a/pkg/cmd/util/output/backup_structured_describer_test.go b/pkg/cmd/util/output/backup_structured_describer_test.go index cb46a4676..88af0f95f 100644 --- a/pkg/cmd/util/output/backup_structured_describer_test.go +++ b/pkg/cmd/util/output/backup_structured_describer_test.go @@ -17,7 +17,6 @@ limitations under the License. package output import ( - "context" "reflect" "testing" "time" @@ -25,8 +24,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -731,96 +728,3 @@ func TestDescribeDeleteBackupRequestsInSF(t *testing.T) { }) } } - -func TestDescribeFineGrainedFilterPoliciesInSF(t *testing.T) { - yamlData := ` -version: v1 -clusterScopedFilterPolicy: - resourceFilters: - - kinds: ["StorageClass"] - labelSelector: {"app": "velero"} - - kinds: ["ClusterRole"] - orLabelSelectors: - - {"app": "velero"} - - {"app": "test"} - names: ["role1"] - excludedNames: ["role2"] -namespacedFilterPolicies: -- namespaces: ["ns1", "ns2"] - resourceFilters: - - kinds: ["Pod", "ConfigMap"] - labelSelector: {"app": "velero"} - - kinds: ["*"] -` - cm := &corev1api.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-policy", - Namespace: "velero", - }, - Data: map[string]string{ - "policy.yaml": yamlData, - }, - } - - client := fake.NewClientBuilder().WithRuntimeObjects(cm).Build() - - backup := builder.ForBackup("velero", "test-backup"). - ResourcePolicies("test-policy").Result() - - sd := &StructuredDescriber{ - output: make(map[string]any), - format: "", - } - - DescribeFineGrainedFilterPoliciesInSF(context.Background(), client, sd, backup) - - expect := map[string]any{ - "clusterScopedFilterPolicy": map[string]any{ - "resourceFilters": []map[string]any{ - { - "kinds": []string{"StorageClass"}, - "labelSelector": map[string]string{"app": "velero"}, - }, - { - "kinds": []string{"ClusterRole"}, - "orLabelSelectors": []map[string]string{ - {"app": "velero"}, - {"app": "test"}, - }, - "names": []string{"role1"}, - "excludedNames": []string{"role2"}, - }, - }, - }, - "namespacedFilterPolicies": []map[string]any{ - { - "namespace": "ns1", - "resourceFilters": []map[string]any{ - { - "kinds": []string{"Pod", "ConfigMap"}, - "labelSelector": map[string]string{"app": "velero"}, - }, - { - "kinds": []string{}, - "isCatchAll": true, - }, - }, - }, - { - "namespace": "ns2", - "resourceFilters": []map[string]any{ - { - "kinds": []string{"Pod", "ConfigMap"}, - "labelSelector": map[string]string{"app": "velero"}, - }, - { - "kinds": []string{}, - "isCatchAll": true, - }, - }, - }, - }, - } - - assert.True(t, reflect.DeepEqual(sd.output, expect)) -} diff --git a/pkg/cmd/util/output/output.go b/pkg/cmd/util/output/output.go index 9dfca040b..9c46030f2 100644 --- a/pkg/cmd/util/output/output.go +++ b/pkg/cmd/util/output/output.go @@ -248,3 +248,17 @@ func NewPrinter(cmd *cobra.Command) (printers.ResourcePrinter, error) { return printer, nil } + +// formatTimestamp renders an optional timestamp for a table cell. +// +// Appending a nil *metav1.Time to a row prints "", which reaches the user +// for any object that has not reached the phase that sets the field: a backup +// that failed validation never gets a start time, and a restore that failed +// validation gets neither a start nor a completion time. An unset timestamp +// shows as "n/a" instead, matching humanReadableTimeFromNow in the same row. +func formatTimestamp(t *metav1.Time) string { + if t == nil || t.IsZero() { + return "n/a" + } + return t.String() +} diff --git a/pkg/cmd/util/output/printer_timestamp_test.go b/pkg/cmd/util/output/printer_timestamp_test.go new file mode 100644 index 000000000..e38d487c4 --- /dev/null +++ b/pkg/cmd/util/output/printer_timestamp_test.go @@ -0,0 +1,150 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package output + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" +) + +func TestFormatTimestamp(t *testing.T) { + set := metav1.NewTime(time.Date(2026, 8, 8, 21, 6, 28, 0, time.UTC)) + + tests := []struct { + name string + input *metav1.Time + want string + }{ + { + name: "nil renders as n/a", + input: nil, + want: "n/a", + }, + { + name: "zero value renders as n/a", + input: &metav1.Time{}, + want: "n/a", + }, + { + name: "a set timestamp is unchanged", + input: &set, + want: set.String(), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, formatTimestamp(tc.input)) + }) + } +} + +// A backup that fails validation never starts, so StartTimestamp stays nil. +func TestPrintBackupWithoutStartTimestamp(t *testing.T) { + backup := &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "failed-validation"}, + Status: velerov1api.BackupStatus{ + Phase: velerov1api.BackupPhaseFailedValidation, + }, + } + + rows := printBackup(backup) + require.Len(t, rows, 1) + + // Name, Status, Errors, Warnings, Created, ... + assert.Equal(t, "n/a", rows[0].Cells[4], "unset start time should not print as ") + assert.Equal(t, string(velerov1api.BackupPhaseFailedValidation), rows[0].Cells[1]) +} + +func TestPrintBackupExpiresForStalledNewBackup(t *testing.T) { + created := metav1.NewTime(time.Now().Add(-20 * 24 * time.Hour)) + backup := &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "clusterstate-20210128123759", + CreationTimestamp: created, + }, + Spec: velerov1api.BackupSpec{ + TTL: metav1.Duration{Duration: 10 * 24 * time.Hour}, + }, + Status: velerov1api.BackupStatus{ + Phase: velerov1api.BackupPhaseNew, + }, + } + + rows := printBackup(backup) + require.Len(t, rows, 1) + assert.Equal(t, "n/a", rows[0].Cells[5], "stalled New backup should not show expiration in the past") +} + +func TestPrintBackupWithStartTimestamp(t *testing.T) { + started := metav1.NewTime(time.Date(2026, 8, 8, 21, 6, 28, 0, time.UTC)) + backup := &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "completed"}, + Status: velerov1api.BackupStatus{ + Phase: velerov1api.BackupPhaseCompleted, + StartTimestamp: &started, + }, + } + + rows := printBackup(backup) + require.Len(t, rows, 1) + assert.Equal(t, started.String(), rows[0].Cells[4]) +} + +// A restore that fails validation gets neither timestamp. +func TestPrintRestoreWithoutTimestamps(t *testing.T) { + restore := &velerov1api.Restore{ + ObjectMeta: metav1.ObjectMeta{Name: "failed-validation"}, + Spec: velerov1api.RestoreSpec{BackupName: "does-not-exist"}, + Status: velerov1api.RestoreStatus{ + Phase: velerov1api.RestorePhaseFailedValidation, + }, + } + + rows := printRestore(restore) + require.Len(t, rows, 1) + + // Name, Backup, Status, Started, Completed, ... + assert.Equal(t, "n/a", rows[0].Cells[3], "unset start time should not print as ") + assert.Equal(t, "n/a", rows[0].Cells[4], "unset completion time should not print as ") +} + +func TestPrintRestoreWithTimestamps(t *testing.T) { + started := metav1.NewTime(time.Date(2026, 8, 8, 21, 9, 40, 0, time.UTC)) + completed := metav1.NewTime(time.Date(2026, 8, 8, 21, 9, 41, 0, time.UTC)) + + restore := &velerov1api.Restore{ + ObjectMeta: metav1.ObjectMeta{Name: "completed"}, + Spec: velerov1api.RestoreSpec{BackupName: "nightly-1"}, + Status: velerov1api.RestoreStatus{ + Phase: velerov1api.RestorePhaseCompleted, + StartTimestamp: &started, + CompletionTimestamp: &completed, + }, + } + + rows := printRestore(restore) + require.Len(t, rows, 1) + assert.Equal(t, started.String(), rows[0].Cells[3]) + assert.Equal(t, completed.String(), rows[0].Cells[4]) +} diff --git a/pkg/cmd/util/output/restore_describer.go b/pkg/cmd/util/output/restore_describer.go index c33da9f69..11e8ff4e4 100644 --- a/pkg/cmd/util/output/restore_describer.go +++ b/pkg/cmd/util/output/restore_describer.go @@ -219,6 +219,10 @@ func DescribeRestore( DescribeResourceModifier(d, restore.Spec.ResourceModifier) } + if boolptr.IsSetToTrue(restore.Spec.SkipDefaultResourceModifier) { + d.Printf("Skip Default Resource Modifier:\ttrue\n") + } + if restore.Spec.ResourcePolicy != nil { d.Println() DescribeResourcePolicies(d, restore.Spec.ResourcePolicy) @@ -413,7 +417,7 @@ func describePodVolumeRestores(d *Describer, restores []velerov1api.PodVolumeRes restoresByPod := new(volumesByPod) for _, restore := range restoresByPhase[phase] { - restoresByPod.Add(restore.Spec.Pod.Namespace, restore.Spec.Pod.Name, restore.Spec.Volume, phase, restore.Status.Progress, 0) + restoresByPod.Add(restore.Spec.Pod.Namespace, restore.Spec.Pod.Name, restore.Spec.Volume, phase, restore.Status.Progress, nil) } d.Printf("\t%s:\n", phase) diff --git a/pkg/cmd/util/output/restore_printer.go b/pkg/cmd/util/output/restore_printer.go index 782eb3485..d9b35a3cb 100644 --- a/pkg/cmd/util/output/restore_printer.go +++ b/pkg/cmd/util/output/restore_printer.go @@ -62,8 +62,8 @@ func printRestore(restore *v1.Restore) []metav1.TableRow { restore.Name, restore.Spec.BackupName, status, - restore.Status.StartTimestamp, - restore.Status.CompletionTimestamp, + formatTimestamp(restore.Status.StartTimestamp), + formatTimestamp(restore.Status.CompletionTimestamp), restore.Status.Errors, restore.Status.Warnings, restore.CreationTimestamp.Time, diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 74b857fd2..569ff18d1 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -58,6 +58,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/collections" + "github.com/vmware-tanzu/velero/pkg/util/datamover" "github.com/vmware-tanzu/velero/pkg/util/encode" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/logging" @@ -84,33 +85,34 @@ var autoExcludeClusterScopedResources = []string{ } type backupReconciler struct { - ctx context.Context - logger logrus.FieldLogger - discoveryHelper discovery.Helper - backupper pkgbackup.Backupper - kbClient kbclient.Client - clock clock.WithTickerAndDelayedExecution - backupLogLevel logrus.Level - newPluginManager func(logrus.FieldLogger) clientmgmt.Manager - backupTracker BackupTracker - defaultBackupLocation string - defaultVolumesToFsBackup bool - defaultBackupTTL time.Duration - defaultVGSLabelKey string - defaultCSISnapshotTimeout time.Duration - resourceTimeout time.Duration - defaultItemOperationTimeout time.Duration - defaultSnapshotLocations map[string]string - metrics *metrics.ServerMetrics - backupStoreGetter persistence.ObjectBackupStoreGetter - formatFlag logging.Format - credentialFileStore credentials.FileStore - maxConcurrentK8SConnections int - defaultSnapshotMoveData bool - globalCRClient kbclient.Client - itemBlockWorkerCount int - concurrentBackups int - globalVolumePoliciesConfigMap string + ctx context.Context + logger logrus.FieldLogger + discoveryHelper discovery.Helper + backupper pkgbackup.Backupper + kbClient kbclient.Client + clock clock.WithTickerAndDelayedExecution + backupLogLevel logrus.Level + newPluginManager func(logrus.FieldLogger) clientmgmt.Manager + backupTracker BackupTracker + defaultBackupLocation string + defaultVolumesToFsBackup bool + defaultBackupTTL time.Duration + defaultVGSLabelKey string + defaultCSISnapshotTimeout time.Duration + resourceTimeout time.Duration + defaultItemOperationTimeout time.Duration + defaultSnapshotLocations map[string]string + metrics *metrics.ServerMetrics + backupStoreGetter persistence.ObjectBackupStoreGetter + formatFlag logging.Format + credentialFileStore credentials.FileStore + maxConcurrentK8SConnections int + defaultSnapshotMoveData bool + globalCRClient kbclient.Client + itemBlockWorkerCount int + concurrentBackups int + globalVolumePoliciesConfigMap string + knownSchedulesWithSuccessfulBackup sets.Set[string] } func NewBackupReconciler( @@ -204,28 +206,43 @@ func (b *backupReconciler) updateTotalBackupMetric() { time.Sleep(5 * time.Second) wait.Until( - func() { - // recompute backup_total metric - backups := &velerov1api.BackupList{} - err := b.kbClient.List(context.Background(), backups, &kbclient.ListOptions{LabelSelector: labels.Everything()}) - if err != nil { - b.logger.Error(err, "Error computing backup_total metric") - } else { - b.metrics.SetBackupTotal(int64(len(backups.Items))) - } - - // recompute backup_last_successful_timestamp metric for each - // schedule (including the empty schedule, i.e. ad-hoc backups) - for schedule, timestamp := range getLastSuccessBySchedule(backups.Items) { - b.metrics.SetBackupLastSuccessfulTimestamp(schedule, timestamp) - } - }, + b.resyncBackupMetrics, backupResyncPeriod, b.ctx.Done(), ) }() } +func (b *backupReconciler) resyncBackupMetrics() { + backups := &velerov1api.BackupList{} + err := b.kbClient.List(context.Background(), backups, &kbclient.ListOptions{LabelSelector: labels.Everything()}) + if err != nil { + b.logger.Error(err, "Error computing backup_total metric") + return + } + + b.metrics.SetBackupTotal(int64(len(backups.Items))) + + currentSchedules := getLastSuccessBySchedule(backups.Items) + for schedule, timestamp := range currentSchedules { + b.metrics.SetBackupLastSuccessfulTimestamp(schedule, timestamp) + } + + // Remove metrics for schedules that no longer have successful backups + if b.knownSchedulesWithSuccessfulBackup != nil { + for schedule := range b.knownSchedulesWithSuccessfulBackup { + if _, exists := currentSchedules[schedule]; !exists { + b.metrics.DeleteBackupLastSuccessfulTimestamp(schedule) + } + } + } + + b.knownSchedulesWithSuccessfulBackup = sets.New[string]() + for schedule := range currentSchedules { + b.knownSchedulesWithSuccessfulBackup.Insert(schedule) + } +} + // getLastSuccessBySchedule finds the most recent completed backup for each schedule // and returns a map of schedule name -> completion time of the most recent completed // backup. This map includes an entry for ad-hoc/non-scheduled backups, where the key @@ -415,6 +432,10 @@ func (b *backupReconciler) prepareBackupRequest(ctx context.Context, backup *vel request.Spec.BackupType = velerov1api.BackupTypeIncremental } + if len(request.Spec.DataMover) == 0 || request.Spec.DataMover == datamover.DataMoverTypeVelero { + request.Spec.DataMover = datamover.GetDefaultBuiltInDataMover() + } + // calculate expiration request.Status.Expiration = &metav1.Time{Time: b.clock.Now().Add(request.Spec.TTL.Duration)} diff --git a/pkg/controller/backup_controller_test.go b/pkg/controller/backup_controller_test.go index bab98efb6..13bac2e4c 100644 --- a/pkg/controller/backup_controller_test.go +++ b/pkg/controller/backup_controller_test.go @@ -31,6 +31,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -63,6 +64,7 @@ import ( ibav1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/itemblockaction/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/datamover" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/logging" ) @@ -804,6 +806,7 @@ func TestProcessBackupCompletions(t *testing.T) { ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, BackupType: velerov1api.BackupTypeIncremental, + DataMover: datamover.GetDefaultBuiltInDataMover(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -845,6 +848,7 @@ func TestProcessBackupCompletions(t *testing.T) { ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, BackupType: velerov1api.BackupTypeIncremental, + DataMover: datamover.GetDefaultBuiltInDataMover(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -890,6 +894,7 @@ func TestProcessBackupCompletions(t *testing.T) { ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, BackupType: velerov1api.BackupTypeIncremental, + DataMover: datamover.GetDefaultBuiltInDataMover(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -932,6 +937,7 @@ func TestProcessBackupCompletions(t *testing.T) { ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, BackupType: velerov1api.BackupTypeIncremental, + DataMover: datamover.GetDefaultBuiltInDataMover(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -974,6 +980,7 @@ func TestProcessBackupCompletions(t *testing.T) { ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, BackupType: velerov1api.BackupTypeIncremental, + DataMover: datamover.GetDefaultBuiltInDataMover(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1017,6 +1024,7 @@ func TestProcessBackupCompletions(t *testing.T) { ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, BackupType: velerov1api.BackupTypeIncremental, + DataMover: datamover.GetDefaultBuiltInDataMover(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1060,6 +1068,7 @@ func TestProcessBackupCompletions(t *testing.T) { ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, BackupType: velerov1api.BackupTypeIncremental, + DataMover: datamover.GetDefaultBuiltInDataMover(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1103,6 +1112,7 @@ func TestProcessBackupCompletions(t *testing.T) { ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, BackupType: velerov1api.BackupTypeIncremental, + DataMover: datamover.GetDefaultBuiltInDataMover(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1146,6 +1156,7 @@ func TestProcessBackupCompletions(t *testing.T) { ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, BackupType: velerov1api.BackupTypeIncremental, + DataMover: datamover.GetDefaultBuiltInDataMover(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1190,6 +1201,7 @@ func TestProcessBackupCompletions(t *testing.T) { ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, BackupType: velerov1api.BackupTypeIncremental, + DataMover: datamover.GetDefaultBuiltInDataMover(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFailed, @@ -1234,6 +1246,7 @@ func TestProcessBackupCompletions(t *testing.T) { ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, BackupType: velerov1api.BackupTypeIncremental, + DataMover: datamover.GetDefaultBuiltInDataMover(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFailed, @@ -1278,6 +1291,7 @@ func TestProcessBackupCompletions(t *testing.T) { ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, BackupType: velerov1api.BackupTypeIncremental, + DataMover: datamover.GetDefaultBuiltInDataMover(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1323,6 +1337,7 @@ func TestProcessBackupCompletions(t *testing.T) { ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, BackupType: velerov1api.BackupTypeIncremental, + DataMover: datamover.GetDefaultBuiltInDataMover(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1368,6 +1383,7 @@ func TestProcessBackupCompletions(t *testing.T) { ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, BackupType: velerov1api.BackupTypeIncremental, + DataMover: datamover.GetDefaultBuiltInDataMover(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1413,6 +1429,7 @@ func TestProcessBackupCompletions(t *testing.T) { ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, BackupType: velerov1api.BackupTypeIncremental, + DataMover: datamover.GetDefaultBuiltInDataMover(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1459,6 +1476,7 @@ func TestProcessBackupCompletions(t *testing.T) { ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, BackupType: velerov1api.BackupTypeIncremental, + DataMover: datamover.GetDefaultBuiltInDataMover(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1504,6 +1522,7 @@ func TestProcessBackupCompletions(t *testing.T) { ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, BackupType: velerov1api.BackupTypeIncremental, + DataMover: datamover.GetDefaultBuiltInDataMover(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1555,6 +1574,7 @@ func TestProcessBackupCompletions(t *testing.T) { IncludedNamespaceScopedResources: []string{"pods"}, ExcludedNamespaceScopedResources: append([]string{"secrets"}, autoExcludeNamespaceScopedResources...), BackupType: velerov1api.BackupTypeIncremental, + DataMover: datamover.GetDefaultBuiltInDataMover(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1606,6 +1626,7 @@ func TestProcessBackupCompletions(t *testing.T) { IncludedNamespaceScopedResources: []string{"pods"}, ExcludedNamespaceScopedResources: append([]string{"secrets"}, autoExcludeNamespaceScopedResources...), BackupType: velerov1api.BackupTypeIncremental, + DataMover: datamover.GetDefaultBuiltInDataMover(), }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -2041,6 +2062,48 @@ func Test_getLastSuccessBySchedule(t *testing.T) { } } +// Test_resyncBackupMetrics_prunesStaleTimestamps verifies that resyncBackupMetrics +// removes backupLastSuccessfulTimestamp entries for schedules that no longer have +// any completed backups (e.g. after the schedule and its backups are deleted). +func Test_resyncBackupMetrics_prunesStaleTimestamps(t *testing.T) { + baseTime, err := time.Parse(time.RFC1123, time.RFC1123) + require.NoError(t, err) + + m := metrics.NewServerMetrics() + gauge := m.Metrics()["backup_last_successful_timestamp"] + + activeBackup := builder.ForBackup("velero", "b1"). + ObjectMeta(builder.WithLabels(velerov1api.ScheduleNameLabel, "active-schedule")). + Phase(velerov1api.BackupPhaseCompleted). + CompletionTimestamp(baseTime). + Result() + + deletedBackup := builder.ForBackup("velero", "b2"). + ObjectMeta(builder.WithLabels(velerov1api.ScheduleNameLabel, "deleted-schedule")). + Phase(velerov1api.BackupPhaseCompleted). + CompletionTimestamp(baseTime). + Result() + + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, activeBackup, deletedBackup) + + c := &backupReconciler{ + kbClient: fakeClient, + logger: logrus.StandardLogger(), + metrics: m, + } + + // First resync: sets metrics for both schedules + c.resyncBackupMetrics() + assert.Equal(t, 2, testutil.CollectAndCount(gauge)) + + // Simulate schedule deletion: remove the backup for "deleted-schedule" + require.NoError(t, fakeClient.Delete(t.Context(), deletedBackup)) + + // Second resync: prunes "deleted-schedule" metric, keeps "active-schedule" + c.resyncBackupMetrics() + assert.Equal(t, 1, testutil.CollectAndCount(gauge)) +} + // Unit tests to make sure that the backup's status is updated correctly during reconcile. // To clear up confusion whether status can be updated with Patch alone without status writer and not kbClient.Status().Patch() func TestPatchResourceWorksWithStatus(t *testing.T) { diff --git a/pkg/controller/backup_deletion_controller.go b/pkg/controller/backup_deletion_controller.go index cd74a3a27..416c28cd4 100644 --- a/pkg/controller/backup_deletion_controller.go +++ b/pkg/controller/backup_deletion_controller.go @@ -321,6 +321,10 @@ func (r *backupDeletionReconciler) Reconcile(ctx context.Context, req ctrl.Reque volumeSnapshotters[snapshot.Spec.Location] = volumeSnapshotter } + if snapshot.Status.ProviderSnapshotID == "" { + log.WithField("volumeSnapshot", snapshot.Spec.PersistentVolumeName).Warn("Skipping snapshot deletion: empty ProviderSnapshotID") + continue + } if err := volumeSnapshotter.DeleteSnapshot(snapshot.Status.ProviderSnapshotID); err != nil { errs = append(errs, errors.Wrapf(err, "error deleting snapshot %s", snapshot.Status.ProviderSnapshotID).Error()) } @@ -531,7 +535,7 @@ func (r *backupDeletionReconciler) deleteCSIVolumeSnapshotsIfAny(ctx context.Con } for _, item := range vsList.Items { vs := item - csi.CleanupVolumeSnapshot(&vs, r.Client, log) + csi.CleanupVolumeSnapshot(ctx, &vs, r.Client, log) } } diff --git a/pkg/controller/backup_deletion_controller_test.go b/pkg/controller/backup_deletion_controller_test.go index 24cc65846..d358fbe5e 100644 --- a/pkg/controller/backup_deletion_controller_test.go +++ b/pkg/controller/backup_deletion_controller_test.go @@ -397,6 +397,74 @@ func TestBackupDeletionControllerReconcile(t *testing.T) { // Make sure snapshot was deleted assert.Equal(t, 0, td.volumeSnapshotter.SnapshotsTaken.Len()) }) + t.Run("empty ProviderSnapshotID skips DeleteSnapshot call", func(t *testing.T) { + input := defaultTestDbr() + + backup := builder.ForBackup(velerov1api.DefaultNamespace, input.Spec.BackupName).Result() + backup.UID = "uid" + backup.Spec.StorageLocation = "primary" + + restore1 := builder.ForRestore(backup.Namespace, "restore-1"). + Phase(velerov1api.RestorePhaseCompleted). + Backup(backup.Name). + Result() + + location := &velerov1api.BackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: backup.Namespace, + Name: "primary", + }, + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "objStoreProvider", + StorageType: velerov1api.StorageType{ + ObjectStorage: &velerov1api.ObjectStorageLocation{ + Bucket: "bucket", + }, + }, + }, + Status: velerov1api.BackupStorageLocationStatus{ + Phase: velerov1api.BackupStorageLocationPhaseAvailable, + }, + } + + snapshotLocation := &velerov1api.VolumeSnapshotLocation{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: backup.Namespace, + Name: "vsl-1", + }, + Spec: velerov1api.VolumeSnapshotLocationSpec{ + Provider: "provider-1", + }, + } + td := setupBackupDeletionControllerTest(t, input, backup, restore1, location, snapshotLocation) + + snapshots := []*volume.Snapshot{ + { + Spec: volume.SnapshotSpec{ + Location: "vsl-1", + PersistentVolumeName: "pv-1", + }, + Status: volume.SnapshotStatus{ + ProviderSnapshotID: "", + }, + }, + } + + pluginManager := &pluginmocks.Manager{} + pluginManager.On("GetVolumeSnapshotter", "provider-1").Return(td.volumeSnapshotter, nil) + pluginManager.On("GetDeleteItemActions").Return(nil, nil) + pluginManager.On("CleanupClients") + td.controller.newPluginManager = func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager } + + td.backupStore.On("GetBackupVolumeSnapshots", input.Spec.BackupName).Return(snapshots, nil) + td.backupStore.On("GetBackupContents", input.Spec.BackupName).Return(io.NopCloser(bytes.NewReader([]byte("hello world"))), nil) + td.backupStore.On("DeleteBackup", input.Spec.BackupName).Return(nil) + + _, err := td.controller.Reconcile(t.Context(), td.req) + require.NoError(t, err) + + td.backupStore.AssertCalled(t, "DeleteBackup", input.Spec.BackupName) + }) t.Run("full delete, no errors, with backup name greater than 63 chars", func(t *testing.T) { backup := defaultBackup(). ObjectMeta( diff --git a/pkg/controller/backup_sync_controller.go b/pkg/controller/backup_sync_controller.go index 07b5b460f..38d79c727 100644 --- a/pkg/controller/backup_sync_controller.go +++ b/pkg/controller/backup_sync_controller.go @@ -164,17 +164,37 @@ func (b *backupSyncReconciler) Reconcile(ctx context.Context, req ctrl.Request) continue } - if backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperations || - backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed || - backup.Status.Phase == velerov1api.BackupPhaseFinalizing || - backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed { + // Only sync backup metadata that has reached a phase Velero itself writes to + // object storage. Anything else (including an empty or New phase) would be + // created in the cluster as a backup that still looks pending, which the backup + // queue controller would then pick up and run as if it were a newly requested + // backup. + switch backup.Status.Phase { + case velerov1api.BackupPhaseCompleted, + velerov1api.BackupPhasePartiallyFailed, + velerov1api.BackupPhaseFailed: + // finished backups are synced as-is + case velerov1api.BackupPhaseWaitingForPluginOperations, + velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed, + velerov1api.BackupPhaseFinalizing, + velerov1api.BackupPhaseFinalizingPartiallyFailed: if backup.Status.Expiration == nil || backup.Status.Expiration.After(time.Now()) { log.Debugf("Skipping non-expired incomplete backup %v", backup.Name) continue } log.Debugf("%v Backup is past expiration, syncing for garbage collection", backup.Status.Phase) backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed + default: + log.Infof("Skipping backup %v, phase %q in the backup store is not a phase that can be synced", backup.Name, backup.Status.Phase) + continue } + + // A synced backup is a record of a backup that already ran somewhere else, not + // a backup to run here. Hooks are only read while a backup is being executed, + // so they have no consumer for a synced backup and are dropped rather than + // stored as an executable payload. + backup.Spec.Hooks = velerov1api.BackupHooks{} + backup.Namespace = b.namespace backup.ResourceVersion = "" @@ -271,11 +291,11 @@ func (b *backupSyncReconciler) filterBackupOwnerReferences(ctx context.Context, case err != nil && apierrors.IsNotFound(err): log.Warnf("Removing missing schedule ownership reference %s/%s from backup", backup.Namespace, v.Name) continue + case err != nil && !apierrors.IsNotFound(err): + log.WithError(errors.WithStack(err)).Error("Error finding schedule ownership reference, keeping schedule on backup") case schedule.UID != v.UID: log.Warnf("Removing schedule ownership reference with mismatched UIDs. Expected %s, got %s", v.UID, schedule.UID) continue - case err != nil && !apierrors.IsNotFound(err): - log.WithError(errors.WithStack(err)).Error("Error finding schedule ownership reference, keeping schedule on backup") } default: log.Warnf("Unable to check ownership reference for unknown kind, %s", v.Kind) diff --git a/pkg/controller/backup_sync_controller_test.go b/pkg/controller/backup_sync_controller_test.go index fe440ff09..75f9c5205 100644 --- a/pkg/controller/backup_sync_controller_test.go +++ b/pkg/controller/backup_sync_controller_test.go @@ -36,6 +36,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" ctrlClient "sigs.k8s.io/controller-runtime/pkg/client" ctrlfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" @@ -203,10 +204,10 @@ var _ = Describe("Backup Sync Reconciler", func() { location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").Result(), + backup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseCompleted).Result(), }, { - backup: builder.ForBackup("ns-1", "backup-2").Result(), + backup: builder.ForBackup("ns-1", "backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), }, }, }, @@ -308,10 +309,10 @@ var _ = Describe("Backup Sync Reconciler", func() { location: defaultLocation("velero"), cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").Result(), + backup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseCompleted).Result(), }, { - backup: builder.ForBackup("ns-1", "backup-2").Result(), + backup: builder.ForBackup("ns-1", "backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), }, }, }, @@ -321,10 +322,10 @@ var _ = Describe("Backup Sync Reconciler", func() { location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").Result(), + backup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseCompleted).Result(), }, { - backup: builder.ForBackup("ns-1", "backup-2").Result(), + backup: builder.ForBackup("ns-1", "backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), }, }, existingBackups: []*velerov1api.Backup{ @@ -340,7 +341,7 @@ var _ = Describe("Backup Sync Reconciler", func() { location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").Result(), + backup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseCompleted).Result(), }, }, existingBackups: []*velerov1api.Backup{ @@ -355,10 +356,10 @@ var _ = Describe("Backup Sync Reconciler", func() { location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").StorageLocation("foo").ObjectMeta(builder.WithLabels(velerov1api.StorageLocationLabel, "foo")).Result(), + backup: builder.ForBackup("ns-1", "backup-1").StorageLocation("foo").ObjectMeta(builder.WithLabels(velerov1api.StorageLocationLabel, "foo")).Phase(velerov1api.BackupPhaseCompleted).Result(), }, { - backup: builder.ForBackup("ns-1", "backup-2").Result(), + backup: builder.ForBackup("ns-1", "backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), }, }, }, @@ -369,10 +370,10 @@ var _ = Describe("Backup Sync Reconciler", func() { longLocationNameEnabled: true, cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").StorageLocation("foo").ObjectMeta(builder.WithLabels(velerov1api.StorageLocationLabel, "foo")).Result(), + backup: builder.ForBackup("ns-1", "backup-1").StorageLocation("foo").ObjectMeta(builder.WithLabels(velerov1api.StorageLocationLabel, "foo")).Phase(velerov1api.BackupPhaseCompleted).Result(), }, { - backup: builder.ForBackup("ns-1", "backup-2").Result(), + backup: builder.ForBackup("ns-1", "backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), }, }, }, @@ -382,13 +383,13 @@ var _ = Describe("Backup Sync Reconciler", func() { location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").Result(), + backup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseCompleted).Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-1").Result(), }, }, { - backup: builder.ForBackup("ns-1", "backup-2").Result(), + backup: builder.ForBackup("ns-1", "backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-2").Result(), }, @@ -401,13 +402,13 @@ var _ = Describe("Backup Sync Reconciler", func() { location: defaultLocation("ns-1"), cloudBackups: []*cloudBackupData{ { - backup: builder.ForBackup("ns-1", "backup-1").Result(), + backup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseCompleted).Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-1").Result(), }, }, { - backup: builder.ForBackup("ns-1", "backup-2").Result(), + backup: builder.ForBackup("ns-1", "backup-2").Phase(velerov1api.BackupPhaseCompleted).Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("ns-1", "pvb-3").Result(), }, @@ -556,6 +557,184 @@ var _ = Describe("Backup Sync Reconciler", func() { } }) + It("Test synced backups are never picked up by the backup queue controller", func() { + fakeClock := testclocks.NewFakeClock(time.Now()) + hooks := velerov1api.BackupHooks{ + Resources: []velerov1api.BackupResourceHookSpec{ + { + Name: "hook-1", + PreHooks: []velerov1api.BackupResourceHook{ + { + Exec: &velerov1api.ExecHook{ + Container: "container-1", + Command: []string{"/bin/sh", "-c", "echo hello"}, + }, + }, + }, + }, + }, + } + + tests := []struct { + name string + cloudBackup *velerov1api.Backup + expectSynced bool + // phase expected in the cluster after the sync and queue reconciles have run. + // only checked when expectSynced is true. + expectPhase velerov1api.BackupPhase + }{ + { + name: "backup metadata with an empty phase is not synced", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Hooks(hooks).Result(), + expectSynced: false, + }, + { + name: "backup metadata in phase New is not synced", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseNew).Hooks(hooks).Result(), + expectSynced: false, + }, + { + name: "backup metadata in phase Queued is not synced", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseQueued).Hooks(hooks).Result(), + expectSynced: false, + }, + { + name: "backup metadata in phase ReadyToStart is not synced", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseReadyToStart).Hooks(hooks).Result(), + expectSynced: false, + }, + { + name: "backup metadata in phase InProgress is not synced", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseInProgress).Hooks(hooks).Result(), + expectSynced: false, + }, + { + name: "backup metadata in phase Deleting is not synced", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseDeleting).Hooks(hooks).Result(), + expectSynced: false, + }, + { + name: "backup metadata in phase Completed is synced and stays Completed", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseCompleted).Hooks(hooks).Result(), + expectSynced: true, + expectPhase: velerov1api.BackupPhaseCompleted, + }, + { + name: "backup metadata in phase PartiallyFailed is synced and stays PartiallyFailed", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhasePartiallyFailed).Result(), + expectSynced: true, + expectPhase: velerov1api.BackupPhasePartiallyFailed, + }, + { + name: "backup metadata in phase Failed is synced and stays Failed", + cloudBackup: builder.ForBackup("ns-1", "backup-1").Phase(velerov1api.BackupPhaseFailed).Result(), + expectSynced: true, + expectPhase: velerov1api.BackupPhaseFailed, + }, + { + name: "non-expired backup waiting for plugin operations is not synced", + cloudBackup: builder.ForBackup("ns-1", "backup-1"). + Phase(velerov1api.BackupPhaseWaitingForPluginOperations). + Expiration(fakeClock.Now().Add(time.Hour)).Result(), + expectSynced: false, + }, + { + name: "expired backup waiting for plugin operations is synced as PartiallyFailed", + cloudBackup: builder.ForBackup("ns-1", "backup-1"). + Phase(velerov1api.BackupPhaseWaitingForPluginOperations). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + expectSynced: true, + expectPhase: velerov1api.BackupPhasePartiallyFailed, + }, + { + name: "expired backup waiting for plugin operations partially failed is synced as PartiallyFailed", + cloudBackup: builder.ForBackup("ns-1", "backup-1"). + Phase(velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + expectSynced: true, + expectPhase: velerov1api.BackupPhasePartiallyFailed, + }, + { + name: "expired finalizing backup is synced as PartiallyFailed", + cloudBackup: builder.ForBackup("ns-1", "backup-1"). + Phase(velerov1api.BackupPhaseFinalizing). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + expectSynced: true, + expectPhase: velerov1api.BackupPhasePartiallyFailed, + }, + { + name: "expired finalizing partially failed backup is synced as PartiallyFailed", + cloudBackup: builder.ForBackup("ns-1", "backup-1"). + Phase(velerov1api.BackupPhaseFinalizingPartiallyFailed). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + expectSynced: true, + expectPhase: velerov1api.BackupPhasePartiallyFailed, + }, + } + + queueScheme := runtime.NewScheme() + Expect(velerov1api.AddToScheme(queueScheme)).ShouldNot(HaveOccurred()) + + for _, test := range tests { + var ( + client = ctrlfake.NewClientBuilder().Build() + pluginManager = &pluginmocks.Manager{} + backupStores = make(map[string]*persistencemocks.BackupStore) + location = defaultLocation("ns-1") + ) + + pluginManager.On("CleanupClients").Return(nil) + syncReconciler := backupSyncReconciler{ + client: client, + namespace: "ns-1", + defaultBackupSyncPeriod: time.Second * 10, + newPluginManager: func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, + backupStoreGetter: NewFakeObjectBackupStoreGetter(backupStores), + logger: velerotest.NewLogger(), + } + + Expect(client.Create(ctx, location)).ShouldNot(HaveOccurred(), test.name) + backupStore := &persistencemocks.BackupStore{} + backupStores[location.Name] = backupStore + backupStore.On("ListBackups").Return([]string{test.cloudBackup.Name}, nil) + backupStore.On("BackupExists", "bucket-1", test.cloudBackup.Name).Return(true, nil) + backupStore.On("GetBackupMetadata", test.cloudBackup.Name).Return(test.cloudBackup, nil) + backupStore.On("GetPodVolumeBackups", test.cloudBackup.Name).Return(nil, nil) + + _, err := syncReconciler.Reconcile(ctx, ctrl.Request{ + NamespacedName: types.NamespacedName{Namespace: location.Namespace, Name: location.Name}, + }) + Expect(err).ShouldNot(HaveOccurred(), test.name) + + backupKey := types.NamespacedName{Namespace: "ns-1", Name: test.cloudBackup.Name} + synced := &velerov1api.Backup{} + err = client.Get(ctx, backupKey, synced) + + if !test.expectSynced { + Expect(apierrors.IsNotFound(err)).To(BeTrue(), test.name) + continue + } + Expect(err).ShouldNot(HaveOccurred(), test.name) + + // Reconcile the synced backup with the queue controller twice: the first + // reconcile would move a New/empty-phase backup to Queued, the second one + // would move it on to ReadyToStart, which is what hands it to the backup + // controller for execution. + queueReconciler := NewBackupQueueReconciler(client, queueScheme, velerotest.NewLogger(), 1, NewBackupTracker()) + for range 2 { + _, err = queueReconciler.Reconcile(ctx, ctrl.Request{NamespacedName: backupKey}) + Expect(err).ShouldNot(HaveOccurred(), test.name) + } + + after := &velerov1api.Backup{} + Expect(client.Get(ctx, backupKey, after)).ShouldNot(HaveOccurred(), test.name) + Expect(after.Status.Phase).To(BeEquivalentTo(test.expectPhase), test.name) + // Hooks are dropped on sync, so the stored metadata cannot carry a payload + // that a later code path could execute. + Expect(after.Spec.Hooks.Resources).To(BeEmpty(), test.name) + } + }) + It("Test deleting orphaned backups.", func() { longLabelName := "the-really-long-location-name-that-is-much-more-than-63-characters" @@ -914,4 +1093,47 @@ var _ = Describe("Backup Sync Reconciler", func() { }) } }) + + It("filterBackupOwnerReferences preserves owner reference on transient API error", func() { + // This test verifies the fix for the switch case ordering bug: + // When client.Get returns a non-NotFound error (e.g. transient API failure), + // the owner reference must be kept on the backup rather than silently dropped + // due to an incorrect UID comparison against a zero-value struct. + scheduleUID := types.UID("schedule-uid-1") + backup := &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-backup", + Namespace: "test-namespace", + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "Schedule", + Name: "my-schedule", + UID: scheduleUID, + }, + }, + }, + } + + // Build a fake client that returns a generic (non-NotFound) error on Get, + // simulating a transient API server failure. + transientErr := fmt.Errorf("transient connection error") + fakeClient := ctrlfake.NewClientBuilder(). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, c ctrlClient.WithWatch, key ctrlClient.ObjectKey, obj ctrlClient.Object, opts ...ctrlClient.GetOption) error { + return transientErr + }, + }). + Build() + + b := backupSyncReconciler{ + client: fakeClient, + } + + logger := velerotest.NewLogger() + references := b.filterBackupOwnerReferences(context.Background(), backup, logger) + + // The owner reference must be preserved when a transient error occurs. + Expect(references).To(HaveLen(1)) + Expect(references[0].UID).To(Equal(scheduleUID)) + }) }) diff --git a/pkg/controller/data_download_controller.go b/pkg/controller/data_download_controller.go index fc7cb1a53..d8062bc72 100644 --- a/pkg/controller/data_download_controller.go +++ b/pkg/controller/data_download_controller.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "strings" + "sync" "time" "github.com/cockroachdb/errors" @@ -58,27 +59,28 @@ import ( // DataDownloadReconciler reconciles a DataDownload object type DataDownloadReconciler struct { - client client.Client - kubeClient kubernetes.Interface - mgr manager.Manager - logger logrus.FieldLogger - Clock clock.WithTickerAndDelayedExecution - restoreExposer exposer.GenericRestoreExposer - nodeName string - dataPathMgr *datapath.Manager - vgdpCounter *exposer.VgdpCounter - loadAffinity []*kube.LoadAffinity - restorePVCConfig velerotypes.RestorePVC - backupRepoConfigs map[string]string - cacheVolumeConfigs *velerotypes.CachePVC - podResources corev1api.ResourceRequirements - preparingTimeout time.Duration - metrics *metrics.ServerMetrics - cancelledDataDownload map[string]time.Time - dataMovePriorityClass string - repoConfigMgr repository.ConfigManager - podLabels map[string]string - podAnnotations map[string]string + client client.Client + kubeClient kubernetes.Interface + mgr manager.Manager + logger logrus.FieldLogger + Clock clock.WithTickerAndDelayedExecution + restoreExposer exposer.GenericRestoreExposer + nodeName string + dataPathMgr *datapath.Manager + vgdpCounter *exposer.VgdpCounter + loadAffinity []*kube.LoadAffinity + restorePVCConfig velerotypes.RestorePVC + backupRepoConfigs map[string]string + cacheVolumeConfigs *velerotypes.CachePVC + podResources corev1api.ResourceRequirements + preparingTimeout time.Duration + metrics *metrics.ServerMetrics + cancelledDataDownload sync.Map + dataMovePriorityClass string + repoConfigMgr repository.ConfigManager + podLabels map[string]string + podAnnotations map[string]string + snapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService } func NewDataDownloadReconciler( @@ -100,29 +102,30 @@ func NewDataDownloadReconciler( repoConfigMgr repository.ConfigManager, podLabels map[string]string, podAnnotations map[string]string, + snapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService, ) *DataDownloadReconciler { return &DataDownloadReconciler{ - client: client, - kubeClient: kubeClient, - mgr: mgr, - logger: logger.WithField("controller", "DataDownload"), - Clock: &clock.RealClock{}, - nodeName: nodeName, - restoreExposer: exposer.NewGenericRestoreExposer(kubeClient, logger), - restorePVCConfig: restorePVCConfig, - backupRepoConfigs: backupRepoConfigs, - cacheVolumeConfigs: cacheVolumeConfigs, - dataPathMgr: dataPathMgr, - vgdpCounter: counter, - loadAffinity: loadAffinity, - podResources: podResources, - preparingTimeout: preparingTimeout, - metrics: metrics, - cancelledDataDownload: make(map[string]time.Time), - dataMovePriorityClass: dataMovePriorityClass, - repoConfigMgr: repoConfigMgr, - podLabels: podLabels, - podAnnotations: podAnnotations, + client: client, + kubeClient: kubeClient, + mgr: mgr, + logger: logger.WithField("controller", "DataDownload"), + Clock: &clock.RealClock{}, + nodeName: nodeName, + restoreExposer: exposer.NewGenericRestoreExposer(kubeClient, client, logger), + restorePVCConfig: restorePVCConfig, + backupRepoConfigs: backupRepoConfigs, + cacheVolumeConfigs: cacheVolumeConfigs, + dataPathMgr: dataPathMgr, + vgdpCounter: counter, + loadAffinity: loadAffinity, + podResources: podResources, + preparingTimeout: preparingTimeout, + metrics: metrics, + dataMovePriorityClass: dataMovePriorityClass, + repoConfigMgr: repoConfigMgr, + podLabels: podLabels, + podAnnotations: podAnnotations, + snapshotMetadataServiceConfigs: snapshotMetadataServiceConfigs, } } @@ -131,6 +134,7 @@ func NewDataDownloadReconciler( // +kubebuilder:rbac:groups="",resources=pods,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumes,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumerclaims,verbs=get +// +kubebuilder:rbac:groups="",resources=secrets;configmaps,verbs=get;list;create;delete func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.logger.WithFields(logrus.Fields{ @@ -198,7 +202,7 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request } } } else { - delete(r.cancelledDataDownload, dd.Name) + r.cancelledDataDownload.Delete(dd.Name) // put the finalizer remove action here for all cr will goes to the final status, we could check finalizer and do remove action in final status // instead of intermediate state. @@ -223,9 +227,9 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request } if dd.Spec.Cancel { - if spotted, found := r.cancelledDataDownload[dd.Name]; !found { - r.cancelledDataDownload[dd.Name] = r.Clock.Now() - } else { + v, loaded := r.cancelledDataDownload.LoadOrStore(dd.Name, r.Clock.Now()) + if loaded { + spotted := v.(time.Time) delay := cancelDelayOthers if dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseInProgress { delay = cancelDelayInProgress @@ -234,7 +238,7 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request if time.Since(spotted) > delay { log.Infof("Data download %s is canceled in Phase %s but not handled in rasonable time", dd.GetName(), dd.Status.Phase) if r.tryCancelDataDownload(ctx, dd, "") { - delete(r.cancelledDataDownload, dd.Name) + r.cancelledDataDownload.Delete(dd.Name) } return ctrl.Result{}, nil @@ -454,7 +458,7 @@ func (r *DataDownloadReconciler) startCancelableDataPath(asyncBR datapath.AsyncB if err := asyncBR.StartRestore(dd.Spec.SnapshotID, datapath.AccessPoint{ ByPath: res.ByPod.VolumeName, - }, dd.Spec.DataMoverConfig); err != nil { + }, dd.Spec.DataMoverConfig, nil); err != nil { return errors.Wrapf(err, "error starting async restore for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName) } @@ -466,7 +470,7 @@ func (r *DataDownloadReconciler) OnDataDownloadCompleted(ctx context.Context, na defer r.dataPathMgr.RemoveAsyncBR(ddName) log := r.logger.WithField("datadownload", ddName) - log.Info("Async fs restore data path completed") + log.Info("Async restore data path completed") var dd velerov2alpha1api.DataDownload if err := r.client.Get(ctx, types.NamespacedName{Name: ddName, Namespace: namespace}, &dd); err != nil { @@ -487,7 +491,9 @@ func (r *DataDownloadReconciler) OnDataDownloadCompleted(ctx context.Context, na } log.Info("Cleaning up exposed environment") - r.restoreExposer.CleanUp(ctx, objRef) + r.restoreExposer.CleanUp(ctx, objRef, &exposer.GenericRestoreCleanUpParam{ + Snapshot: dd.Spec.CSISnapshot, + }) if err := UpdateDataDownloadWithRetry(ctx, r.client, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name}, log, func(dd *velerov2alpha1api.DataDownload) bool { if isDataDownloadInFinalState(dd) { @@ -513,7 +519,7 @@ func (r *DataDownloadReconciler) OnDataDownloadFailed(ctx context.Context, names log := r.logger.WithField("datadownload", ddName) - log.WithError(err).Error("Async fs restore data path failed") + log.WithError(err).Error("Async restore data path failed") var dd velerov2alpha1api.DataDownload if getErr := r.client.Get(ctx, types.NamespacedName{Name: ddName, Namespace: namespace}, &dd); getErr != nil { @@ -528,7 +534,7 @@ func (r *DataDownloadReconciler) OnDataDownloadCancelled(ctx context.Context, na log := r.logger.WithField("datadownload", ddName) - log.Warn("Async fs backup data path canceled") + log.Warn("Async restore data path canceled") var dd velerov2alpha1api.DataDownload if getErr := r.client.Get(ctx, types.NamespacedName{Name: ddName, Namespace: namespace}, &dd); getErr != nil { @@ -536,7 +542,9 @@ func (r *DataDownloadReconciler) OnDataDownloadCancelled(ctx context.Context, na return } // cleans up any objects generated during the snapshot expose - r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(&dd)) + r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(&dd), &exposer.GenericRestoreCleanUpParam{ + Snapshot: dd.Spec.CSISnapshot, + }) if err := UpdateDataDownloadWithRetry(ctx, r.client, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name}, log, func(dd *velerov2alpha1api.DataDownload) bool { if isDataDownloadInFinalState(dd) { @@ -556,7 +564,7 @@ func (r *DataDownloadReconciler) OnDataDownloadCancelled(ctx context.Context, na log.WithError(err).Error("error updating data download status") } else { r.metrics.RegisterDataDownloadCancel(r.nodeName) - delete(r.cancelledDataDownload, dd.Name) + r.cancelledDataDownload.Delete(dd.Name) } } @@ -586,7 +594,9 @@ func (r *DataDownloadReconciler) tryCancelDataDownload(ctx context.Context, dd * // success update r.metrics.RegisterDataDownloadCancel(r.nodeName) - r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(dd)) + r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(dd), &exposer.GenericRestoreCleanUpParam{ + Snapshot: dd.Spec.CSISnapshot, + }) log.Warn("data download is canceled") @@ -693,11 +703,11 @@ func (r *DataDownloadReconciler) findSnapshotRestoreForPod(ctx context.Context, r.prepareDataDownload(dd) return true }); err != nil { - log.WithError(err).Warn("failed to update dataudownload, prepare will halt for this dataudownload") + log.WithError(err).Warn("failed to update datadownload, prepare will halt for this datadownload") return []reconcile.Request{} } } else if unrecoverable, reason := kube.IsPodUnrecoverable(pod, log); unrecoverable { - err := UpdateDataDownloadWithRetry(context.Background(), r.client, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name}, r.logger.WithField("datadownlad", dd.Name), + err := UpdateDataDownloadWithRetry(context.Background(), r.client, types.NamespacedName{Namespace: dd.Namespace, Name: dd.Name}, r.logger.WithField("datadownload", dd.Name), func(dataDownload *velerov2alpha1api.DataDownload) bool { if dataDownload.Spec.Cancel { return false @@ -734,7 +744,9 @@ func (r *DataDownloadReconciler) prepareDataDownload(ssb *velerov2alpha1api.Data func (r *DataDownloadReconciler) errorOut(ctx context.Context, dd *velerov2alpha1api.DataDownload, err error, msg string, log logrus.FieldLogger) (ctrl.Result, error) { if r.restoreExposer != nil { - r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(dd)) + r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(dd), &exposer.GenericRestoreCleanUpParam{ + Snapshot: dd.Spec.CSISnapshot, + }) } return ctrl.Result{}, r.updateStatusToFailed(ctx, dd, err, msg, log) } @@ -824,7 +836,9 @@ func (r *DataDownloadReconciler) onPrepareTimeout(ctx context.Context, dd *veler log.Warnf("[Diagnose DD expose]%s", diag) } - r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(dd)) + r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(dd), &exposer.GenericRestoreCleanUpParam{ + Snapshot: dd.Spec.CSISnapshot, + }) log.Info("Datadownload has been cleaned up") @@ -936,6 +950,7 @@ func (r *DataDownloadReconciler) setupExposeParam(dd *velerov2alpha1api.DataDown return exposer.GenericRestoreExposeParam{ TargetPVCName: dd.Spec.TargetVolume.PVC, + TargetPVName: dd.Spec.TargetVolume.PV, TargetNamespace: dd.Spec.TargetVolume.Namespace, HostingPodLabels: hostingPodLabels, HostingPodAnnotations: hostingPodAnnotation, @@ -950,6 +965,10 @@ func (r *DataDownloadReconciler) setupExposeParam(dd *velerov2alpha1api.DataDown RestoreSize: dd.Spec.SnapshotSize, CacheVolume: cacheVolume, DataMover: dd.Spec.DataMover, + CSI: &exposer.GenericRestoreExposeCSI{ + Snapshot: dd.Spec.CSISnapshot, + SnapshotMetadataServiceConfigs: r.snapshotMetadataServiceConfigs, + }, }, nil } @@ -1096,7 +1115,7 @@ func (r *DataDownloadReconciler) resumeCancellableDataPath(ctx context.Context, if err := asyncBR.StartRestore(dd.Spec.SnapshotID, datapath.AccessPoint{ ByPath: res.ByPod.VolumeName, - }, nil); err != nil { + }, nil, nil); err != nil { return errors.Wrapf(err, "error to resume asyncBR watcher for dd %s", dd.Name) } diff --git a/pkg/controller/data_download_controller_test.go b/pkg/controller/data_download_controller_test.go index 518788635..72d51167b 100644 --- a/pkg/controller/data_download_controller_test.go +++ b/pkg/controller/data_download_controller_test.go @@ -19,9 +19,12 @@ package controller import ( "context" "fmt" + "sync" "testing" "time" + clocktesting "k8s.io/utils/clock/testing" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -61,7 +64,6 @@ func dataDownloadBuilder() *builder.DataDownloadBuilder { BackupStorageLocation("bsl-loc"). DataMover("velero"). SnapshotID("test-snapshot-id").TargetVolume(velerov2alpha1api.TargetVolumeSpec{ - PV: "test-pv", PVC: "test-pvc", Namespace: "test-ns", }) @@ -148,6 +150,7 @@ func initDataDownloadReconcilerWithError(t *testing.T, objects []any, needError nil, nil, // podLabels nil, // podAnnotations + nil, // snapshotMetadataServiceConfigs ), nil } @@ -183,6 +186,7 @@ func TestDataDownloadReconcile(t *testing.T) { dd *velerov2alpha1api.DataDownload notCreateDD bool targetPVC *corev1api.PersistentVolumeClaim + targetPV *corev1api.PersistentVolume dataMgr *datapath.Manager needErrs []bool needCreateFSBR bool @@ -194,6 +198,7 @@ func TestDataDownloadReconcile(t *testing.T) { isPeekExposeErr bool isNilExposer bool notNilExpose bool + mockExpose bool notMockCleanUp bool mockInit bool mockInitErr error @@ -351,6 +356,16 @@ func TestDataDownloadReconcile(t *testing.T) { targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").StorageClass("sc").Result(), expected: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Phase(velerov2alpha1api.DataDownloadPhaseAccepted).Result(), }, + { + name: "dd succeeds for accepted with target PV set", + dd: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).TargetVolume(velerov2alpha1api.TargetVolumeSpec{PVC: "test-pvc", Namespace: "test-ns", PV: "test-pv"}).Result(), + targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").StorageClass("sc").Result(), + targetPV: builder.ForPersistentVolume("test-pv").Result(), + expected: dataDownloadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).TargetVolume(velerov2alpha1api.TargetVolumeSpec{PVC: "test-pvc", Namespace: "test-ns", PV: "test-pv"}).Phase(velerov2alpha1api.DataDownloadPhaseAccepted).Result(), + mockExpose: true, + notMockCleanUp: true, + notNilExpose: true, + }, { name: "prepare timeout on accepted", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseAccepted).Finalizers([]string{DataUploadDownloadFinalizer}).AcceptedTimestamp(&metav1.Time{Time: time.Now().Add(-time.Minute * 30)}).Result(), @@ -487,6 +502,10 @@ func TestDataDownloadReconcile(t *testing.T) { objects = append(objects, test.targetPVC) } + if test.targetPV != nil { + objects = append(objects, test.targetPV) + } + r, err := initDataDownloadReconciler(t, objects, test.needErrs...) require.NoError(t, err) @@ -507,7 +526,7 @@ func TestDataDownloadReconcile(t *testing.T) { } if test.sportTime != nil { - r.cancelledDataDownload[test.dd.Name] = test.sportTime.Time + r.cancelledDataDownload.Store(test.dd.Name, test.sportTime.Time) } if test.constrained { @@ -529,7 +548,7 @@ func TestDataDownloadReconcile(t *testing.T) { } if test.mockStart { - asyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(test.mockStartErr) + asyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.mockStartErr) } if test.mockCancel { @@ -543,7 +562,7 @@ func TestDataDownloadReconcile(t *testing.T) { return asyncBR } - if test.isExposeErr || test.isGetExposeErr || test.isGetExposeNil || test.isPeekExposeErr || test.isNilExposer || test.notNilExpose { + if test.isExposeErr || test.isGetExposeErr || test.isGetExposeNil || test.isPeekExposeErr || test.isNilExposer || test.notNilExpose || test.mockExpose { if test.isNilExposer { r.restoreExposer = nil } else { @@ -551,6 +570,8 @@ func TestDataDownloadReconcile(t *testing.T) { ep := exposermockes.NewGenericRestoreExposer(t) if test.isExposeErr { ep.On("Expose", mock.Anything, mock.Anything, mock.Anything).Return(errors.New("Error to expose restore exposer")) + } else if test.mockExpose { + ep.On("Expose", mock.Anything, mock.Anything, mock.Anything).Return(nil) } else if test.notNilExpose { hostingPod := builder.ForPod("test-ns", "test-name").Volumes(&corev1api.Volume{Name: "test-pvc"}).Result() hostingPod.ObjectMeta.SetUID("test-uid") @@ -565,7 +586,7 @@ func TestDataDownloadReconcile(t *testing.T) { } if !test.notMockCleanUp { - ep.On("CleanUp", mock.Anything, mock.Anything).Return() + ep.On("CleanUp", mock.Anything, mock.Anything, mock.Anything).Return() } return ep }() @@ -624,9 +645,15 @@ func TestDataDownloadReconcile(t *testing.T) { } if test.expectCancelRecord { - assert.Contains(t, r.cancelledDataDownload, test.dd.Name) + _, ok := r.cancelledDataDownload.Load(test.dd.Name) + assert.True(t, ok) } else { - assert.Empty(t, r.cancelledDataDownload) + empty := true + r.cancelledDataDownload.Range(func(key, value any) bool { + empty = false + return false + }) + assert.True(t, empty) } if isDataDownloadInFinalState(&dd) || dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseInProgress { @@ -718,7 +745,7 @@ func TestOnDataDownloadCompleted(t *testing.T) { } else { ep.On("RebindVolume", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) } - ep.On("CleanUp", mock.Anything, mock.Anything).Return() + ep.On("CleanUp", mock.Anything, mock.Anything, mock.Anything).Return() return ep }() @@ -1096,7 +1123,8 @@ func (dt *ddResumeTestHelper) RebindVolume(context.Context, corev1api.ObjectRefe return nil } -func (dt *ddResumeTestHelper) CleanUp(context.Context, corev1api.ObjectReference) {} +func (dt *ddResumeTestHelper) CleanUp(context.Context, corev1api.ObjectReference, *exposer.GenericRestoreCleanUpParam) { +} func (dt *ddResumeTestHelper) newMicroServiceBRWatcher(kbclient.Client, kubernetes.Interface, manager.Manager, string, string, string, string, string, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR { @@ -1288,7 +1316,7 @@ func TestResumeCancellableRestore(t *testing.T) { } if test.mockStart { - mockAsyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(test.startWatcherErr) + mockAsyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.startWatcherErr) } if test.mockClose { @@ -1319,6 +1347,7 @@ func TestDataDownloadSetupExposeParam(t *testing.T) { baseDataDownload := dataDownloadBuilder().Result() baseDataDownload.Namespace = velerov1api.DefaultNamespace + baseDataDownload.Spec.TargetVolume.PV = "pv-1" baseDataDownload.Spec.OperationTimeout = metav1.Duration{Duration: time.Minute * 10} baseDataDownload.Spec.SnapshotSize = 5368709120 // 5Gi @@ -1418,6 +1447,7 @@ func TestDataDownloadSetupExposeParam(t *testing.T) { nil, // repoConfigMgr (unused when cacheVolumeConfigs is nil) tt.args.customLabels, tt.args.customAnnotations, + nil, ) // Act @@ -1428,6 +1458,7 @@ func TestDataDownloadSetupExposeParam(t *testing.T) { // Core fields assert.Equal(t, baseDataDownload.Spec.TargetVolume.PVC, got.TargetPVCName) + assert.Equal(t, baseDataDownload.Spec.TargetVolume.PV, got.TargetPVName) assert.Equal(t, baseDataDownload.Spec.TargetVolume.Namespace, got.TargetNamespace) assert.Equal(t, baseDataDownload.Spec.DataMover, got.DataMover) @@ -1437,3 +1468,50 @@ func TestDataDownloadSetupExposeParam(t *testing.T) { }) } } + +type sequenceClock struct { + *clocktesting.FakeClock + mu sync.Mutex +} + +func (c *sequenceClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + c.FakeClock.Step(time.Second) + return c.FakeClock.Now() +} + +func TestDataDownloadCancelConcurrency(t *testing.T) { + ctx := t.Context() + dd := dataDownloadBuilder().Cancel(true).Phase(velerov2alpha1api.DataDownloadPhaseInProgress).Result() + + r, err := initDataDownloadReconciler(t, nil) + require.NoError(t, err) + + err = r.client.Create(ctx, dd) + require.NoError(t, err) + + firstTime := time.Now() + // manually store the initial time + r.cancelledDataDownload.Store(dd.Name, firstTime) + + // Custom clock that returns a different time each call + r.Clock = &sequenceClock{FakeClock: clocktesting.NewFakeClock(firstTime)} + + var wg sync.WaitGroup + routines := 50 + wg.Add(routines) + + for i := 0; i < routines; i++ { + go func() { + defer wg.Done() + _, _ = r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: dd.Name, Namespace: dd.Namespace}}) + }() + } + + wg.Wait() + + v, ok := r.cancelledDataDownload.Load(dd.Name) + assert.True(t, ok) + assert.Equal(t, firstTime, v.(time.Time), "The initially recorded timestamp should be preserved") +} diff --git a/pkg/controller/data_upload_controller.go b/pkg/controller/data_upload_controller.go index 78e4d1ed3..ae50a7741 100644 --- a/pkg/controller/data_upload_controller.go +++ b/pkg/controller/data_upload_controller.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "strings" + "sync" "time" "github.com/cockroachdb/errors" @@ -81,7 +82,7 @@ type DataUploadReconciler struct { podResources corev1api.ResourceRequirements preparingTimeout time.Duration metrics *metrics.ServerMetrics - cancelledDataUpload map[string]time.Time + cancelledDataUpload sync.Map dataMovePriorityClass string podLabels map[string]string podAnnotations map[string]string @@ -130,7 +131,6 @@ func NewDataUploadReconciler( podResources: podResources, preparingTimeout: preparingTimeout, metrics: metrics, - cancelledDataUpload: make(map[string]time.Time), dataMovePriorityClass: dataMovePriorityClass, podLabels: podLabels, podAnnotations: podAnnotations, @@ -143,6 +143,7 @@ func NewDataUploadReconciler( // +kubebuilder:rbac:groups="",resources=pods,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumes,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumerclaims,verbs=get +// +kubebuilder:rbac:groups="",resources=secrets;configmaps,verbs=get;list;create;delete func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.logger.WithFields(logrus.Fields{ @@ -207,7 +208,7 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } } else { - delete(r.cancelledDataUpload, du.Name) + r.cancelledDataUpload.Delete(du.Name) // put the finalizer remove action here for all cr will goes to the final status, we could check finalizer and do remove action in final status // instead of intermediate state. @@ -232,9 +233,9 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) } if du.Spec.Cancel { - if spotted, found := r.cancelledDataUpload[du.Name]; !found { - r.cancelledDataUpload[du.Name] = r.Clock.Now() - } else { + v, loaded := r.cancelledDataUpload.LoadOrStore(du.Name, r.Clock.Now()) + if loaded { + spotted := v.(time.Time) delay := cancelDelayOthers if du.Status.Phase == velerov2alpha1api.DataUploadPhaseInProgress { delay = cancelDelayInProgress @@ -243,7 +244,7 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) if time.Since(spotted) > delay { log.Infof("Data upload %s is canceled in Phase %s but not handled in reasonable time", du.GetName(), du.Status.Phase) if r.tryCancelDataUpload(ctx, du, "") { - delete(r.cancelledDataUpload, du.Name) + r.cancelledDataUpload.Delete(du.Name) } return ctrl.Result{}, nil @@ -482,7 +483,7 @@ func (r *DataUploadReconciler) OnDataUploadCompleted(ctx context.Context, namesp log := r.logger.WithField("dataupload", duName) - log.Info("Async fs backup data path completed") + log.Info("Async backup data path completed") var du velerov2alpha1api.DataUpload if err := r.client.Get(ctx, types.NamespacedName{Name: duName, Namespace: namespace}, &du); err != nil { @@ -534,7 +535,7 @@ func (r *DataUploadReconciler) OnDataUploadFailed(ctx context.Context, namespace log := r.logger.WithField("dataupload", duName) - log.WithError(err).Error("Async fs backup data path failed") + log.WithError(err).Error("Async backup data path failed") var du velerov2alpha1api.DataUpload if getErr := r.client.Get(ctx, types.NamespacedName{Name: duName, Namespace: namespace}, &du); getErr != nil { @@ -549,7 +550,7 @@ func (r *DataUploadReconciler) OnDataUploadCancelled(ctx context.Context, namesp log := r.logger.WithField("dataupload", duName) - log.Warn("Async fs backup data path canceled") + log.Warn("Async backup data path canceled") du := &velerov2alpha1api.DataUpload{} if getErr := r.client.Get(ctx, types.NamespacedName{Name: duName, Namespace: namespace}, du); getErr != nil { @@ -577,7 +578,7 @@ func (r *DataUploadReconciler) OnDataUploadCancelled(ctx context.Context, namesp log.WithError(err).Error("error updating DataUpload status") } else { r.metrics.RegisterDataUploadCancel(r.nodeName) - delete(r.cancelledDataUpload, du.Name) + r.cancelledDataUpload.Delete(du.Name) } } diff --git a/pkg/controller/data_upload_controller_test.go b/pkg/controller/data_upload_controller_test.go index 9703abe92..30b5926ac 100644 --- a/pkg/controller/data_upload_controller_test.go +++ b/pkg/controller/data_upload_controller_test.go @@ -19,6 +19,7 @@ package controller import ( "context" "fmt" + "sync" "testing" "time" @@ -348,7 +349,7 @@ func (f *fakeFSBR) StartBackup(source datapath.AccessPoint, uploaderConfigs map[ return f.startErr } -func (f *fakeFSBR) StartRestore(snapshotID string, target datapath.AccessPoint, uploaderConfigs map[string]string) error { +func (f *fakeFSBR) StartRestore(snapshotID string, target datapath.AccessPoint, uploaderConfigs map[string]string, param any) error { return nil } @@ -672,7 +673,7 @@ func TestReconcile(t *testing.T) { } if test.sportTime != nil { - r.cancelledDataUpload[test.du.Name] = test.sportTime.Time + r.cancelledDataUpload.Store(test.du.Name, test.sportTime.Time) } if test.constrained { @@ -752,9 +753,15 @@ func TestReconcile(t *testing.T) { } if test.expectCancelRecord { - assert.Contains(t, r.cancelledDataUpload, test.du.Name) + _, ok := r.cancelledDataUpload.Load(test.du.Name) + assert.True(t, ok) } else { - assert.Empty(t, r.cancelledDataUpload) + empty := true + r.cancelledDataUpload.Range(func(key, value any) bool { + empty = false + return false + }) + assert.True(t, empty) } if isDataUploadInFinalState(&du) || du.Status.Phase == velerov2alpha1api.DataUploadPhaseInProgress { @@ -1561,3 +1568,50 @@ func TestDataUploadSetupExposeParam(t *testing.T) { }) } } + +type dataUploadSequenceClock struct { + *testclocks.FakeClock + mu sync.Mutex +} + +func (c *dataUploadSequenceClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + c.FakeClock.Step(time.Second) + return c.FakeClock.Now() +} + +func TestDataUploadCancelConcurrency(t *testing.T) { + ctx := t.Context() + du := dataUploadBuilder().Cancel(true).Phase(velerov2alpha1api.DataUploadPhaseInProgress).Result() + + r, err := initDataUploaderReconciler() + require.NoError(t, err) + + err = r.client.Create(ctx, du) + require.NoError(t, err) + + firstTime := time.Now() + // manually store the initial time + r.cancelledDataUpload.Store(du.Name, firstTime) + + // Custom clock that returns a different time each call + r.Clock = &dataUploadSequenceClock{FakeClock: testclocks.NewFakeClock(firstTime)} + + var wg sync.WaitGroup + routines := 50 + wg.Add(routines) + + for i := 0; i < routines; i++ { + go func() { + defer wg.Done() + _, _ = r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: du.Name, Namespace: du.Namespace}}) + }() + } + + wg.Wait() + + v, ok := r.cancelledDataUpload.Load(du.Name) + assert.True(t, ok) + assert.Equal(t, firstTime, v.(time.Time), "The initially recorded timestamp should be preserved") +} diff --git a/pkg/controller/download_request_controller.go b/pkg/controller/download_request_controller.go index 02d385bec..a2a865251 100644 --- a/pkg/controller/download_request_controller.go +++ b/pkg/controller/download_request_controller.go @@ -18,6 +18,7 @@ package controller import ( "context" + "fmt" "time" "github.com/cockroachdb/errors" @@ -132,11 +133,13 @@ func (r *downloadRequestReconciler) Reconcile(ctx context.Context, req ctrl.Requ // Update the expiration. downloadRequest.Status.Expiration = &metav1.Time{Time: r.clock.Now().Add(persistence.DownloadURLTTL)} - if downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindRestoreLog || + isRestoreTarget := downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindRestoreLog || downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindRestoreResults || downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindRestoreResourceList || downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindRestoreItemOperations || - downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindRestoreVolumeInfo { + downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindRestoreVolumeInfo + + if isRestoreTarget { restore := &velerov1api.Restore{} if err := r.client.Get(ctx, kbclient.ObjectKey{ Namespace: downloadRequest.Namespace, @@ -149,6 +152,16 @@ func (r *downloadRequestReconciler) Reconcile(ctx context.Context, req ctrl.Requ log.Warnf("fail to get restore for DownloadRequest %s. Retry later.", err.Error()) return ctrl.Result{}, errors.WithStack(err) } + + if restorePhaseHasNoArtifacts(restore.Status.Phase) { + msg := fmt.Sprintf("restore %q is in phase %q and has not written any artifacts", + restore.Name, restore.Status.Phase) + log.Infof("%s, not signing a URL", msg) + downloadRequest.Status.Phase = velerov1api.DownloadRequestPhaseFailed + downloadRequest.Status.Message = msg + return ctrl.Result{}, nil + } + backupName = restore.Spec.BackupName } @@ -165,6 +178,15 @@ func (r *downloadRequestReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, errors.WithStack(err) } + if !isRestoreTarget && backupPhaseHasNoArtifacts(backup.Status.Phase) { + msg := fmt.Sprintf("backup %q is in phase %q and has not written any artifacts", + backup.Name, backup.Status.Phase) + log.Infof("%s, not signing a URL", msg) + downloadRequest.Status.Phase = velerov1api.DownloadRequestPhaseFailed + downloadRequest.Status.Message = msg + return ctrl.Result{}, nil + } + location := &velerov1api.BackupStorageLocation{} if err := r.client.Get(ctx, kbclient.ObjectKey{ Namespace: backup.Namespace, @@ -236,3 +258,32 @@ func (r *downloadRequestReconciler) SetupWithManager(mgr ctrl.Manager) error { WatchesRawSource(downloadRequestSource). Complete(r) } + +// backupPhaseHasNoArtifacts reports whether a backup in this phase is known to have +// written nothing to object storage yet, so no DownloadTargetKind can exist for it. +// +// Only pre-execution phases are listed. InProgress and everything after it may have a +// partial log or other artifacts, and Deleting may still have all of them, so those are +// left alone: signing there preserves the behavior callers have today. +func backupPhaseHasNoArtifacts(phase velerov1api.BackupPhase) bool { + switch phase { + case velerov1api.BackupPhaseNew, + velerov1api.BackupPhaseQueued, + velerov1api.BackupPhaseReadyToStart, + velerov1api.BackupPhaseFailedValidation: + return true + default: + return false + } +} + +// restorePhaseHasNoArtifacts is the same test for a restore. +func restorePhaseHasNoArtifacts(phase velerov1api.RestorePhase) bool { + switch phase { + case velerov1api.RestorePhaseNew, + velerov1api.RestorePhaseFailedValidation: + return true + default: + return false + } +} diff --git a/pkg/controller/download_request_phase_test.go b/pkg/controller/download_request_phase_test.go new file mode 100644 index 000000000..ac3b44e35 --- /dev/null +++ b/pkg/controller/download_request_phase_test.go @@ -0,0 +1,277 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + testclocks "k8s.io/utils/clock/testing" + ctrl "sigs.k8s.io/controller-runtime" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + v1crds "github.com/vmware-tanzu/velero/config/crd/v1" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" + persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" + "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" + pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" + velerotest "github.com/vmware-tanzu/velero/pkg/test" +) + +// Expectations are declared once and reused by both the behavior tests and the coverage +// tests below, so a phase can only be tested by being classified here first. +var backupPhaseExpectations = map[velerov1api.BackupPhase]bool{ + // Pre-execution: nothing has been written for any target kind. + velerov1api.BackupPhaseNew: true, + velerov1api.BackupPhaseQueued: true, + velerov1api.BackupPhaseReadyToStart: true, + velerov1api.BackupPhaseFailedValidation: true, + + // From here on there may be a partial log or other artifacts, and Deleting may + // still have all of them, so these keep the behavior callers have today. + velerov1api.BackupPhaseInProgress: false, + velerov1api.BackupPhaseWaitingForPluginOperations: false, + velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed: false, + velerov1api.BackupPhaseFinalizing: false, + velerov1api.BackupPhaseFinalizingPartiallyFailed: false, + velerov1api.BackupPhaseCompleted: false, + velerov1api.BackupPhasePartiallyFailed: false, + velerov1api.BackupPhaseFailed: false, + velerov1api.BackupPhaseDeleting: false, +} + +var restorePhaseExpectations = map[velerov1api.RestorePhase]bool{ + velerov1api.RestorePhaseNew: true, + velerov1api.RestorePhaseFailedValidation: true, + + velerov1api.RestorePhaseInProgress: false, + velerov1api.RestorePhaseWaitingForPluginOperations: false, + velerov1api.RestorePhaseWaitingForPluginOperationsPartiallyFailed: false, + velerov1api.RestorePhaseFinalizing: false, + velerov1api.RestorePhaseFinalizingPartiallyFailed: false, + velerov1api.RestorePhaseCompleted: false, + velerov1api.RestorePhasePartiallyFailed: false, + velerov1api.RestorePhaseFailed: false, +} + +func TestBackupPhaseHasNoArtifacts(t *testing.T) { + for phase, want := range backupPhaseExpectations { + t.Run(string(phase), func(t *testing.T) { + assert.Equal(t, want, backupPhaseHasNoArtifacts(phase)) + }) + } + + // A backup that has not been reconciled yet has an empty phase. It is left alone + // deliberately: the state is transient and the caller can retry. + assert.False(t, backupPhaseHasNoArtifacts(velerov1api.BackupPhase(""))) +} + +func TestRestorePhaseHasNoArtifacts(t *testing.T) { + for phase, want := range restorePhaseExpectations { + t.Run(string(phase), func(t *testing.T) { + assert.Equal(t, want, restorePhaseHasNoArtifacts(phase)) + }) + } + + assert.False(t, restorePhaseHasNoArtifacts(velerov1api.RestorePhase(""))) +} + +// The two tests below read the phase enum out of the generated CRDs, which come from the +// same kubebuilder markers as the Go constants. Adding a phase to the API without +// classifying it here fails, which a hand-written list of phases cannot do. +func TestBackupPhaseExpectationsCoverTheCRD(t *testing.T) { + phases := statusPhaseEnum(t, "backups.velero.io") + require.NotEmpty(t, phases, "no status.phase enum found in the Backup CRD") + + for _, phase := range phases { + _, ok := backupPhaseExpectations[velerov1api.BackupPhase(phase)] + assert.True(t, ok, "BackupPhase %q is served by the CRD but not classified by backupPhaseHasNoArtifacts", phase) + } + assert.Len(t, backupPhaseExpectations, len(phases), "backupPhaseExpectations and the CRD enum have drifted apart") +} + +func TestRestorePhaseExpectationsCoverTheCRD(t *testing.T) { + phases := statusPhaseEnum(t, "restores.velero.io") + require.NotEmpty(t, phases, "no status.phase enum found in the Restore CRD") + + for _, phase := range phases { + _, ok := restorePhaseExpectations[velerov1api.RestorePhase(phase)] + assert.True(t, ok, "RestorePhase %q is served by the CRD but not classified by restorePhaseHasNoArtifacts", phase) + } + assert.Len(t, restorePhaseExpectations, len(phases), "restorePhaseExpectations and the CRD enum have drifted apart") +} + +// statusPhaseEnum returns the allowed values of status.phase for a generated CRD. +func statusPhaseEnum(t *testing.T, crdName string) []string { + t.Helper() + + for _, crd := range v1crds.CRDs { + if crd.Name != crdName { + continue + } + for _, version := range crd.Spec.Versions { + if version.Schema == nil || version.Schema.OpenAPIV3Schema == nil { + continue + } + status, ok := version.Schema.OpenAPIV3Schema.Properties["status"] + if !ok { + continue + } + phase, ok := status.Properties["phase"] + if !ok { + continue + } + + values := make([]string, 0, len(phase.Enum)) + for _, raw := range phase.Enum { + var value string + require.NoError(t, json.Unmarshal(raw.Raw, &value)) + values = append(values, value) + } + return values + } + } + + t.Fatalf("CRD %q not found", crdName) + return nil +} + +// The guard is only useful if a caller can tell why it fired. These reconcile a real +// request against a fake client and assert on what a client would actually observe. +func TestGuardSetsFailedPhaseWithReason(t *testing.T) { + tests := []struct { + name string + targetKind velerov1api.DownloadTargetKind + backupPhase velerov1api.BackupPhase + wantPhase velerov1api.DownloadRequestPhase + wantMessage string + }{ + { + name: "backup that never ran fails with the phase named", + targetKind: velerov1api.DownloadTargetKindBackupLog, + backupPhase: velerov1api.BackupPhaseFailedValidation, + wantPhase: velerov1api.DownloadRequestPhaseFailed, + wantMessage: `backup "a-backup" is in phase "FailedValidation" and has not written any artifacts`, + }, + { + name: "backup that ran is untouched by the guard", + targetKind: velerov1api.DownloadTargetKindBackupLog, + backupPhase: velerov1api.BackupPhaseCompleted, + wantPhase: velerov1api.DownloadRequestPhaseProcessed, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + harness := newDownloadRequestHarness(t, tc.targetKind, tc.backupPhase) + got := harness.reconcile(t) + + assert.Equal(t, tc.wantPhase, got.Status.Phase) + assert.Equal(t, tc.wantMessage, got.Status.Message, + "the message is the only thing telling a caller why no URL arrived") + + if tc.wantPhase == velerov1api.DownloadRequestPhaseFailed { + assert.Empty(t, got.Status.DownloadURL, + "a failed request must not carry a URL that would 404") + } + }) + } +} + +// A message is set only on failure. An empty message alongside Failed would put the CLI +// back on its generic storage-location error, which is the thing this replaces. +func TestFailedPhaseAlwaysCarriesAMessage(t *testing.T) { + harness := newDownloadRequestHarness(t, + velerov1api.DownloadTargetKindBackupLog, velerov1api.BackupPhaseNew) + + got := harness.reconcile(t) + + require.Equal(t, velerov1api.DownloadRequestPhaseFailed, got.Status.Phase) + assert.NotEmpty(t, got.Status.Message) +} + +// downloadRequestHarness builds the smallest cluster a DownloadRequest reconcile needs: +// the request, its backup, and a storage location whose store returns a URL. +type downloadRequestHarness struct { + client kbclient.Client + reqName string + r *downloadRequestReconciler +} + +func newDownloadRequestHarness( + t *testing.T, + kind velerov1api.DownloadTargetKind, + backupPhase velerov1api.BackupPhase, +) *downloadRequestHarness { + t.Helper() + + s := runtime.NewScheme() + require.NoError(t, velerov1api.AddToScheme(s)) + + backup := builder.ForBackup(velerov1api.DefaultNamespace, "a-backup"). + StorageLocation("a-location").Phase(backupPhase).Result() + location := builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "a-location").Result() + request := builder.ForDownloadRequest(velerov1api.DefaultNamespace, "a-request"). + Target(kind, "a-backup").Result() + + c := fake.NewClientBuilder().WithScheme(s). + WithObjects(request, backup, location).Build() + + store := &persistencemocks.BackupStore{} + store.On("GetDownloadURL", request.Spec.Target).Return("a-url", nil) + + pluginManager := &pluginmocks.Manager{} + pluginManager.On("CleanupClients").Return(nil) + + r := NewDownloadRequestReconciler( + c, + testclocks.NewFakeClock(time.Now()), + func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, + NewFakeObjectBackupStoreGetter(map[string]*persistencemocks.BackupStore{"a-location": store}), + velerotest.NewLogger(), + nil, + nil, + ) + + return &downloadRequestHarness{client: c, reqName: request.Name, r: r} +} + +func (h *downloadRequestHarness) reconcile(t *testing.T) *velerov1api.DownloadRequest { + t.Helper() + + _, err := h.r.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: kbclient.ObjectKey{ + Namespace: velerov1api.DefaultNamespace, + Name: h.reqName, + }, + }) + require.NoError(t, err) + + got := &velerov1api.DownloadRequest{} + require.NoError(t, h.client.Get(context.Background(), kbclient.ObjectKey{ + Namespace: velerov1api.DefaultNamespace, Name: h.reqName, + }, got)) + return got +} diff --git a/pkg/controller/gc_controller.go b/pkg/controller/gc_controller.go index 6b3ade484..f477ae9c6 100644 --- a/pkg/controller/gc_controller.go +++ b/pkg/controller/gc_controller.go @@ -156,6 +156,10 @@ func (c *gcReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re if !veleroutil.BSLIsAvailable(*loc) { log.Infof("BSL %s is unavailable, cannot gc backup", loc.Name) + backup.Labels[garbageCollectionFailure] = gcFailureBSLUnavailable + if err := c.Update(ctx, backup); err != nil { + log.WithError(err).Error("error updating backup labels") + } return ctrl.Result{}, fmt.Errorf("bsl %s is unavailable, cannot gc backup", loc.Name) } diff --git a/pkg/controller/gc_controller_test.go b/pkg/controller/gc_controller_test.go index 754b46e0a..be7553888 100644 --- a/pkg/controller/gc_controller_test.go +++ b/pkg/controller/gc_controller_test.go @@ -21,6 +21,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -48,11 +49,12 @@ func TestGCReconcile(t *testing.T) { defaultBackupLocation := builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "default").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() tests := []struct { - name string - backup *velerov1api.Backup - deleteBackupRequests []*velerov1api.DeleteBackupRequest - backupLocation *velerov1api.BackupStorageLocation - expectError bool + name string + backup *velerov1api.Backup + deleteBackupRequests []*velerov1api.DeleteBackupRequest + backupLocation *velerov1api.BackupStorageLocation + expectError bool + expectedGCFailureLabel string }{ { name: "can't find backup - no error", @@ -118,10 +120,11 @@ func TestGCReconcile(t *testing.T) { }, }, { - name: "BSL is unavailable", - backup: defaultBackup().Expiration(fakeClock.Now().Add(-time.Second)).StorageLocation("default").Result(), - backupLocation: builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "default").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result(), - expectError: true, + name: "BSL is unavailable", + backup: defaultBackup().Expiration(fakeClock.Now().Add(-time.Second)).StorageLocation("default").Result(), + backupLocation: builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "default").Phase(velerov1api.BackupStorageLocationPhaseUnavailable).Result(), + expectError: true, + expectedGCFailureLabel: gcFailureBSLUnavailable, }, } @@ -147,6 +150,12 @@ func TestGCReconcile(t *testing.T) { _, err := reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: test.backup.Namespace, Name: test.backup.Name}}) gotErr := err != nil assert.Equal(t, test.expectError, gotErr) + + if test.expectedGCFailureLabel != "" { + updatedBackup := &velerov1api.Backup{} + require.NoError(t, fakeClient.Get(t.Context(), types.NamespacedName{Namespace: test.backup.Namespace, Name: test.backup.Name}, updatedBackup)) + assert.Equal(t, test.expectedGCFailureLabel, updatedBackup.Labels[garbageCollectionFailure]) + } }) } } diff --git a/pkg/controller/pod_volume_backup_controller.go b/pkg/controller/pod_volume_backup_controller.go index 2372bf25b..13dbd5d79 100644 --- a/pkg/controller/pod_volume_backup_controller.go +++ b/pkg/controller/pod_volume_backup_controller.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "strings" + "sync" "time" "github.com/cockroachdb/errors" @@ -89,7 +90,6 @@ func NewPodVolumeBackupReconciler( preparingTimeout: preparingTimeout, resourceTimeout: resourceTimeout, exposer: exposer.NewPodVolumeExposer(kubeClient, logger), - cancelledPVB: make(map[string]time.Time), dataMovePriorityClass: dataMovePriorityClass, privileged: privileged, podLabels: podLabels, @@ -112,7 +112,7 @@ type PodVolumeBackupReconciler struct { vgdpCounter *exposer.VgdpCounter preparingTimeout time.Duration resourceTimeout time.Duration - cancelledPVB map[string]time.Time + cancelledPVB sync.Map dataMovePriorityClass string privileged bool podLabels map[string]string @@ -183,7 +183,7 @@ func (r *PodVolumeBackupReconciler) Reconcile(ctx context.Context, req ctrl.Requ } } } else { - delete(r.cancelledPVB, pvb.Name) + r.cancelledPVB.Delete(pvb.Name) if controllerutil.ContainsFinalizer(pvb, PodVolumeFinalizer) { if err := UpdatePVBWithRetry(ctx, r.client, req.NamespacedName, log, func(pvb *velerov1api.PodVolumeBackup) bool { @@ -204,9 +204,9 @@ func (r *PodVolumeBackupReconciler) Reconcile(ctx context.Context, req ctrl.Requ } if pvb.Spec.Cancel { - if spotted, found := r.cancelledPVB[pvb.Name]; !found { - r.cancelledPVB[pvb.Name] = r.clock.Now() - } else { + v, loaded := r.cancelledPVB.LoadOrStore(pvb.Name, r.clock.Now()) + if loaded { + spotted := v.(time.Time) delay := cancelDelayOthers if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseInProgress { delay = cancelDelayInProgress @@ -215,7 +215,7 @@ func (r *PodVolumeBackupReconciler) Reconcile(ctx context.Context, req ctrl.Requ if time.Since(spotted) > delay { log.Infof("PVB %s is canceled in Phase %s but not handled in reasonable time", pvb.GetName(), pvb.Status.Phase) if r.tryCancelPodVolumeBackup(ctx, pvb, "") { - delete(r.cancelledPVB, pvb.Name) + r.cancelledPVB.Delete(pvb.Name) } return ctrl.Result{}, nil @@ -620,7 +620,7 @@ func (r *PodVolumeBackupReconciler) OnDataPathCancelled(ctx context.Context, nam }); err != nil { log.WithError(err).Error("error updating PVB status on cancel") } else { - delete(r.cancelledPVB, pvb.Name) + r.cancelledPVB.Delete(pvb.Name) } } diff --git a/pkg/controller/pod_volume_backup_controller_test.go b/pkg/controller/pod_volume_backup_controller_test.go index 8b05f0e3b..21e30d5db 100644 --- a/pkg/controller/pod_volume_backup_controller_test.go +++ b/pkg/controller/pod_volume_backup_controller_test.go @@ -19,9 +19,12 @@ package controller import ( "context" "fmt" + "sync" "testing" "time" + clocktesting "k8s.io/utils/clock/testing" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -489,7 +492,7 @@ func TestPVBReconcile(t *testing.T) { } if test.sportTime != nil { - r.cancelledPVB[test.pvb.Name] = test.sportTime.Time + r.cancelledPVB.Store(test.pvb.Name, test.sportTime.Time) } if test.constrained { @@ -567,9 +570,15 @@ func TestPVBReconcile(t *testing.T) { } if test.expectCancelRecord { - assert.Contains(t, r.cancelledPVB, test.pvb.Name) + _, ok := r.cancelledPVB.Load(test.pvb.Name) + assert.True(t, ok) } else { - assert.Empty(t, r.cancelledPVB) + empty := true + r.cancelledPVB.Range(func(key, value any) bool { + empty = false + return false + }) + assert.True(t, empty) } if isPVBInFinalState(&pvb) || pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseInProgress { @@ -1308,3 +1317,50 @@ func TestPodVolumeBackupSetupExposeParam(t *testing.T) { }) } } + +type pvbSequenceClock struct { + *clocktesting.FakeClock + mu sync.Mutex +} + +func (c *pvbSequenceClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + c.FakeClock.Step(time.Second) + return c.FakeClock.Now() +} + +func TestPodVolumeBackupCancelConcurrency(t *testing.T) { + ctx := t.Context() + pvb := builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, "pvb-1").Cancel(true).Phase(velerov1api.PodVolumeBackupPhaseInProgress).Result() + + r, err := initPVBReconciler() + require.NoError(t, err) + + err = r.client.Create(ctx, pvb) + require.NoError(t, err) + + firstTime := time.Now() + // manually store the initial time + r.cancelledPVB.Store(pvb.Name, firstTime) + + // Custom clock that returns a different time each call + r.clock = &pvbSequenceClock{FakeClock: clocktesting.NewFakeClock(firstTime)} + + var wg sync.WaitGroup + routines := 50 + wg.Add(routines) + + for i := 0; i < routines; i++ { + go func() { + defer wg.Done() + _, _ = r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: pvb.Name, Namespace: pvb.Namespace}}) + }() + } + + wg.Wait() + + v, ok := r.cancelledPVB.Load(pvb.Name) + assert.True(t, ok) + assert.Equal(t, firstTime, v.(time.Time), "The initially recorded timestamp should be preserved") +} diff --git a/pkg/controller/pod_volume_restore_controller.go b/pkg/controller/pod_volume_restore_controller.go index 12ba49d10..b6d4985fa 100644 --- a/pkg/controller/pod_volume_restore_controller.go +++ b/pkg/controller/pod_volume_restore_controller.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "strings" + "sync" "time" "github.com/cockroachdb/errors" @@ -90,7 +91,6 @@ func NewPodVolumeRestoreReconciler( preparingTimeout: preparingTimeout, resourceTimeout: resourceTimeout, exposer: exposer.NewPodVolumeExposer(kubeClient, logger), - cancelledPVR: make(map[string]time.Time), dataMovePriorityClass: dataMovePriorityClass, privileged: privileged, repoConfigMgr: repoConfigMgr, @@ -114,7 +114,7 @@ type PodVolumeRestoreReconciler struct { vgdpCounter *exposer.VgdpCounter preparingTimeout time.Duration resourceTimeout time.Duration - cancelledPVR map[string]time.Time + cancelledPVR sync.Map dataMovePriorityClass string privileged bool repoConfigMgr repository.ConfigManager @@ -188,7 +188,7 @@ func (r *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Req } } } else { - delete(r.cancelledPVR, pvr.Name) + r.cancelledPVR.Delete(pvr.Name) if controllerutil.ContainsFinalizer(pvr, PodVolumeFinalizer) { if err := UpdatePVRWithRetry(ctx, r.client, req.NamespacedName, log, func(pvr *velerov1api.PodVolumeRestore) bool { @@ -209,9 +209,9 @@ func (r *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Req } if pvr.Spec.Cancel { - if spotted, found := r.cancelledPVR[pvr.Name]; !found { - r.cancelledPVR[pvr.Name] = r.clock.Now() - } else { + v, loaded := r.cancelledPVR.LoadOrStore(pvr.Name, r.clock.Now()) + if loaded { + spotted := v.(time.Time) delay := cancelDelayOthers if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseInProgress { delay = cancelDelayInProgress @@ -220,7 +220,7 @@ func (r *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Req if time.Since(spotted) > delay { log.Infof("PVR %s is canceled in Phase %s but not handled in rasonable time", pvr.GetName(), pvr.Status.Phase) if r.tryCancelPodVolumeRestore(ctx, pvr, "") { - delete(r.cancelledPVR, pvr.Name) + r.cancelledPVR.Delete(pvr.Name) } return ctrl.Result{}, nil @@ -236,7 +236,15 @@ func (r *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, nil } - shouldProcess, pod, err := shouldProcess(ctx, r.client, log, pvr, r.resourceTimeout) + pod, err := getTargetPod(ctx, r.client, log, pvr) + if err != nil { + return ctrl.Result{}, err + } + if pod == nil { + return ctrl.Result{}, nil + } + + shouldProcess, err := shouldProcess(pod, log) if err != nil { return r.errorOut(ctx, pvr, err, "Pod for this PVR is not ready", log) } @@ -255,12 +263,6 @@ func (r *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, errors.Wrapf(err, "error accepting PVR %s", pvr.Name) } - initContainerIndex := getInitContainerIndex(pod) - if initContainerIndex > 0 { - log.Warnf(`Init containers before the %s container may cause issues - if they interfere with volumes being restored: %s index %d`, restorehelper.WaitInitContainer, restorehelper.WaitInitContainer, initContainerIndex) - } - log.Info("Exposing PVR") exposeParam := r.setupExposeParam(pvr) @@ -528,7 +530,7 @@ func (r *PodVolumeRestoreReconciler) startCancelableDataPath(asyncBR datapath.As if err := asyncBR.StartRestore(pvr.Spec.SnapshotID, datapath.AccessPoint{ ByPath: res.ByPod.VolumeName, - }, pvr.Spec.UploaderSettings); err != nil { + }, pvr.Spec.UploaderSettings, nil); err != nil { return errors.Wrapf(err, "error starting async restore for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName) } @@ -565,71 +567,61 @@ func UpdatePVRStatusToFailed(ctx context.Context, c client.Client, pvr *velerov1 return err } -func shouldProcess(ctx context.Context, client client.Client, log logrus.FieldLogger, pvr *velerov1api.PodVolumeRestore, timeout time.Duration) (bool, *corev1api.Pod, error) { - if !isPVRNew(pvr) { - log.Debug("PVR is not new, skip") - return false, nil, nil - } - +func getTargetPod(ctx context.Context, client client.Client, log logrus.FieldLogger, pvr *velerov1api.PodVolumeRestore) (*corev1api.Pod, error) { // we filter the pods during the initialization of cache, if we can get a pod here, the pod must be in the same node with the controller // so we don't need to compare the node anymore - var targetPod *corev1api.Pod - err := wait.PollUntilContextTimeout(ctx, time.Millisecond*100, timeout, true, func(ctx context.Context) (bool, error) { - updated := &corev1api.Pod{} - if err := client.Get(ctx, types.NamespacedName{Namespace: pvr.Spec.Pod.Namespace, Name: pvr.Spec.Pod.Name}, updated); err != nil { - if apierrors.IsNotFound(err) { - return false, nil - } - - return false, err - } - - targetPod = updated - - return true, nil - }) - - if err != nil { - if errors.Is(err, context.DeadlineExceeded) { - return false, nil, errors.Errorf("timeout to wait for pod %s/%s", pvr.Spec.Pod.Namespace, pvr.Spec.Pod.Name) - } else { - return false, nil, errors.Wrapf(err, "error waiting for pod %s/%s", pvr.Spec.Pod.Namespace, pvr.Spec.Pod.Name) + pod := &corev1api.Pod{} + if err := client.Get(ctx, types.NamespacedName{Namespace: pvr.Spec.Pod.Namespace, Name: pvr.Spec.Pod.Name}, pod); err != nil { + if apierrors.IsNotFound(err) { + log.WithError(err).Debug("Pod not found on this node, skip") + return nil, nil } + log.WithError(err).Error("Unable to get pod") + return nil, err } + return pod, nil +} + +func shouldProcess(targetPod *corev1api.Pod, log logrus.FieldLogger) (bool, error) { if targetPod.Status.Phase == corev1api.PodFailed || targetPod.Status.Phase == corev1api.PodUnknown { - return false, nil, errors.Errorf("unexpected state for pod %s/%s", targetPod.Namespace, targetPod.Name) + return false, errors.Errorf("unexpected state for pod %s/%s", targetPod.Namespace, targetPod.Name) } idx := getInitContainerIndex(targetPod) if idx < 0 { - return false, nil, errors.Errorf("no restore-wait init container in pod %s/%s", targetPod.Namespace, targetPod.Name) + return false, errors.Errorf("no restore-wait init container in pod %s/%s", targetPod.Namespace, targetPod.Name) } if len(targetPod.Status.InitContainerStatuses) <= idx { log.Debug("Pod init container statuses are not fully populated yet, skip") - return false, nil, nil + return false, nil + } + + if idx > 0 { + log.Warnf(`Init containers before the %s container may cause issues + if they interfere with volumes being restored: %s index %d`, restorehelper.WaitInitContainer, restorehelper.WaitInitContainer, idx) } containerStatus := targetPod.Status.InitContainerStatuses[idx] if containerStatus.State.Terminated != nil { - return false, nil, errors.Errorf("restore-wait init container has already completed in pod %s/%s", targetPod.Namespace, targetPod.Name) + return false, errors.Errorf("restore-wait init container has already completed in pod %s/%s", targetPod.Namespace, targetPod.Name) } if containerStatus.State.Waiting != nil { reason := containerStatus.State.Waiting.Reason if reason == "ImagePullBackOff" || reason == "ErrImageNeverPull" || reason == "CreateContainerConfigError" || reason == "CreateContainerError" || reason == "InvalidImageName" || reason == "ErrImagePull" { - return false, nil, errors.Errorf("restore-wait init container in pod %s/%s is in unrecoverable waiting state with reason %s", targetPod.Namespace, targetPod.Name, reason) + return false, errors.Errorf("restore-wait init container in pod %s/%s is in unrecoverable waiting state with reason %s", targetPod.Namespace, targetPod.Name, reason) } } if containerStatus.State.Running == nil { log.Debug("Pod is not running restore-wait init container, skip") - return false, nil, nil + return false, nil } - return true, targetPod, nil + return true, nil } func (r *PodVolumeRestoreReconciler) closeDataPath(ctx context.Context, pvrName string) { @@ -903,7 +895,7 @@ func (r *PodVolumeRestoreReconciler) OnDataPathCancelled(ctx context.Context, na }); err != nil { log.WithError(err).Error("error updating PVR status on cancel") } else { - delete(r.cancelledPVR, pvr.Name) + r.cancelledPVR.Delete(pvr.Name) } } @@ -1146,7 +1138,7 @@ func (r *PodVolumeRestoreReconciler) resumeCancellableDataPath(ctx context.Conte if err := asyncBR.StartRestore(pvr.Spec.SnapshotID, datapath.AccessPoint{ ByPath: res.ByPod.VolumeName, - }, pvr.Spec.UploaderSettings); err != nil { + }, pvr.Spec.UploaderSettings, nil); err != nil { return errors.Wrapf(err, "error to resume asyncBR watcher for PVR %s", pvr.Name) } diff --git a/pkg/controller/pod_volume_restore_controller_test.go b/pkg/controller/pod_volume_restore_controller_test.go index 61d34fae3..73167c76f 100644 --- a/pkg/controller/pod_volume_restore_controller_test.go +++ b/pkg/controller/pod_volume_restore_controller_test.go @@ -19,9 +19,12 @@ package controller import ( "context" "fmt" + "sync" "testing" "time" + clocktesting "k8s.io/utils/clock/testing" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -117,8 +120,6 @@ func TestShouldProcess(t *testing.T) { }, }, shouldProcessed: false, - expectError: true, - errString: "timeout to wait for pod ns-1/pod-1", }, { name: "Empty phase pvr with pod on node not running init container should not be processed", @@ -470,8 +471,6 @@ func TestShouldProcess(t *testing.T) { for _, ts := range tests { t.Run(ts.name, func(t *testing.T) { - ctx := t.Context() - var objs []runtime.Object if ts.obj != nil { objs = append(objs, ts.obj) @@ -487,7 +486,21 @@ func TestShouldProcess(t *testing.T) { clock: &clocks.RealClock{}, } - shouldProcess, _, err := shouldProcess(ctx, c.client, c.logger, ts.obj, time.Second) + if !isPVRNew(ts.obj) { + require.False(t, ts.shouldProcessed) + return + } + + if ts.pod == nil { + _, err := getTargetPod(context.Background(), c.client, c.logger, ts.obj) + if ts.expectError { + require.Error(t, err) + } + require.False(t, ts.shouldProcessed) + return + } + + shouldProcess, err := shouldProcess(ts.pod, c.logger) require.Equal(t, ts.shouldProcessed, shouldProcess) if ts.expectError { require.Error(t, err) @@ -1077,7 +1090,7 @@ func TestPodVolumeRestoreReconcile(t *testing.T) { } if test.sportTime != nil { - r.cancelledPVR[test.pvr.Name] = test.sportTime.Time + r.cancelledPVR.Store(test.pvr.Name, test.sportTime.Time) } if test.constrained { @@ -1099,7 +1112,7 @@ func TestPodVolumeRestoreReconcile(t *testing.T) { } if test.mockStart { - asyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(test.mockStartErr) + asyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.mockStartErr) } if test.mockCancel { @@ -1198,9 +1211,15 @@ func TestPodVolumeRestoreReconcile(t *testing.T) { } if test.expectCancelRecord { - assert.Contains(t, r.cancelledPVR, test.pvr.Name) + _, ok := r.cancelledPVR.Load(test.pvr.Name) + assert.True(t, ok) } else { - assert.Empty(t, r.cancelledPVR) + empty := true + r.cancelledPVR.Range(func(key, value any) bool { + empty = false + return false + }) + assert.True(t, empty) } if isPVRInFinalState(&pvr) || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseInProgress { @@ -1901,7 +1920,7 @@ func TestResumeCancellablePodVolumeRestore(t *testing.T) { } if test.mockStart { - mockAsyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(test.startWatcherErr) + mockAsyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.startWatcherErr) } if test.mockClose { @@ -1925,3 +1944,47 @@ func TestResumeCancellablePodVolumeRestore(t *testing.T) { }) } } + +type pvrSequenceClock struct { + *clocktesting.FakeClock + mu sync.Mutex +} + +func (c *pvrSequenceClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + c.FakeClock.Step(time.Second) + return c.FakeClock.Now() +} + +func TestPodVolumeRestoreCancelConcurrency(t *testing.T) { + ctx := t.Context() + pvr := builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, "pvr-1").Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result() + + r, err := initPodVolumeRestoreReconciler(nil, []client.Object{pvr}) + require.NoError(t, err) + + firstTime := time.Now() + // manually store the initial time + r.cancelledPVR.Store(pvr.Name, firstTime) + + // Custom clock that returns a different time each call + r.clock = &pvrSequenceClock{FakeClock: clocktesting.NewFakeClock(firstTime)} + + var wg sync.WaitGroup + routines := 50 + wg.Add(routines) + + for i := 0; i < routines; i++ { + go func() { + defer wg.Done() + _, _ = r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: pvr.Name, Namespace: pvr.Namespace}}) + }() + } + + wg.Wait() + + v, ok := r.cancelledPVR.Load(pvr.Name) + assert.True(t, ok) + assert.Equal(t, firstTime, v.(time.Time), "The initially recorded timestamp should be preserved") +} diff --git a/pkg/controller/restore_controller.go b/pkg/controller/restore_controller.go index 5b055bc6c..69f8636b7 100644 --- a/pkg/controller/restore_controller.go +++ b/pkg/controller/restore_controller.go @@ -55,6 +55,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" "github.com/vmware-tanzu/velero/pkg/plugin/framework" pkgrestore "github.com/vmware-tanzu/velero/pkg/restore" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/collections" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/logging" @@ -109,10 +110,11 @@ type restoreReconciler struct { defaultItemOperationTimeout time.Duration disableInformerCache bool - newPluginManager func(logger logrus.FieldLogger) clientmgmt.Manager - backupStoreGetter persistence.ObjectBackupStoreGetter - globalCrClient client.Client - resourceTimeout time.Duration + newPluginManager func(logger logrus.FieldLogger) clientmgmt.Manager + backupStoreGetter persistence.ObjectBackupStoreGetter + globalCrClient client.Client + resourceTimeout time.Duration + defaultResourceModifierConfigMap string } type backupInfo struct { @@ -135,6 +137,7 @@ func NewRestoreReconciler( disableInformerCache bool, globalCrClient client.Client, resourceTimeout time.Duration, + defaultResourceModifierConfigMap string, ) *restoreReconciler { r := &restoreReconciler{ ctx: ctx, @@ -154,8 +157,9 @@ func NewRestoreReconciler( newPluginManager: newPluginManager, backupStoreGetter: backupStoreGetter, - globalCrClient: globalCrClient, - resourceTimeout: resourceTimeout, + globalCrClient: globalCrClient, + resourceTimeout: resourceTimeout, + defaultResourceModifierConfigMap: defaultResourceModifierConfigMap, } // Move the periodical backup and restore metrics computing logic from controllers to here. @@ -361,10 +365,15 @@ func (r *restoreReconciler) validateAndComplete(ctx context.Context, restore *ap } // validate ExistingResourcePolicy - if restore.Spec.ExistingResourcePolicy != "" && !pkgrestoreUtil.IsResourcePolicyValid(string(restore.Spec.ExistingResourcePolicy)) { + if !pkgrestoreUtil.IsResourcePolicyValid(string(restore.Spec.ExistingResourcePolicy)) { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("Invalid ExistingResourcePolicy: %s", restore.Spec.ExistingResourcePolicy)) } + // validate ExistingVolumeDataPolicy + if !pkgrestoreUtil.IsVolumeDataPolicyValid(string(restore.Spec.ExistingVolumeDataPolicy)) { + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("Invalid ExistingVolumeDataPolicy: %s", restore.Spec.ExistingVolumeDataPolicy)) + } + // if ScheduleName is specified, fill in BackupName with the most recent successful backup from // the schedule if restore.Spec.ScheduleName != "" { @@ -431,27 +440,72 @@ func (r *restoreReconciler) validateAndComplete(ctx context.Context, restore *ap } var resourceModifiers *resourcemodifiers.ResourceModifiers - if restore.Spec.ResourceModifier != nil && strings.EqualFold(restore.Spec.ResourceModifier.Kind, resourcemodifiers.ConfigmapRefType) { - ResourceModifierConfigMap := &corev1api.ConfigMap{} - err := r.kbClient.Get(ctx, client.ObjectKey{Namespace: restore.Namespace, Name: restore.Spec.ResourceModifier.Name}, ResourceModifierConfigMap) - if err != nil { - restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("failed to get resource modifiers configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name)) - return backupInfo{}, nil, nil + if restore.Spec.ResourceModifier != nil { + if strings.EqualFold(restore.Spec.ResourceModifier.Kind, resourcemodifiers.ConfigmapRefType) { + resourceModifiers = r.loadResourceModifierConfigMap(ctx, restore, restore.Spec.ResourceModifier.Name, false) + if resourceModifiers == nil && len(restore.Status.ValidationErrors) > 0 { + return backupInfo{}, nil, nil + } + } else { + r.logger.Warnf("Unsupported resource modifier kind %q, only %q is supported", restore.Spec.ResourceModifier.Kind, resourcemodifiers.ConfigmapRefType) } - resourceModifiers, err = resourcemodifiers.GetResourceModifiersFromConfig(ResourceModifierConfigMap) - if err != nil { - restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, errors.Wrapf(err, "Error in parsing resource modifiers provided in configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name).Error()) - return backupInfo{}, nil, nil - } else if err = resourceModifiers.Validate(); err != nil { - restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, errors.Wrapf(err, "Validation error in resource modifiers provided in configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name).Error()) - return backupInfo{}, nil, nil + } else if r.defaultResourceModifierConfigMap != "" { + if boolptr.IsSetToTrue(restore.Spec.SkipDefaultResourceModifier) { + r.logger.Infof("Skipping default resource modifier configmap %s/%s as SkipDefaultResourceModifier is set", restore.Namespace, r.defaultResourceModifierConfigMap) + } else { + resourceModifiers = r.loadResourceModifierConfigMap(ctx, restore, r.defaultResourceModifierConfigMap, true) } - r.logger.Infof("Retrieved Resource modifiers provided in configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name) } return info, resourceModifiers, restoreResPolicies } +// loadResourceModifierConfigMap loads and validates a resource modifier ConfigMap. +// When isDefault is true, errors are non-fatal (logged as warnings, returns nil). +// When isDefault is false, errors are added to restore.Status.ValidationErrors. +func (r *restoreReconciler) loadResourceModifierConfigMap( + ctx context.Context, restore *api.Restore, cmName string, isDefault bool, +) *resourcemodifiers.ResourceModifiers { + cm := &corev1api.ConfigMap{} + if err := r.kbClient.Get(ctx, client.ObjectKey{Namespace: restore.Namespace, Name: cmName}, cm); err != nil { + if isDefault { + r.logger.WithError(err).Warnf("Failed to retrieve default resource modifier configmap %s/%s, skipping", restore.Namespace, cmName) + return nil + } + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, + fmt.Sprintf("failed to get resource modifiers configmap %s/%s: %v", restore.Namespace, cmName, err)) + return nil + } + + modifiers, err := resourcemodifiers.GetResourceModifiersFromConfig(cm) + if err != nil { + if isDefault { + r.logger.WithError(err).Warnf("Error parsing default resource modifier configmap %s/%s, skipping", restore.Namespace, cmName) + return nil + } + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, + errors.Wrapf(err, "Error in parsing resource modifiers provided in configmap %s/%s", restore.Namespace, cmName).Error()) + return nil + } + + if err = modifiers.Validate(); err != nil { + if isDefault { + r.logger.WithError(err).Warnf("Validation error in default resource modifier configmap %s/%s, skipping", restore.Namespace, cmName) + return nil + } + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, + errors.Wrapf(err, "Validation error in resource modifiers provided in configmap %s/%s", restore.Namespace, cmName).Error()) + return nil + } + + source := "per-restore" + if isDefault { + source = "default" + } + r.logger.Infof("Retrieved %s resource modifiers from configmap %s/%s", source, restore.Namespace, cmName) + return modifiers +} + // backupXorScheduleProvided returns true if exactly one of BackupName and // ScheduleName are non-empty for the restore, or false otherwise. func backupXorScheduleProvided(restore *api.Restore) bool { diff --git a/pkg/controller/restore_controller_test.go b/pkg/controller/restore_controller_test.go index 6a2f4d8d1..4fb77c8fd 100644 --- a/pkg/controller/restore_controller_test.go +++ b/pkg/controller/restore_controller_test.go @@ -116,6 +116,7 @@ func TestFetchBackupInfo(t *testing.T) { false, fakeGlobalClient, 10*time.Minute, + "", ) if test.backupStoreError == nil { @@ -197,6 +198,7 @@ func TestProcessQueueItemSkips(t *testing.T) { false, fakeGlobalClient, 10*time.Minute, + "", ) _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{ @@ -348,6 +350,39 @@ func TestRestoreReconcile(t *testing.T) { expectedCompletedTime: ×tamp, expectedRestorerCall: nil, // this restore should fail validation and not be passed to the restorer }, + { + name: "valid restore with update existingvolumedatapolicy(full) gets executed", + location: defaultStorageLocation, + restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).ExistingVolumeDataPolicy("full").Result(), + backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseCompleted).Result(), + expectedErr: false, + expectedPhase: string(velerov1api.RestorePhaseInProgress), + expectedStartTime: ×tamp, + expectedCompletedTime: ×tamp, + expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseInProgress).ExistingVolumeDataPolicy("full").Result(), + }, + { + name: "valid restore with update existingvolumedatapolicy(incremental) gets executed", + location: defaultStorageLocation, + restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).ExistingVolumeDataPolicy("incremental").Result(), + backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseCompleted).Result(), + expectedErr: false, + expectedPhase: string(velerov1api.RestorePhaseInProgress), + expectedStartTime: ×tamp, + expectedCompletedTime: ×tamp, + expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseInProgress).ExistingVolumeDataPolicy("incremental").Result(), + }, + { + name: "invalid restore with invalid existingvolumedatapolicy errors", + location: defaultStorageLocation, + restore: NewRestore("foo", "invalidexistingvolumedatapolicy", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).ExistingVolumeDataPolicy("invalid").Result(), + backup: defaultBackup().StorageLocation("default").Result(), + expectedErr: false, + expectedPhase: string(velerov1api.RestorePhaseFailedValidation), + expectedStartTime: ×tamp, + expectedCompletedTime: ×tamp, + expectedRestorerCall: nil, // this restore should fail validation and not be passed to the restorer + }, { name: "valid restore gets executed", location: defaultStorageLocation, @@ -579,6 +614,7 @@ func TestRestoreReconcile(t *testing.T) { false, fakeGlobalClient, 10*time.Minute, + "", ) r.clock = clocktesting.NewFakeClock(now) @@ -767,6 +803,7 @@ func TestValidateAndCompleteWhenScheduleNameSpecified(t *testing.T) { false, fakeGlobalClient, 10*time.Minute, + "", ) restore := &velerov1api.Restore{ @@ -863,6 +900,7 @@ func TestValidateAndCompleteWithResourcePolicySpecified(t *testing.T) { false, fakeGlobalClient, 10*time.Minute, + "", ) restore := &velerov1api.Restore{ @@ -992,6 +1030,7 @@ func TestValidateAndCompleteWithResourceModifierSpecified(t *testing.T) { false, fakeGlobalClient, 10*time.Minute, + "", ) restore := &velerov1api.Restore{ @@ -1110,6 +1149,184 @@ func TestValidateAndCompleteWithResourceModifierSpecified(t *testing.T) { assert.Contains(t, restore3.Status.ValidationErrors[0], "Validation error in resource modifiers provided in configmap") } +func TestValidateAndCompleteWithDefaultResourceModifier(t *testing.T) { + formatFlag := logging.FormatText + + validCMData := map[string]string{ + "modifiers.yaml": "version: v1\nresourceModifierRules:\n- conditions:\n groupResource: pods\n mergePatches:\n - patchData: |\n metadata:\n annotations:\n k8s.ovn.org/pod-networks: null\n", + } + + setupReconciler := func(t *testing.T, defaultCM string) *restoreReconciler { + t.Helper() + fakeClient := velerotest.NewFakeControllerRuntimeClient(t) + fakeGlobalClient := velerotest.NewFakeControllerRuntimeClient(t) + pluginManager := &pluginmocks.Manager{} + backupStore := &persistencemocks.BackupStore{} + + r := NewRestoreReconciler( + t.Context(), + velerov1api.DefaultNamespace, + nil, + fakeClient, + velerotest.NewLogger(), + logrus.DebugLevel, + func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, + NewFakeSingleObjectBackupStoreGetter(backupStore), + metrics.NewServerMetrics(), + formatFlag, + 60*time.Minute, + false, + fakeGlobalClient, + 10*time.Minute, + defaultCM, + ) + + location := builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() + require.NoError(t, r.kbClient.Create(t.Context(), location)) + require.NoError(t, r.kbClient.Create(t.Context(), + defaultBackup().ObjectMeta(builder.WithName("backup-1")).StorageLocation("default").Phase(velerov1api.BackupPhaseCompleted).Result(), + )) + return r + } + + newRestore := func(perRestoreCM string, skip *bool) *velerov1api.Restore { + restore := &velerov1api.Restore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1api.DefaultNamespace, + Name: "restore-1", + }, + Spec: velerov1api.RestoreSpec{ + BackupName: "backup-1", + SkipDefaultResourceModifier: skip, + }, + } + if perRestoreCM != "" { + restore.Spec.ResourceModifier = &corev1api.TypedLocalObjectReference{ + Kind: resourcemodifiers.ConfigmapRefType, + Name: perRestoreCM, + } + } + return restore + } + + t.Run("default modifier applied when no per-restore modifier", func(t *testing.T) { + r := setupReconciler(t, "default-rm") + require.NoError(t, r.kbClient.Create(t.Context(), &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "default-rm", Namespace: velerov1api.DefaultNamespace}, + Data: validCMData, + })) + + restore := newRestore("", nil) + _, rm, _ := r.validateAndComplete(t.Context(), restore) + assert.NotNil(t, rm) + assert.Empty(t, restore.Status.ValidationErrors) + }) + + t.Run("per-restore modifier takes exclusive precedence over default", func(t *testing.T) { + // Default ConfigMap does NOT exist, but per-restore does. + // If default were applied, it would fail. Per-restore should succeed. + r := setupReconciler(t, "nonexistent-default") + require.NoError(t, r.kbClient.Create(t.Context(), &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "per-restore-rm", Namespace: velerov1api.DefaultNamespace}, + Data: validCMData, + })) + + restore := newRestore("per-restore-rm", nil) + _, rm, _ := r.validateAndComplete(t.Context(), restore) + assert.NotNil(t, rm) + assert.Empty(t, restore.Status.ValidationErrors) + }) + + t.Run("skip default modifier when SkipDefaultResourceModifier is true", func(t *testing.T) { + r := setupReconciler(t, "default-rm") + require.NoError(t, r.kbClient.Create(t.Context(), &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "default-rm", Namespace: velerov1api.DefaultNamespace}, + Data: validCMData, + })) + + skipTrue := true + restore := newRestore("", &skipTrue) + _, rm, _ := r.validateAndComplete(t.Context(), restore) + assert.Nil(t, rm) + assert.Empty(t, restore.Status.ValidationErrors) + }) + + t.Run("default modifier with invalid data is non-fatal", func(t *testing.T) { + r := setupReconciler(t, "invalid-default") + require.NoError(t, r.kbClient.Create(t.Context(), &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "invalid-default", Namespace: velerov1api.DefaultNamespace}, + Data: map[string]string{ + "modifiers.yaml": "not-valid-yaml: [", + }, + })) + + restore := newRestore("", nil) + _, rm, _ := r.validateAndComplete(t.Context(), restore) + assert.Nil(t, rm) + assert.Empty(t, restore.Status.ValidationErrors) + }) + + t.Run("default modifier missing is non-fatal", func(t *testing.T) { + r := setupReconciler(t, "nonexistent-cm") + + restore := newRestore("", nil) + _, rm, _ := r.validateAndComplete(t.Context(), restore) + assert.Nil(t, rm) + assert.Empty(t, restore.Status.ValidationErrors) + }) + + t.Run("per-restore modifier missing is fatal", func(t *testing.T) { + r := setupReconciler(t, "") + + restore := newRestore("nonexistent-cm", nil) + _, rm, _ := r.validateAndComplete(t.Context(), restore) + assert.Nil(t, rm) + assert.NotEmpty(t, restore.Status.ValidationErrors) + assert.Contains(t, restore.Status.ValidationErrors[0], "failed to get resource modifiers configmap") + }) + + t.Run("no default configured and no per-restore modifier", func(t *testing.T) { + r := setupReconciler(t, "") + + restore := newRestore("", nil) + _, rm, _ := r.validateAndComplete(t.Context(), restore) + assert.Nil(t, rm) + assert.Empty(t, restore.Status.ValidationErrors) + }) + + t.Run("unsupported resource modifier kind does not apply default", func(t *testing.T) { + r := setupReconciler(t, "default-rm") + require.NoError(t, r.kbClient.Create(t.Context(), &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "default-rm", Namespace: velerov1api.DefaultNamespace}, + Data: validCMData, + })) + + restore := newRestore("", nil) + restore.Spec.ResourceModifier = &corev1api.TypedLocalObjectReference{ + Kind: "Secret", + Name: "some-secret", + } + _, rm, _ := r.validateAndComplete(t.Context(), restore) + assert.Nil(t, rm) + assert.Empty(t, restore.Status.ValidationErrors) + }) + + t.Run("default modifier validation failure is non-fatal", func(t *testing.T) { + r := setupReconciler(t, "invalid-validation") + require.NoError(t, r.kbClient.Create(t.Context(), &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "invalid-validation", Namespace: velerov1api.DefaultNamespace}, + Data: map[string]string{ + "modifiers.yaml": "version: v1\nresourceModifierRules:\n- conditions:\n groupResource: pods\n patches:\n - operation: invalid\n path: \"/spec\"\n value: \"test\"\n", + }, + })) + + restore := newRestore("", nil) + _, rm, _ := r.validateAndComplete(t.Context(), restore) + assert.Nil(t, rm) + assert.Empty(t, restore.Status.ValidationErrors) + }) +} + func TestBackupXorScheduleProvided(t *testing.T) { r := &velerov1api.Restore{} assert.False(t, backupXorScheduleProvided(r)) diff --git a/pkg/controller/restore_finalizer_controller.go b/pkg/controller/restore_finalizer_controller.go index 4e02bb0ef..d9acd0a09 100644 --- a/pkg/controller/restore_finalizer_controller.go +++ b/pkg/controller/restore_finalizer_controller.go @@ -39,6 +39,7 @@ import ( "github.com/vmware-tanzu/velero/internal/hook" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + serverconfig "github.com/vmware-tanzu/velero/pkg/cmd/server/config" "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/metrics" @@ -374,19 +375,20 @@ func (ctx *finalizerContext) patchDynamicPVWithVolumeInfo() (errs results.Result // failures due to the PVC not being bound, which could cause a timeout and result in a failed restore. if pvc.Status.Phase == corev1api.ClaimPending { // check if storage class used has VolumeBindingMode as WaitForFirstConsumer - scName := *pvc.Spec.StorageClassName - sc := &storagev1api.StorageClass{} - err = ctx.crClient.Get(context.Background(), client.ObjectKey{Name: scName}, sc) + if pvc.Spec.StorageClassName != nil && *pvc.Spec.StorageClassName != "" { + scName := *pvc.Spec.StorageClassName + sc := &storagev1api.StorageClass{} + err = ctx.crClient.Get(context.Background(), client.ObjectKey{Name: scName}, sc) - if err != nil { - errs.Add(restoredNamespace, err) - return false, err - } - // skip PV patch step for this scenario - // because pvc would not be bound and the PV patch step would fail due to timeout thus failing the restore - if *sc.VolumeBindingMode == storagev1api.VolumeBindingWaitForFirstConsumer { - log.Warnf("skipping PV patch to restore custom reclaim policy, if any: StorageClass %s used by PVC %s has VolumeBindingMode set to WaitForFirstConsumer, and the PVC is also in a pending state", scName, pvc.Name) - return true, nil + if err != nil { + return false, err + } + // skip PV patch step for this scenario + // because pvc would not be bound and the PV patch step would fail due to timeout thus failing the restore + if sc.VolumeBindingMode != nil && *sc.VolumeBindingMode == storagev1api.VolumeBindingWaitForFirstConsumer { + log.Warnf("skipping PV patch to restore custom reclaim policy, if any: StorageClass %s used by PVC %s has VolumeBindingMode set to WaitForFirstConsumer, and the PVC is also in a pending state", scName, pvc.Name) + return true, nil + } } } @@ -576,8 +578,18 @@ func (ctx *finalizerContext) WaitRestoreExecHook() (errs results.Result) { log := ctx.logger.WithField("restore", ctx.restore.Name) log.Info("Waiting for restore exec hooks starts") - // wait for restore exec hooks to finish - err := wait.PollUntilContextCancel(context.Background(), 1*time.Second, true, func(context.Context) (bool, error) { + // Bound the wait by resourceTimeout (the same budget Velero already + // applies to other finalizer phases). Previously this poll had no + // deadline, so a hook that was registered via Add() but never + // recorded as executed left the restore stuck in Finalizing forever + // and blocked every other restore on the cluster. + timeout := ctx.resourceTimeout + if timeout <= 0 { + timeout = serverconfig.DefaultResourceTimeout + } + pollCtx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + err := wait.PollUntilContextCancel(pollCtx, 1*time.Second, true, func(context.Context) (bool, error) { log.Debug("Checking the progress of hooks execution") if ctx.multiHookTracker.IsComplete(ctx.restore.Name) { return true, nil diff --git a/pkg/controller/restore_finalizer_controller_test.go b/pkg/controller/restore_finalizer_controller_test.go index 8f2618f9d..226a2283c 100644 --- a/pkg/controller/restore_finalizer_controller_test.go +++ b/pkg/controller/restore_finalizer_controller_test.go @@ -482,6 +482,10 @@ func TestWaitRestoreExecHook(t *testing.T) { hookFailed, hookErr := true, fmt.Errorf("hook failed") hookTracker3.Add(restoreName3, podNs, podName, container, source, hookName, hook.PhasePre, 0) + hookTracker4 := hook.NewMultiHookTracker() + restoreName4 := "restore4" + hookTracker4.Add(restoreName4, "ns", "pod", "con1", "s1", "h1", hook.PhasePre, 0) + tests := []struct { name string hookTracker *hook.MultiHookTracker @@ -497,6 +501,8 @@ func TestWaitRestoreExecHook(t *testing.T) { hookName string hookFailed bool hookErr error + resourceTimeout time.Duration + expectTimeoutErr bool }{ { name: "no restore exec hooks", @@ -530,6 +536,16 @@ func TestWaitRestoreExecHook(t *testing.T) { hookFailed: hookFailed, hookErr: hookErr, }, + { + name: "hook never recorded should timeout instead of hanging", + hookTracker: hookTracker4, + restore: builder.ForRestore(velerov1api.DefaultNamespace, restoreName4).Result(), + expectedHooksAttempted: 0, + expectedHooksFailed: 0, + expectedHookErrs: 1, + resourceTimeout: 3 * time.Second, + expectTimeoutErr: true, + }, } for _, tc := range tests { @@ -542,6 +558,7 @@ func TestWaitRestoreExecHook(t *testing.T) { crClient: fakeClient, restore: tc.restore, multiHookTracker: tc.hookTracker, + resourceTimeout: tc.resourceTimeout, } require.NoError(t, ctx.crClient.Create(t.Context(), tc.restore)) @@ -553,6 +570,10 @@ func TestWaitRestoreExecHook(t *testing.T) { } errs := ctx.WaitRestoreExecHook() + if tc.expectTimeoutErr { + assert.NotEmpty(t, errs.Namespaces, "expected timeout error but got none") + continue + } assert.Len(t, errs.Namespaces, tc.expectedHookErrs) updated := &velerov1api.Restore{} @@ -676,11 +697,11 @@ func TestNeedPatch(t *testing.T) { { name: "same label key different values", newPV: builder.ForPersistentVolume("pv1"). - ObjectMeta(builder.WithLabels("topology.kubernetes.io/zone", "us-west-2a")). + ObjectMeta(builder.WithLabels(corev1api.LabelTopologyZone, "us-west-2a")). ReclaimPolicy(corev1api.PersistentVolumeReclaimDelete).Result(), pvInfo: &volume.PVInfo{ ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), - Labels: map[string]string{"topology.kubernetes.io/zone": "us-east-1a"}, + Labels: map[string]string{corev1api.LabelTopologyZone: "us-east-1a"}, }, expected: false, }, diff --git a/pkg/controller/schedule_controller.go b/pkg/controller/schedule_controller.go index d71c86ca4..2e707bdc0 100644 --- a/pkg/controller/schedule_controller.go +++ b/pkg/controller/schedule_controller.go @@ -111,7 +111,11 @@ func (c *scheduleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c original := schedule.DeepCopy() if schedule.Spec.SkipImmediately == nil { - schedule.Spec.SkipImmediately = &c.skipImmediately + // Copy the value rather than aliasing &c.skipImmediately: c is a long-lived + // singleton reconciler, and the block below can write through this pointer, + // which would otherwise mutate the reconciler's shared default field. + skipImmediately := c.skipImmediately + schedule.Spec.SkipImmediately = &skipImmediately } if schedule.Spec.SkipImmediately != nil && *schedule.Spec.SkipImmediately { *schedule.Spec.SkipImmediately = false diff --git a/pkg/controller/schedule_controller_test.go b/pkg/controller/schedule_controller_test.go index 85b87474a..1c134d2bc 100644 --- a/pkg/controller/schedule_controller_test.go +++ b/pkg/controller/schedule_controller_test.go @@ -246,6 +246,54 @@ func parseTime(timeString string) time.Time { return res } +// TestReconcileDoesNotCorruptReconcilerSkipImmediately guards against a regression where +// aliasing &c.skipImmediately into a Schedule's spec (when SkipImmediately is nil) let a +// subsequent write-through-pointer mutate the reconciler's own shared default field, +// silently corrupting it for every later reconcile in the process. +func TestReconcileDoesNotCorruptReconcilerSkipImmediately(t *testing.T) { + require.NoError(t, velerov1.AddToScheme(scheme.Scheme)) + + client := fake.NewClientBuilder().WithScheme(scheme.Scheme).Build() + logger := velerotest.NewLogger() + + // Server configured with schedule-skip-immediately=true. + reconciler := NewScheduleReconciler("ns", logger, client, metrics.NewServerMetrics(), true) + reconciler.clock = testclocks.NewFakeClock(time.Now()) + + makeSchedule := func(name string) *velerov1.Schedule { + return builder.ForSchedule("ns", name). + Phase(velerov1.SchedulePhaseEnabled). + CronSchedule("@every 5m"). + LastBackupTime("2000-01-01 00:00:00"). // long past due, but should be skipped + Result() // SkipImmediately left nil + } + + sched1 := makeSchedule("sched-1") + require.NoError(t, client.Create(ctx, sched1)) + _, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "ns", Name: "sched-1"}}) + require.NoError(t, err) + + // The reconciler's own default must be unchanged after processing a schedule with a nil + // SkipImmediately -- every later schedule relies on this field still being true. + assert.True(t, reconciler.skipImmediately, "reconciler's shared skipImmediately default was mutated by reconciling sched-1") + + sched2 := makeSchedule("sched-2") + require.NoError(t, client.Create(ctx, sched2)) + _, err = reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "ns", Name: "sched-2"}}) + require.NoError(t, err) + + assert.True(t, reconciler.skipImmediately, "reconciler's shared skipImmediately default was mutated by reconciling sched-2") + + // Functional check: sched-2 should ALSO have been skipped (server default still true), + // proving the bug's user-visible symptom (second+ schedule silently loses the skip + // behavior) is fixed, not just the internal field. + got := &velerov1.Schedule{} + require.NoError(t, client.Get(ctx, types.NamespacedName{Namespace: "ns", Name: "sched-2"}, got)) + require.NotNil(t, got.Status.LastSkipped, "sched-2 should have been skipped due to server-wide skipImmediately default") + require.NotNil(t, got.Status.LastBackup) + assert.Equal(t, parseTime("2000-01-01 00:00:00").Unix(), got.Status.LastBackup.Unix(), "sched-2 should not have triggered a new backup") +} + func TestGetNextRunTime(t *testing.T) { defaultSchedule := func() *velerov1.Schedule { return builder.ForSchedule("velero", "schedule-1").CronSchedule("@every 5m").Result() diff --git a/pkg/datamover/backup_micro_service.go b/pkg/datamover/backup_micro_service.go index 08a005217..5ac1ce6ee 100644 --- a/pkg/datamover/backup_micro_service.go +++ b/pkg/datamover/backup_micro_service.go @@ -23,23 +23,23 @@ import ( "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" - "sigs.k8s.io/controller-runtime/pkg/client" - cachetool "k8s.io/client-go/tools/cache" "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/credentials" + veleroshared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + "github.com/vmware-tanzu/velero/pkg/cbtservice" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/repository" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/kube" - - apierrors "k8s.io/apimachinery/pkg/api/errors" ) const ( @@ -71,6 +71,7 @@ type BackupMicroService struct { changeID string volumeID string snapshotID string + cbtService cbtservice.Service } type dataPathResult struct { @@ -80,7 +81,7 @@ type dataPathResult struct { func NewBackupMicroService(ctx context.Context, client client.Client, kubeClient kubernetes.Interface, dataUploadName string, namespace string, nodeName string, sourceTargetPath datapath.AccessPoint, dataPathMgr *datapath.Manager, repoEnsurer *repository.Ensurer, cred *credentials.CredentialGetter, - duInformer cache.Informer, changeID string, volumeID string, snapshotID string, log logrus.FieldLogger) *BackupMicroService { + duInformer cache.Informer, changeID string, volumeID string, snapshotID string, cbtService cbtservice.Service, log logrus.FieldLogger) *BackupMicroService { return &BackupMicroService{ ctx: ctx, client: client, @@ -98,6 +99,7 @@ func NewBackupMicroService(ctx context.Context, client client.Client, kubeClient changeID: changeID, volumeID: volumeID, snapshotID: snapshotID, + cbtService: cbtService, } } @@ -182,7 +184,7 @@ func (r *BackupMicroService) RunCancelableDataPath(ctx context.Context) (string, return "", errors.Wrap(err, "error to create data path") } - log.Debug("Async fs br created") + log.Debug("Async br created") if err := dp.Init(ctx, &datapath.InitParam{ BSLName: du.Spec.BackupStorageLocation, @@ -196,31 +198,45 @@ func (r *BackupMicroService) RunCancelableDataPath(ctx context.Context) (string, return "", errors.Wrap(err, "error to initialize data path") } - log.Info("Async fs br init") + log.Info("Async br init") tags := map[string]string{ velerov1api.AsyncOperationIDLabel: du.Labels[velerov1api.AsyncOperationIDLabel], } + // "none" requests a full backup. "auto" requests that the data mover finds the most + // recent backup of the same volume as parent, which is what an empty ParentSnapshot + // already does, so both map to "". + parentSnapshot := du.Spec.ParentSnapshot + forceFull := false + switch du.Spec.ParentSnapshot { + case veleroshared.ParentSnapshotNone: + parentSnapshot = "" + forceFull = true + case veleroshared.ParentSnapshotAuto: + parentSnapshot = "" + } + if err := dp.StartBackup(r.sourceTargetPath, du.Spec.DataMoverConfig, &datapath.BackupStartParam{ RealSource: GetRealSource(du.Spec.SourceNamespace, du.Spec.SourcePVC), - ParentSnapshot: "", - ForceFull: false, + ParentSnapshot: parentSnapshot, + ForceFull: forceFull, Tags: tags, VolumeID: r.volumeID, ChangeID: r.changeID, SnapshotID: r.snapshotID, + CBTService: r.cbtService, }); err != nil { return "", errors.Wrap(err, "error starting data path backup") } - log.Info("Async fs backup data path started") + log.Info("Async backup data path started") r.eventRecorder.Event(du, false, datapath.EventReasonStarted, "Data path for %s started", du.Name) result := "" select { case <-ctx.Done(): - err = errors.New("timed out waiting for fs backup to complete") + err = errors.New("timed out waiting for backup to complete") break case res := <-r.resultSignal: err = res.err @@ -229,7 +245,7 @@ func (r *BackupMicroService) RunCancelableDataPath(ctx context.Context) (string, } if err != nil { - log.WithError(err).Error("Async fs backup was not completed") + log.WithError(err).Error("Async backup was not completed") } r.eventRecorder.EndingEvent(du, false, datapath.EventReasonStopped, "Data path for %s stopped", du.Name) @@ -266,12 +282,12 @@ func (r *BackupMicroService) OnDataUploadCompleted(ctx context.Context, namespac } } - log.Info("Async fs backup completed") + log.Info("Async backup completed") } func (r *BackupMicroService) OnDataUploadFailed(ctx context.Context, namespace string, duName string, err error) { log := r.logger.WithField("dataupload", duName) - log.WithError(err).Error("Async fs backup data path failed") + log.WithError(err).Error("Async backup data path failed") r.eventRecorder.Event(r.dataUpload, false, datapath.EventReasonFailed, "Data path for data upload %s failed, error %v", r.dataUploadName, err) r.resultSignal <- dataPathResult{ @@ -281,7 +297,7 @@ func (r *BackupMicroService) OnDataUploadFailed(ctx context.Context, namespace s func (r *BackupMicroService) OnDataUploadCancelled(ctx context.Context, namespace string, duName string) { log := r.logger.WithField("dataupload", duName) - log.Warn("Async fs backup data path canceled") + log.Warn("Async backup data path canceled") r.eventRecorder.Event(r.dataUpload, false, datapath.EventReasonCancelled, "Data path for data upload %s canceled", duName) r.resultSignal <- dataPathResult{ @@ -304,9 +320,9 @@ func (r *BackupMicroService) OnDataUploadProgress(ctx context.Context, namespace } func (r *BackupMicroService) closeDataPath(ctx context.Context, duName string) { - fsBackup := r.dataPathMgr.GetAsyncBR(duName) - if fsBackup != nil { - fsBackup.Close(ctx) + asyncBR := r.dataPathMgr.GetAsyncBR(duName) + if asyncBR != nil { + asyncBR.Close(ctx) } r.dataPathMgr.RemoveAsyncBR(duName) @@ -317,11 +333,11 @@ func (r *BackupMicroService) cancelDataUpload(du *velerov2alpha1api.DataUpload) r.eventRecorder.Event(du, false, datapath.EventReasonCancelling, "Canceling for data upload %s", du.Name) - fsBackup := r.dataPathMgr.GetAsyncBR(du.Name) - if fsBackup == nil { + asyncBR := r.dataPathMgr.GetAsyncBR(du.Name) + if asyncBR == nil { r.OnDataUploadCancelled(r.ctx, du.GetNamespace(), du.GetName()) r.eventRecorder.EndingEvent(du, false, datapath.EventReasonStopped, "Data path for %s exited without start", du.Name) } else { - fsBackup.Cancel() + asyncBR.Cancel() } } diff --git a/pkg/datamover/backup_micro_service_test.go b/pkg/datamover/backup_micro_service_test.go index e6291244b..69a4a1381 100644 --- a/pkg/datamover/backup_micro_service_test.go +++ b/pkg/datamover/backup_micro_service_test.go @@ -32,6 +32,7 @@ import ( kbclient "sigs.k8s.io/controller-runtime/pkg/client" clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake" + veleroshared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/builder" @@ -151,7 +152,7 @@ func TestOnDataUploadCompleted(t *testing.T) { { name: "marshal fail", marshalErr: errors.New("fake-marshal-error"), - expectedErr: "Failed to marshal backup result { false { } 0 0}: fake-marshal-error", + expectedErr: "Failed to marshal backup result { false { } 0 }: fake-marshal-error", }, { name: "succeed", @@ -345,7 +346,7 @@ func TestRunCancelableDataPath(t *testing.T) { kubeClientObj: []runtime.Object{duInProgress}, dataPathStarted: true, expectedEventMsg: fmt.Sprintf("Data path for %s stopped", dataUploadName), - expectedErr: "timed out waiting for fs backup to complete", + expectedErr: "timed out waiting for backup to complete", }, { name: "data path returns error", @@ -445,3 +446,89 @@ func TestRunCancelableDataPath(t *testing.T) { cancel() } + +func TestRunCancelableDataPathParentSnapshot(t *testing.T) { + dataUploadName := "fake-data-upload" + + tests := []struct { + name string + parentSnapshot string + expectedParentSnapshot string + expectedForceFull bool + }{ + { + name: "empty lets the data mover search for a parent", + parentSnapshot: "", + expectedParentSnapshot: "", + expectedForceFull: false, + }, + { + name: "auto lets the data mover search for a parent", + parentSnapshot: veleroshared.ParentSnapshotAuto, + expectedParentSnapshot: "", + expectedForceFull: false, + }, + { + name: "none forces a full backup", + parentSnapshot: veleroshared.ParentSnapshotNone, + expectedParentSnapshot: "", + expectedForceFull: true, + }, + { + name: "a specific snapshot ID is passed through unchanged", + parentSnapshot: "fake-parent-snapshot-id", + expectedParentSnapshot: "fake-parent-snapshot-id", + expectedForceFull: false, + }, + } + + scheme := runtime.NewScheme() + velerov2alpha1api.AddToScheme(scheme) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + duInProgress := builder.ForDataUpload(velerov1api.DefaultNamespace, dataUploadName). + Phase(velerov2alpha1api.DataUploadPhaseInProgress). + CSISnapshot(&velerov2alpha1api.CSISnapshotSpec{VolumeSnapshot: "fake-snapshot"}). + Result() + duInProgress.Spec.ParentSnapshot = test.parentSnapshot + + fakeClient := clientFake.NewClientBuilder().WithScheme(scheme). + WithRuntimeObjects(duInProgress).Build() + + bs := &BackupMicroService{ + namespace: velerov1api.DefaultNamespace, + dataUploadName: dataUploadName, + ctx: t.Context(), + client: fakeClient, + dataPathMgr: datapath.NewManager(1), + eventRecorder: &backupMsTestHelper{}, + resultSignal: make(chan dataPathResult), + logger: velerotest.NewLogger(), + } + + var startParam *datapath.BackupStartParam + datapath.VGDPCreator = func(string, string, kbclient.Client, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR { + fsBR := datapathmockes.NewAsyncBR(t) + fsBR.On("Init", mock.Anything, mock.Anything).Return(nil) + fsBR.On("StartBackup", mock.Anything, mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + startParam = args.Get(2).(*datapath.BackupStartParam) + }).Return(nil) + return fsBR + } + + go func() { + time.Sleep(time.Millisecond * 500) + bs.resultSignal <- dataPathResult{result: "fake-succeed-result"} + }() + + _, err := bs.RunCancelableDataPath(t.Context()) + require.NoError(t, err) + + require.NotNil(t, startParam) + assert.Equal(t, test.expectedParentSnapshot, startParam.ParentSnapshot) + assert.Equal(t, test.expectedForceFull, startParam.ForceFull) + }) + } +} diff --git a/pkg/datamover/restore_micro_service.go b/pkg/datamover/restore_micro_service.go index d918667f9..7711fc503 100644 --- a/pkg/datamover/restore_micro_service.go +++ b/pkg/datamover/restore_micro_service.go @@ -32,6 +32,7 @@ import ( "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + "github.com/vmware-tanzu/velero/pkg/cbtservice" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/repository" "github.com/vmware-tanzu/velero/pkg/uploader" @@ -62,11 +63,14 @@ type RestoreMicroService struct { ddHandler cachetool.ResourceEventHandlerRegistration nodeName string cacheDir string + + volumeID string + cbtService cbtservice.Service } func NewRestoreMicroService(ctx context.Context, client client.Client, kubeClient kubernetes.Interface, dataDownloadName string, namespace string, nodeName string, sourceTargetPath datapath.AccessPoint, dataPathMgr *datapath.Manager, repoEnsurer *repository.Ensurer, cred *credentials.CredentialGetter, - ddInformer cache.Informer, cacheDir string, log logrus.FieldLogger) *RestoreMicroService { + ddInformer cache.Informer, cacheDir string, volumeID string, cbtService cbtservice.Service, log logrus.FieldLogger) *RestoreMicroService { return &RestoreMicroService{ ctx: ctx, client: client, @@ -82,6 +86,8 @@ func NewRestoreMicroService(ctx context.Context, client client.Client, kubeClien resultSignal: make(chan dataPathResult), ddInformer: ddInformer, cacheDir: cacheDir, + volumeID: volumeID, + cbtService: cbtService, } } @@ -178,19 +184,28 @@ func (r *RestoreMicroService) RunCancelableDataPath(ctx context.Context) (string }); err != nil { return "", errors.Wrap(err, "error to initialize data path") } - log.Info("fs init") + log.Info("Async br init") - if err := dp.StartRestore(dd.Spec.SnapshotID, r.sourceTargetPath, dd.Spec.DataMoverConfig); err != nil { + param := &datapath.RestoreStartParam{ + Incremental: dd.Spec.RestoreType == string(velerov1api.VolumeDataPolicyTypeIncremental), + CBTService: r.cbtService, + } + if dd.Spec.CSISnapshot != nil { + param.VolumeSnapshotNamespace = dd.Spec.CSISnapshot.VolumeSnapshotNamespace + param.VolumeSnapshotName = dd.Spec.CSISnapshot.VolumeSnapshot + param.VolumeID = r.volumeID + } + if err := dp.StartRestore(dd.Spec.SnapshotID, r.sourceTargetPath, dd.Spec.DataMoverConfig, param); err != nil { return "", errors.Wrap(err, "error starting data path restore") } - log.Info("Async fs restore data path started") + log.Info("Async restore data path started") r.eventRecorder.Event(dd, false, datapath.EventReasonStarted, "Data path for %s started", dd.Name) result := "" select { case <-ctx.Done(): - err = errors.New("timed out waiting for fs restore to complete") + err = errors.New("timed out waiting for restore to complete") break case res := <-r.resultSignal: err = res.err @@ -199,7 +214,7 @@ func (r *RestoreMicroService) RunCancelableDataPath(ctx context.Context) (string } if err != nil { - log.WithError(err).Error("Async fs restore was not completed") + log.WithError(err).Error("Async restore was not completed") } r.eventRecorder.EndingEvent(dd, false, datapath.EventReasonStopped, "Data path for %s stopped", dd.Name) @@ -234,12 +249,12 @@ func (r *RestoreMicroService) OnDataDownloadCompleted(ctx context.Context, names } } - log.Info("Async fs restore data path completed") + log.Info("Async restore data path completed") } func (r *RestoreMicroService) OnDataDownloadFailed(ctx context.Context, namespace string, ddName string, err error) { log := r.logger.WithField("datadownload", ddName) - log.WithError(err).Error("Async fs restore data path failed") + log.WithError(err).Error("Async restore data path failed") r.eventRecorder.Event(r.dataDownload, false, datapath.EventReasonFailed, "Data path for data download %s failed, error %v", r.dataDownloadName, err) r.resultSignal <- dataPathResult{ @@ -249,7 +264,7 @@ func (r *RestoreMicroService) OnDataDownloadFailed(ctx context.Context, namespac func (r *RestoreMicroService) OnDataDownloadCancelled(ctx context.Context, namespace string, ddName string) { log := r.logger.WithField("datadownload", ddName) - log.Warn("Async fs restore data path canceled") + log.Warn("Async restore data path canceled") r.eventRecorder.Event(r.dataDownload, false, datapath.EventReasonCancelled, "Data path for data download %s canceled", ddName) r.resultSignal <- dataPathResult{ @@ -272,9 +287,9 @@ func (r *RestoreMicroService) OnDataDownloadProgress(ctx context.Context, namesp } func (r *RestoreMicroService) closeDataPath(ctx context.Context, ddName string) { - fsRestore := r.dataPathMgr.GetAsyncBR(ddName) - if fsRestore != nil { - fsRestore.Close(ctx) + asyncBR := r.dataPathMgr.GetAsyncBR(ddName) + if asyncBR != nil { + asyncBR.Close(ctx) } r.dataPathMgr.RemoveAsyncBR(ddName) @@ -285,11 +300,11 @@ func (r *RestoreMicroService) cancelDataDownload(dd *velerov2alpha1api.DataDownl r.eventRecorder.Event(dd, false, datapath.EventReasonCancelling, "Canceling for data download %s", dd.Name) - fsBackup := r.dataPathMgr.GetAsyncBR(dd.Name) - if fsBackup == nil { + asyncBR := r.dataPathMgr.GetAsyncBR(dd.Name) + if asyncBR == nil { r.OnDataDownloadCancelled(r.ctx, dd.GetNamespace(), dd.GetName()) r.eventRecorder.EndingEvent(dd, false, datapath.EventReasonStopped, "Data path for %s exited without start", dd.Name) } else { - fsBackup.Cancel() + asyncBR.Cancel() } } diff --git a/pkg/datamover/restore_micro_service_test.go b/pkg/datamover/restore_micro_service_test.go index 33e22eab3..311c015a7 100644 --- a/pkg/datamover/restore_micro_service_test.go +++ b/pkg/datamover/restore_micro_service_test.go @@ -291,7 +291,7 @@ func TestRunCancelableRestore(t *testing.T) { kubeClientObj: []runtime.Object{ddInProgress}, dataPathStarted: true, expectedEventMsg: fmt.Sprintf("Data path for %s stopped", dataDownloadName), - expectedErr: "timed out waiting for fs restore to complete", + expectedErr: "timed out waiting for restore to complete", }, { name: "data path returns error", @@ -355,12 +355,12 @@ func TestRunCancelableRestore(t *testing.T) { if test.startErr != nil { fsBR.On("Init", mock.Anything, mock.Anything).Return(nil) - fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(test.startErr) + fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.startErr) } if test.dataPathStarted { fsBR.On("Init", mock.Anything, mock.Anything).Return(nil) - fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(nil) + fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) } return fsBR diff --git a/pkg/datamover/util.go b/pkg/datamover/util.go index ed66d497a..c82184f31 100644 --- a/pkg/datamover/util.go +++ b/pkg/datamover/util.go @@ -19,12 +19,15 @@ package datamover import ( "fmt" + "github.com/vmware-tanzu/velero/pkg/uploader" datamoverutil "github.com/vmware-tanzu/velero/pkg/util/datamover" ) func GetUploaderType(dataMover string) string { - if datamoverutil.IsBuiltInDataMover(dataMover) { - return "kopia" + if datamoverutil.IsVeleroFSDataMover(dataMover) { + return uploader.KopiaType + } else if datamoverutil.IsVeleroBlockDataMover(dataMover) { + return uploader.BlockType } else { return dataMover } diff --git a/pkg/datamover/util_test.go b/pkg/datamover/util_test.go index d44f3c307..d29b3de12 100644 --- a/pkg/datamover/util_test.go +++ b/pkg/datamover/util_test.go @@ -22,6 +22,16 @@ func TestGetUploaderType(t *testing.T) { input: "velero", want: "kopia", }, + { + name: "velero-fs dataMover is kopia", + input: "velero-fs", + want: "kopia", + }, + { + name: "velero-block dataMover is velero-block", + input: "velero-block", + want: "velero-block", + }, { name: "kopia dataMover is kopia", input: "kopia", diff --git a/pkg/datapath/data_path.go b/pkg/datapath/data_path.go index 6cef1af26..1e7ae948e 100644 --- a/pkg/datapath/data_path.go +++ b/pkg/datapath/data_path.go @@ -22,6 +22,7 @@ import ( "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/credentials" @@ -57,6 +58,16 @@ type BackupStartParam struct { VolumeID string ChangeID string SnapshotID string + CBTService cbtservice.Service +} + +// RestoreStartParam define the input param for restore start +type RestoreStartParam struct { + Incremental bool + VolumeSnapshotNamespace string + VolumeSnapshotName string + VolumeID string + CBTService cbtservice.Service } type generalDataPath struct { @@ -199,6 +210,7 @@ func (dp *generalDataPath) StartBackup(source AccessPoint, uploaderConfig map[st VolumeID: backupParam.VolumeID, ChangeID: backupParam.ChangeID, }, + Service: backupParam.CBTService, }, source.VolMode, uploaderConfig, @@ -214,20 +226,22 @@ func (dp *generalDataPath) StartBackup(source AccessPoint, uploaderConfig map[st } dp.callbacks.OnFailed(context.Background(), dp.namespace, dp.jobName, dataPathErr) } else { - dp.callbacks.OnCompleted(context.Background(), dp.namespace, dp.jobName, Result{Backup: BackupResult{snapshotID, emptySnapshot, source, totalBytes, incrementalBytes}}) + dp.callbacks.OnCompleted(context.Background(), dp.namespace, dp.jobName, Result{Backup: BackupResult{snapshotID, emptySnapshot, source, totalBytes, ptr.To(incrementalBytes)}}) } }() return nil } -func (dp *generalDataPath) StartRestore(snapshotID string, target AccessPoint, uploaderConfigs map[string]string) error { +func (dp *generalDataPath) StartRestore(snapshotID string, target AccessPoint, uploaderConfigs map[string]string, param any) error { if !dp.initialized { return errors.New("data path is not initialized") } dp.wgDataPath.Add(1) + restoreParam := param.(*RestoreStartParam) + go func() { dp.log.Info("Start data path restore") @@ -236,7 +250,14 @@ func (dp *generalDataPath) StartRestore(snapshotID string, target AccessPoint, u dp.wgDataPath.Done() }() - totalBytes, err := dp.uploaderProv.RunRestore(dp.ctx, snapshotID, target.ByPath, target.VolMode, uploaderConfigs, dp) + totalBytes, err := dp.uploaderProv.RunRestore(dp.ctx, snapshotID, target.ByPath, restoreParam.Incremental, + provider.CBTParam{ + Source: cbtservice.SourceInfo{ + Snapshot: restoreParam.VolumeSnapshotName, + VolumeID: restoreParam.VolumeID, + }, + Service: restoreParam.CBTService, + }, target.VolMode, uploaderConfigs, dp) if err == provider.ErrorCanceled { dp.callbacks.OnCancelled(context.Background(), dp.namespace, dp.jobName) diff --git a/pkg/datapath/data_path_test.go b/pkg/datapath/data_path_test.go index 65d7f9b65..34f989517 100644 --- a/pkg/datapath/data_path_test.go +++ b/pkg/datapath/data_path_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "k8s.io/utils/ptr" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/uploader/provider" @@ -82,10 +83,11 @@ func TestAsyncBackup(t *testing.T) { }, result: Result{ Backup: BackupResult{ - SnapshotID: "fake-snapshot", - EmptySnapshot: false, - Source: AccessPoint{ByPath: "fake-path"}, - TotalBytes: 1000, + SnapshotID: "fake-snapshot", + EmptySnapshot: false, + Source: AccessPoint{ByPath: "fake-path"}, + TotalBytes: 1000, + IncrementalBytes: ptr.To(int64(0)), }, }, path: "fake-path", @@ -96,7 +98,11 @@ func TestAsyncBackup(t *testing.T) { t.Run(test.name, func(t *testing.T) { dp := newGeneralDataPath("job-1", "test", nil, "velero", Callbacks{}, velerotest.NewLogger()).(*generalDataPath) mockProvider := providerMock.NewProvider(t) - mockProvider.On("RunBackup", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Backup.SnapshotID, test.result.Backup.EmptySnapshot, test.result.Backup.TotalBytes, test.result.Backup.IncrementalBytes, test.err) + var incrementalBytes int64 + if test.result.Backup.IncrementalBytes != nil { + incrementalBytes = *test.result.Backup.IncrementalBytes + } + mockProvider.On("RunBackup", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Backup.SnapshotID, test.result.Backup.EmptySnapshot, test.result.Backup.TotalBytes, incrementalBytes, test.err) mockProvider.On("Close", mock.Anything).Return(nil) dp.uploaderProv = mockProvider dp.initialized = true @@ -184,13 +190,13 @@ func TestAsyncRestore(t *testing.T) { t.Run(test.name, func(t *testing.T) { dp := newGeneralDataPath("job-1", "test", nil, "velero", Callbacks{}, velerotest.NewLogger()).(*generalDataPath) mockProvider := providerMock.NewProvider(t) - mockProvider.On("RunRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Restore.TotalBytes, test.err) + mockProvider.On("RunRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Restore.TotalBytes, test.err) mockProvider.On("Close", mock.Anything).Return(nil) dp.uploaderProv = mockProvider dp.initialized = true dp.callbacks = test.callbacks - err := dp.StartRestore(test.snapshot, AccessPoint{ByPath: test.path}, map[string]string{}) + err := dp.StartRestore(test.snapshot, AccessPoint{ByPath: test.path}, map[string]string{}, &RestoreStartParam{}) require.NoError(t, err) <-finish diff --git a/pkg/datapath/micro_service_watcher.go b/pkg/datapath/micro_service_watcher.go index 3e8ace651..a824ea469 100644 --- a/pkg/datapath/micro_service_watcher.go +++ b/pkg/datapath/micro_service_watcher.go @@ -19,9 +19,11 @@ package datapath import ( "context" "encoding/json" + "fmt" "os" "strings" "sync" + "sync/atomic" "time" "github.com/cockroachdb/errors" @@ -54,6 +56,7 @@ const ( EventReasonProgress = "Data-Path-Progress" EventReasonCancelling = "Data-Path-Canceling" EventReasonStopped = "Data-Path-Stopped" + EventReasonEvicted = "Evicted" ) type microServiceBRWatcher struct { @@ -72,14 +75,15 @@ type microServiceBRWatcher struct { associatedObject string eventCh chan *corev1api.Event podCh chan *corev1api.Pod - startedFromEvent bool - terminatedFromEvent bool + startedFromEvent atomic.Bool + terminatedFromEvent atomic.Bool wgWatcher sync.WaitGroup eventInformer ctrlcache.Informer podInformer ctrlcache.Informer eventHandler cache.ResourceEventHandlerRegistration podHandler cache.ResourceEventHandlerRegistration watcherLock sync.Mutex + eventMessages sync.Map } func newMicroServiceBRWatcher(client client.Client, kubeClient kubernetes.Interface, mgr manager.Manager, taskType string, taskName string, namespace string, @@ -221,7 +225,7 @@ func (ms *microServiceBRWatcher) StartBackup(source AccessPoint, uploaderConfig return nil } -func (ms *microServiceBRWatcher) StartRestore(snapshotID string, target AccessPoint, uploaderConfigs map[string]string) error { +func (ms *microServiceBRWatcher) StartRestore(snapshotID string, target AccessPoint, uploaderConfigs map[string]string, param any) error { ms.log.Infof("Start watching restore ms to target %s, from snapshot %s", target.ByPath, snapshotID) ms.startWatch() @@ -285,7 +289,7 @@ func (ms *microServiceBRWatcher) startWatch() { } epilogLoop: - for !ms.startedFromEvent || !ms.terminatedFromEvent { + for !ms.startedFromEvent.Load() || !ms.terminatedFromEvent.Load() { select { case <-ms.ctx.Done(): ms.log.Warn("Watch loop is canceled on waiting final event") @@ -303,11 +307,11 @@ func (ms *microServiceBRWatcher) startWatch() { logger.Infof("Finish waiting data path pod, phase %s, message %s", lastPod.Status.Phase, terminateMessage) - if !ms.startedFromEvent { + if !ms.startedFromEvent.Load() { logger.Warn("VGDP seems not started") } - if ms.startedFromEvent && !ms.terminatedFromEvent { + if ms.startedFromEvent.Load() && !ms.terminatedFromEvent.Load() { logger.Warn("VGDP started but termination event is not received") } @@ -326,8 +330,12 @@ func (ms *microServiceBRWatcher) startWatch() { } else { if strings.HasSuffix(terminateMessage, ErrCancelled) { ms.callbacks.OnCancelled(ms.ctx, ms.namespace, ms.taskName) - } else { + } else if terminateMessage != "" { ms.callbacks.OnFailed(ms.ctx, ms.namespace, ms.taskName, errors.New(terminateMessage)) + } else if msg, evicted := ms.eventMessages.Load(EventReasonEvicted); evicted { + ms.callbacks.OnFailed(ms.ctx, ms.namespace, ms.taskName, errors.New(msg.(string))) + } else { + ms.callbacks.OnFailed(ms.ctx, ms.namespace, ms.taskName, errors.New(lastPod.Status.Message)) } } @@ -338,7 +346,7 @@ func (ms *microServiceBRWatcher) startWatch() { func (ms *microServiceBRWatcher) onEvent(evt *corev1api.Event) { switch evt.Reason { case EventReasonStarted: - ms.startedFromEvent = true + ms.startedFromEvent.Store(true) ms.log.Infof("Received data path start message: %s", evt.Message) case EventReasonProgress: ms.callbacks.OnProgress(ms.ctx, ms.namespace, ms.taskName, funcGetProgressFromMessage(evt.Message, ms.log)) @@ -351,8 +359,11 @@ func (ms *microServiceBRWatcher) onEvent(evt *corev1api.Event) { case EventReasonCancelling: ms.log.Infof("Received data path canceling message: %s", evt.Message) case EventReasonStopped: - ms.terminatedFromEvent = true + ms.terminatedFromEvent.Store(true) ms.log.Infof("Received data path stop message: %s", evt.Message) + case EventReasonEvicted: + ms.eventMessages.Store(EventReasonEvicted, fmt.Sprintf("data path pod was evicted, message: %s", evt.Message)) + ms.log.Infof("Pod was evicted for data path %s, message: %s", ms.taskName, evt.Message) default: ms.log.Infof("Received event for data path %s, reason: %s, message: %s", ms.taskName, evt.Reason, evt.Message) } diff --git a/pkg/datapath/micro_service_watcher_test.go b/pkg/datapath/micro_service_watcher_test.go index 6724c290c..315c791ea 100644 --- a/pkg/datapath/micro_service_watcher_test.go +++ b/pkg/datapath/micro_service_watcher_test.go @@ -34,6 +34,7 @@ import ( "k8s.io/client-go/kubernetes" kubeclientfake "k8s.io/client-go/kubernetes/fake" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/vmware-tanzu/velero/pkg/builder" @@ -120,6 +121,7 @@ type startWatchFake struct { redirectErr error complete bool failed bool + failedErr error canceled bool progress int } @@ -142,6 +144,7 @@ func (sw *startWatchFake) OnCompleted(ctx context.Context, namespace string, tas func (sw *startWatchFake) OnFailed(ctx context.Context, namespace string, task string, err error) { sw.failed = true + sw.failedErr = err } func (sw *startWatchFake) OnCancelled(ctx context.Context, namespace string, task string) { @@ -175,6 +178,7 @@ func TestStartWatch(t *testing.T) { expectComplete bool expectCancel bool expectFail bool + expectFailMsg string expectProgress int }{ { @@ -370,6 +374,27 @@ func TestStartWatch(t *testing.T) { expectTerminateEvent: true, expectCancel: true, }, + { + name: "evicted", + thisPod: "fak-pod-1", + thisContainer: "fake-container-1", + insertPod: builder.ForPod("velero", "fake-pod-1").Phase(corev1api.PodFailed).Result(), + insertEventsBefore: []insertEvent{ + { + event: &corev1api.Event{Reason: EventReasonStarted}, + }, + { + event: &corev1api.Event{Reason: EventReasonEvicted, Message: "fake-evicted-message"}, + }, + { + event: &corev1api.Event{Reason: EventReasonStopped}, + }, + }, + expectStartEvent: true, + expectTerminateEvent: true, + expectFail: true, + expectFailMsg: "data path pod was evicted, message: fake-evicted-message", + }, } for _, test := range tests { @@ -437,11 +462,14 @@ func TestStartWatch(t *testing.T) { ms.wgWatcher.Wait() - assert.Equal(t, test.expectStartEvent, ms.startedFromEvent) - assert.Equal(t, test.expectTerminateEvent, ms.terminatedFromEvent) + assert.Equal(t, test.expectStartEvent, ms.startedFromEvent.Load()) + assert.Equal(t, test.expectTerminateEvent, ms.terminatedFromEvent.Load()) assert.Equal(t, test.expectComplete, sw.complete) assert.Equal(t, test.expectCancel, sw.canceled) assert.Equal(t, test.expectFail, sw.failed) + if test.expectFailMsg != "" { + require.EqualError(t, sw.failedErr, test.expectFailMsg) + } assert.Equal(t, test.expectProgress, sw.progress) cancel() @@ -483,6 +511,42 @@ func TestGetResultFromMessage(t *testing.T) { }, }, }, + { + // An old data mover (release-1.17 and earlier) predates IncrementalBytes and + // never writes the key at all -- this pins that its absence unmarshals to nil + // ("not measured"), not a zero value. + name: "old mover message omits incrementalBytes -> nil", + taskType: TaskTypeBackup, + message: "{\"snapshotID\":\"fake-snapshot-id\",\"emptySnapshot\":false,\"source\":{\"byPath\":\"fake-path-1\",\"volumeMode\":\"Block\"}}", + expectResult: Result{ + Backup: BackupResult{ + SnapshotID: "fake-snapshot-id", + Source: AccessPoint{ + ByPath: "fake-path-1", + VolMode: uploader.PersistentVolumeBlock, + }, + IncrementalBytes: nil, + }, + }, + }, + { + // A current mover reports a genuine zero explicitly -- this pins that the key + // being present with value 0 unmarshals to a non-nil pointer to 0 ("measured + // zero"), distinguishing it from the omitted-key case above. + name: "current mover reports measured zero incrementalBytes -> non-nil zero", + taskType: TaskTypeBackup, + message: "{\"snapshotID\":\"fake-snapshot-id\",\"emptySnapshot\":false,\"source\":{\"byPath\":\"fake-path-1\",\"volumeMode\":\"Block\"},\"incrementalBytes\":0}", + expectResult: Result{ + Backup: BackupResult{ + SnapshotID: "fake-snapshot-id", + Source: AccessPoint{ + ByPath: "fake-path-1", + VolMode: uploader.PersistentVolumeBlock, + }, + IncrementalBytes: ptr.To(int64(0)), + }, + }, + }, { name: "succeed to unmarshall restore result", taskType: TaskTypeRestore, diff --git a/pkg/datapath/mocks/asyncBR.go b/pkg/datapath/mocks/asyncBR.go index ef87fde83..deec61dae 100644 --- a/pkg/datapath/mocks/asyncBR.go +++ b/pkg/datapath/mocks/asyncBR.go @@ -60,17 +60,17 @@ func (_m *AsyncBR) StartBackup(source datapath.AccessPoint, dataMoverConfig map[ return r0 } -// StartRestore provides a mock function with given fields: snapshotID, target, dataMoverConfig -func (_m *AsyncBR) StartRestore(snapshotID string, target datapath.AccessPoint, dataMoverConfig map[string]string) error { - ret := _m.Called(snapshotID, target, dataMoverConfig) +// StartRestore provides a mock function with given fields: snapshotID, target, dataMoverConfig, param +func (_m *AsyncBR) StartRestore(snapshotID string, target datapath.AccessPoint, dataMoverConfig map[string]string, param interface{}) error { + ret := _m.Called(snapshotID, target, dataMoverConfig, param) if len(ret) == 0 { panic("no return value specified for StartRestore") } var r0 error - if rf, ok := ret.Get(0).(func(string, datapath.AccessPoint, map[string]string) error); ok { - r0 = rf(snapshotID, target, dataMoverConfig) + if rf, ok := ret.Get(0).(func(string, datapath.AccessPoint, map[string]string, interface{}) error); ok { + r0 = rf(snapshotID, target, dataMoverConfig, param) } else { r0 = ret.Error(0) } diff --git a/pkg/datapath/types.go b/pkg/datapath/types.go index a9c2331a6..339aa6ca4 100644 --- a/pkg/datapath/types.go +++ b/pkg/datapath/types.go @@ -30,11 +30,15 @@ type Result struct { // BackupResult represents the result of a backup type BackupResult struct { - SnapshotID string `json:"snapshotID"` - EmptySnapshot bool `json:"emptySnapshot"` - Source AccessPoint `json:"source,omitempty"` - TotalBytes int64 `json:"totalBytes,omitempty"` - IncrementalBytes int64 `json:"incrementalBytes,omitempty"` + SnapshotID string `json:"snapshotID"` + EmptySnapshot bool `json:"emptySnapshot"` + Source AccessPoint `json:"source,omitempty"` + TotalBytes int64 `json:"totalBytes,omitempty"` + // IncrementalBytes is a pointer so an old data mover (release-1.17 and earlier, + // which predates this field) that omits it unmarshals to nil -- "not measured" -- + // while a current mover reporting a genuine zero still serializes the key and + // unmarshals to a non-nil zero, distinguishing "measured zero" from "not measured". + IncrementalBytes *int64 `json:"incrementalBytes,omitempty"` } // RestoreResult represents the result of a restore @@ -66,7 +70,7 @@ type AsyncBR interface { StartBackup(source AccessPoint, dataMoverConfig map[string]string, param any) error // StartRestore starts an asynchronous data path instance for restore - StartRestore(snapshotID string, target AccessPoint, dataMoverConfig map[string]string) error + StartRestore(snapshotID string, target AccessPoint, dataMoverConfig map[string]string, param any) error // Cancel cancels an asynchronous data path instance Cancel() diff --git a/pkg/exposer/csi_snapshot.go b/pkg/exposer/csi_snapshot.go index ed510c798..a5639537c 100644 --- a/pkg/exposer/csi_snapshot.go +++ b/pkg/exposer/csi_snapshot.go @@ -1,5 +1,5 @@ /* -Copyright The Velero Contributors. +Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -20,7 +20,6 @@ import ( "context" "fmt" "maps" - "strings" "time" "github.com/cockroachdb/errors" @@ -44,6 +43,11 @@ import ( "github.com/vmware-tanzu/velero/pkg/util/kube" ) +// BackupPVCSecretLabel is the label applied to secrets and configmaps copied to the +// Velero namespace for backup PVC provisioning. The value is the owning DataUpload/DataDownload +// UID, which is a stable, valid label value (the owner name may exceed the label-value limit). +const BackupPVCSecretLabel = "velero.io/backup-pvc-secret" //nolint:gosec // not a credential + // CSISnapshotExposeParam define the input param for Expose of CSI snapshots type CSISnapshotExposeParam struct { // SnapshotName is the original volume snapshot name @@ -111,12 +115,6 @@ type CSISnapshotExposeWaitParam struct { NodeName string } -type cbtInfo struct { - changeID string - volumeID string - snapshotID string -} - // NewCSISnapshotExposer create a new instance of CSI snapshot exposer func NewCSISnapshotExposer(kubeClient kubernetes.Interface, csiSnapshotClient snapshotter.SnapshotV1Interface, log logrus.FieldLogger) SnapshotExposer { return &csiSnapshotExposer{ @@ -157,7 +155,29 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O curLog.Info("Volumesnapshot is ready") - vsc, err := csi.GetVolumeSnapshotContentForVolumeSnapshot(volumeSnapshot, e.csiSnapshotClient) + // Copy secrets and configmaps from source namespace to Velero namespace if configured. + // Done before creating any intermediate objects so failure doesn't require cleanup. + // These are needed by CSI drivers that require namespace-scoped resources for volume + // provisioning (e.g., encrypted volumes with KMS tokens and tenant Vault configs). + if value, exists := csiExposeParam.BackupPVCConfig[csiExposeParam.StorageClass]; exists { + copyLabels := map[string]string{BackupPVCSecretLabel: string(ownerObject.UID)} + for _, secretName := range value.SecretNames { + if copyErr := kube.CopySecret(ctx, e.kubeClient.CoreV1(), secretName, + csiExposeParam.SourceNamespace, ownerObject.Namespace, copyLabels, curLog); copyErr != nil { + return errors.Wrapf(copyErr, "error copying secret %s from %s to %s", + secretName, csiExposeParam.SourceNamespace, ownerObject.Namespace) + } + } + for _, cmName := range value.ConfigMapNames { + if copyErr := kube.CopyConfigMap(ctx, e.kubeClient.CoreV1(), cmName, + csiExposeParam.SourceNamespace, ownerObject.Namespace, copyLabels, curLog); copyErr != nil { + return errors.Wrapf(copyErr, "error copying configmap %s from %s to %s", + cmName, csiExposeParam.SourceNamespace, ownerObject.Namespace) + } + } + } + + vsc, err := csi.GetVolumeSnapshotContentForVolumeSnapshot(ctx, volumeSnapshot, e.csiSnapshotClient) if err != nil { return errors.Wrap(err, "error to get volume snapshot content") } @@ -219,6 +239,7 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O backupPVCStorageClass := csiExposeParam.StorageClass backupPVCReadOnly := false spcNoRelabeling := false + backupPVCReadWriteOncePod := false backupPVCAnnotations := map[string]string{} intoleratableNodes := []string{} if value, exists := csiExposeParam.BackupPVCConfig[csiExposeParam.StorageClass]; exists { @@ -235,6 +256,14 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O } } + if value.ReadWriteOncePod { + if backupPVCReadOnly { + curLog.WithField("vs name", volumeSnapshot.Name).Warn("Ignoring readWriteOncePod for read-only volume") + } else { + backupPVCReadWriteOncePod = true + } + } + if len(value.Annotations) > 0 { backupPVCAnnotations = value.Annotations } @@ -249,7 +278,7 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O } } - backupPVC, err := e.createBackupPVC(ctx, ownerObject, backupVS.Name, backupPVCStorageClass, csiExposeParam.AccessMode, volumeSize, backupPVCReadOnly, backupPVCAnnotations, csiExposeParam.DataMover) + backupPVC, err := e.createBackupPVC(ctx, ownerObject, backupVS.Name, backupPVCStorageClass, csiExposeParam.AccessMode, volumeSize, backupPVCReadOnly, backupPVCReadWriteOncePod, backupPVCAnnotations, csiExposeParam.DataMover) if err != nil { return errors.Wrap(err, "error to create backup pvc") } @@ -263,9 +292,9 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O affinity := kube.GetLoadAffinityByStorageClass(csiExposeParam.Affinity, backupPVCStorageClass, curLog) - var cbtInfo cbtInfo + var cbtInfo csi.CBTInfo if csiExposeParam.DataMover == datamover.DataMoverTypeVeleroBlock { - cbtInfo, err = e.getCBTInfo(ctx, backupVS, backupVSC, csiExposeParam.SourcePVName) + cbtInfo, err = csi.GetCBTInfo(ctx, e.kubeClient, e.log, backupVS, backupVSC, csiExposeParam.SourcePVName) if err != nil { return errors.Wrap(err, "error to get CBT info") } @@ -305,49 +334,6 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O return nil } -func (e *csiSnapshotExposer) getCBTInfo(ctx context.Context, vs *snapshotv1api.VolumeSnapshot, vsc *snapshotv1api.VolumeSnapshotContent, sourcePVName string) (cbtInfo, error) { - cbtInfo := cbtInfo{} - if vs == nil || vsc == nil { - return cbtInfo, errors.New("vs or vsc is nil") - } - - cbtInfo.snapshotID = vs.Name - - if vs.Annotations != nil && - (vs.Annotations[util.VSphereCNSChangeIDAnno] != "" || - vs.Annotations[util.VSphereCNSSnapshotAnno] != "") { - cbtInfo.changeID = vs.Annotations[util.VSphereCNSChangeIDAnno] - - splitSnapshotAnno := strings.Split(vs.Annotations[util.VSphereCNSSnapshotAnno], "+") - if len(splitSnapshotAnno) >= 2 { - cbtInfo.volumeID = splitSnapshotAnno[0] - } - - e.log.Debugf("volumeID %s and changeID %s are read from VKS annotations.", cbtInfo.volumeID, cbtInfo.changeID) - } else { - pv, err := e.kubeClient.CoreV1().PersistentVolumes().Get(ctx, sourcePVName, metav1.GetOptions{}) - if err != nil { - return cbtInfo, fmt.Errorf("failed to get pv %s: %w", sourcePVName, err) - } - - if vsc.Status != nil && vsc.Status.SnapshotHandle != nil { - cbtInfo.changeID = *vsc.Status.SnapshotHandle - } - - if pv.Spec.CSI != nil && pv.Spec.CSI.VolumeHandle != "" { - cbtInfo.volumeID = pv.Spec.CSI.VolumeHandle - } - - e.log.Debugf("volumeID %s and changeID %s are read from PV and VS's handles.", cbtInfo.volumeID, cbtInfo.changeID) - } - - if cbtInfo.volumeID == "" { - return cbtInfo, fmt.Errorf("volumeID must not be empty for CBT") - } - - return cbtInfo, nil -} - func (e *csiSnapshotExposer) GetExposed(ctx context.Context, ownerObject corev1api.ObjectReference, timeout time.Duration, param any) (*ExposeResult, error) { exposeWaitParam := param.(*CSISnapshotExposeWaitParam) @@ -514,6 +500,11 @@ func (e *csiSnapshotExposer) CleanUp(ctx context.Context, ownerObject corev1api. kube.DeletePodIfAny(ctx, e.kubeClient.CoreV1(), backupPodName, ownerObject.Namespace, e.log) kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), backupPVCName, ownerObject.Namespace, cleanUpTimeout, e.log) + kube.DeleteSecretsWithLabel(ctx, e.kubeClient.CoreV1(), ownerObject.Namespace, + BackupPVCSecretLabel, string(ownerObject.UID), e.log) + kube.DeleteConfigMapsWithLabel(ctx, e.kubeClient.CoreV1(), ownerObject.Namespace, + BackupPVCSecretLabel, string(ownerObject.UID), e.log) + csi.DeleteVolumeSnapshotIfAny(ctx, e.csiSnapshotClient, backupVSName, ownerObject.Namespace, e.log) csi.DeleteVolumeSnapshotIfAny(ctx, e.csiSnapshotClient, vsName, sourceNamespace, e.log) } @@ -600,7 +591,7 @@ func (e *csiSnapshotExposer) createBackupVSC(ctx context.Context, ownerObject co return e.csiSnapshotClient.VolumeSnapshotContents().Create(ctx, vsc, metav1.CreateOptions{}) } -func (e *csiSnapshotExposer) createBackupPVC(ctx context.Context, ownerObject corev1api.ObjectReference, backupVS, storageClass, accessMode string, resource resource.Quantity, readOnly bool, annotations map[string]string, dataMover string) (*corev1api.PersistentVolumeClaim, error) { +func (e *csiSnapshotExposer) createBackupPVC(ctx context.Context, ownerObject corev1api.ObjectReference, backupVS, storageClass, accessMode string, resource resource.Quantity, readOnly bool, readWriteOncePod bool, annotations map[string]string, dataMover string) (*corev1api.PersistentVolumeClaim, error) { backupPVCName := ownerObject.Name volumeMode, err := getVolumeModeByAccessMode(accessMode, dataMover) @@ -612,6 +603,8 @@ func (e *csiSnapshotExposer) createBackupPVC(ctx context.Context, ownerObject co if readOnly { pvcAccessMode = corev1api.ReadOnlyMany + } else if readWriteOncePod { + pvcAccessMode = corev1api.ReadWriteOncePod } dataSource := &corev1api.TypedLocalObjectReference{ @@ -677,14 +670,16 @@ func (e *csiSnapshotExposer) createBackupPod( intoleratableNodes []string, volumeTopology *corev1api.NodeSelector, csiSnapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService, - cbtInfo *cbtInfo, + cbtInfo *csi.CBTInfo, ) (*corev1api.Pod, error) { podName := ownerObject.Name containerName := string(ownerObject.UID) volumeName := string(ownerObject.UID) - podInfo, err := getInheritedPodInfo(ctx, e.kubeClient, ownerObject.Namespace, nodeOS) + // The backup pod reads the data through the backup PVC only, so the node-agent's host + // path volumes to the kubelet root directory are not inherited. + podInfo, err := getInheritedPodInfo(ctx, e.kubeClient, ownerObject.Namespace, nodeOS, excludeHostPathVolumes) if err != nil { return nil, errors.Wrap(err, "error to get inherited pod info from node-agent") } @@ -731,9 +726,9 @@ func (e *csiSnapshotExposer) createBackupPod( } if cbtInfo != nil { - args = append(args, fmt.Sprintf("--change-id=%s", cbtInfo.changeID)) - args = append(args, fmt.Sprintf("--volume-id=%s", cbtInfo.volumeID)) - args = append(args, fmt.Sprintf("--snapshot-id=%s", cbtInfo.snapshotID)) + args = append(args, fmt.Sprintf("--change-id=%s", cbtInfo.ChangeID)) + args = append(args, fmt.Sprintf("--volume-id=%s", cbtInfo.VolumeID)) + args = append(args, fmt.Sprintf("--snapshot-id=%s", cbtInfo.SnapshotID)) } args = append(args, podInfo.logFormatArgs...) @@ -809,7 +804,7 @@ func (e *csiSnapshotExposer) createBackupPod( } affinity.NodeSelector.MatchExpressions = append(affinity.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ - Key: "kubernetes.io/hostname", + Key: corev1api.LabelHostname, Values: intoleratableNodes, Operator: metav1.LabelSelectorOpNotIn, }) @@ -837,7 +832,7 @@ func (e *csiSnapshotExposer) createBackupPod( TopologySpreadConstraints: []corev1api.TopologySpreadConstraint{ { MaxSkew: 1, - TopologyKey: "kubernetes.io/hostname", + TopologyKey: corev1api.LabelHostname, WhenUnsatisfiable: corev1api.ScheduleAnyway, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ diff --git a/pkg/exposer/csi_snapshot_test.go b/pkg/exposer/csi_snapshot_test.go index e5a7aa9a7..688c439a9 100644 --- a/pkg/exposer/csi_snapshot_test.go +++ b/pkg/exposer/csi_snapshot_test.go @@ -47,6 +47,7 @@ import ( velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util" "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/csi" "github.com/vmware-tanzu/velero/pkg/util/datamover" "github.com/vmware-tanzu/velero/pkg/util/kube" ) @@ -219,6 +220,7 @@ func TestExpose(t *testing.T) { err string expectedVolumeSize *resource.Quantity expectedReadOnlyPVC bool + expectedRWOPPVC bool expectedBackupPVCStorageClass string expectedAffinity *corev1api.Affinity expectedPVCAnnotation map[string]string @@ -492,7 +494,7 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -530,7 +532,7 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -570,7 +572,7 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -615,7 +617,7 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -661,7 +663,96 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "backupPVC uses ReadWriteOncePod access mode", + ownerBackup: backup, + exposeParam: CSISnapshotExposeParam{ + SnapshotName: "fake-vs", + SourceNamespace: "fake-ns", + StorageClass: "fake-sc", + SourcePVName: "fake-pv", + AccessMode: AccessModeFileSystem, + OperationTimeout: time.Millisecond, + ExposeTimeout: time.Millisecond, + BackupPVCConfig: map[string]velerotypes.BackupPVC{ + "fake-sc": { + ReadWriteOncePod: true, + }, + }, + }, + snapshotClientObj: []runtime.Object{ + vsObject, + vscObj, + }, + kubeClientObj: []runtime.Object{ + daemonSet, + scObj, + }, + expectedRWOPPVC: true, + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: corev1api.LabelOSStable, + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "readOnly takes precedence over readWriteOncePod", + ownerBackup: backup, + exposeParam: CSISnapshotExposeParam{ + SnapshotName: "fake-vs", + SourceNamespace: "fake-ns", + StorageClass: "fake-sc", + SourcePVName: "fake-pv", + AccessMode: AccessModeFileSystem, + OperationTimeout: time.Millisecond, + ExposeTimeout: time.Millisecond, + BackupPVCConfig: map[string]velerotypes.BackupPVC{ + "fake-sc": { + ReadOnly: true, + ReadWriteOncePod: true, + }, + }, + }, + snapshotClientObj: []runtime.Object{ + vsObject, + vscObj, + }, + kubeClientObj: []runtime.Object{ + daemonSet, + scObj, + }, + expectedReadOnlyPVC: true, + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -705,7 +796,7 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -732,7 +823,7 @@ func TestExpose(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"Linux"}, }, @@ -757,12 +848,12 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpIn, Values: []string{"Linux"}, }, { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -794,7 +885,7 @@ func TestExpose(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/arch", + Key: corev1api.LabelArchStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"amd64"}, }, @@ -820,12 +911,12 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/arch", + Key: corev1api.LabelArchStable, Operator: corev1api.NodeSelectorOpIn, Values: []string{"amd64"}, }, { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -870,7 +961,7 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -923,7 +1014,7 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -968,7 +1059,7 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -1015,12 +1106,12 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, { - Key: "kubernetes.io/hostname", + Key: corev1api.LabelHostname, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"node-1", "node-2"}, }, @@ -1061,7 +1152,7 @@ func TestExpose(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: corev1api.NodeSelectorOpNotIn, Values: []string{"windows"}, }, @@ -1150,6 +1241,12 @@ func TestExpose(t *testing.T) { assert.Equal(t, test.expectedReadOnlyPVC, gotReadOnlyAccessMode) } + if test.expectedRWOPPVC { + assert.Equal(t, []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOncePod}, backupPVC.Spec.AccessModes) + } else { + assert.NotContains(t, backupPVC.Spec.AccessModes, corev1api.ReadWriteOncePod) + } + if test.expectedBackupPVCStorageClass != "" { assert.Equal(t, test.expectedBackupPVCStorageClass, *backupPVC.Spec.StorageClassName) } @@ -1230,6 +1327,9 @@ func TestGetExpose(t *testing.T) { Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "fake-pv-name", }, + Status: corev1api.PersistentVolumeClaimStatus{ + Phase: corev1api.ClaimBound, + }, } backupPV := &corev1api.PersistentVolume{ @@ -1521,6 +1621,37 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { }, } + backupPVCReadWriteOncePod := corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1.DefaultNamespace, + Name: "fake-backup", + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: backup.APIVersion, + Kind: backup.Kind, + Name: backup.Name, + UID: backup.UID, + Controller: ptr.To(true), + }, + }, + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + AccessModes: []corev1api.PersistentVolumeAccessMode{ + corev1api.ReadWriteOncePod, + }, + VolumeMode: &volumeMode, + DataSource: dataSource, + DataSourceRef: nil, + StorageClassName: ptr.To("fake-storage-class"), + Resources: corev1api.VolumeResourceRequirements{ + Requests: corev1api.ResourceList{ + corev1api.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + } + tests := []struct { name string ownerBackup *velerov1.Backup @@ -1529,6 +1660,7 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { accessMode string resource resource.Quantity readOnly bool + readWriteOncePod bool kubeClientObj []runtime.Object snapshotClientObj []runtime.Object want *corev1api.PersistentVolumeClaim @@ -1556,6 +1688,30 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { want: &backupPVCReadOnly, wantErr: assert.NoError, }, + { + name: "backupPVC gets created with ReadWriteOncePod access mode when readWriteOncePod is set", + ownerBackup: backup, + backupVS: "fake-snapshot", + storageClass: "fake-storage-class", + accessMode: AccessModeFileSystem, + resource: resource.MustParse("1Gi"), + readOnly: false, + readWriteOncePod: true, + want: &backupPVCReadWriteOncePod, + wantErr: assert.NoError, + }, + { + name: "readOnly takes precedence over readWriteOncePod", + ownerBackup: backup, + backupVS: "fake-snapshot", + storageClass: "fake-storage-class", + accessMode: AccessModeFileSystem, + resource: resource.MustParse("1Gi"), + readOnly: true, + readWriteOncePod: true, + want: &backupPVCReadOnly, + wantErr: assert.NoError, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -1576,7 +1732,7 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { APIVersion: tt.ownerBackup.APIVersion, } } - got, err := e.createBackupPVC(t.Context(), ownerObject, tt.backupVS, tt.storageClass, tt.accessMode, tt.resource, tt.readOnly, map[string]string{}, "") + got, err := e.createBackupPVC(t.Context(), ownerObject, tt.backupVS, tt.storageClass, tt.accessMode, tt.resource, tt.readOnly, tt.readWriteOncePod, map[string]string{}, "") if !tt.wantErr(t, err, fmt.Sprintf("createBackupPVC(%v, %v, %v, %v, %v, %v)", ownerObject, tt.backupVS, tt.storageClass, tt.accessMode, tt.resource, tt.readOnly)) { return } @@ -2061,7 +2217,7 @@ func TestGetCBTInfo(t *testing.T) { vsc *snapshotv1api.VolumeSnapshotContent pv *corev1api.PersistentVolume sourcePVName string - want cbtInfo + want csi.CBTInfo wantErrSubstr string }{ { @@ -2084,10 +2240,10 @@ func TestGetCBTInfo(t *testing.T) { }, vsc: &snapshotv1api.VolumeSnapshotContent{}, sourcePVName: "pv-ignored", - want: cbtInfo{ - changeID: "change-id-1", - volumeID: "volume-id-1", - snapshotID: "vs-anno", + want: csi.CBTInfo{ + ChangeID: "change-id-1", + VolumeID: "volume-id-1", + SnapshotID: "vs-anno", }, }, { @@ -2111,10 +2267,10 @@ func TestGetCBTInfo(t *testing.T) { }, }, sourcePVName: "pv-1", - want: cbtInfo{ - changeID: "snapshot-handle-1", - volumeID: "csi-volume-handle-1", - snapshotID: "vs-fallback", + want: csi.CBTInfo{ + ChangeID: "snapshot-handle-1", + VolumeID: "csi-volume-handle-1", + SnapshotID: "vs-fallback", }, }, { @@ -2177,7 +2333,7 @@ func TestGetCBTInfo(t *testing.T) { log: logrus.StandardLogger(), } - got, err := exposer.getCBTInfo(context.Background(), tc.vs, tc.vsc, tc.sourcePVName) + got, err := csi.GetCBTInfo(context.Background(), exposer.kubeClient, exposer.log, tc.vs, tc.vsc, tc.sourcePVName) if tc.wantErrSubstr != "" { if err == nil { @@ -2192,9 +2348,192 @@ func TestGetCBTInfo(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if got.changeID != tc.want.changeID || got.volumeID != tc.want.volumeID || got.snapshotID != tc.want.snapshotID { - t.Fatalf("unexpected cbtInfo, want %+v, got %+v", tc.want, got) + if got.ChangeID != tc.want.ChangeID || got.VolumeID != tc.want.VolumeID || got.SnapshotID != tc.want.SnapshotID { + t.Fatalf("unexpected CBTInfo, want %+v, got %+v", tc.want, got) } }) } } + +func TestExpose_SecretCopy(t *testing.T) { + backup := &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: velerov1.SchemeGroupVersion.String(), + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1.DefaultNamespace, + Name: "fake-backup", + UID: "fake-uid", + }, + } + + ownerObject := corev1api.ObjectReference{ + Kind: backup.Kind, + Namespace: backup.Namespace, + Name: backup.Name, + UID: backup.UID, + APIVersion: backup.APIVersion, + } + + // The secret/configmap copy runs after GetVolumeTopology and WaitVolumeSnapshotReady, + // so a StorageClass and a ready VolumeSnapshot are needed to reach the copy block. + scObj := &storagev1api.StorageClass{ + ObjectMeta: metav1.ObjectMeta{Name: "encrypted-sc"}, + } + readyVS := func() *snapshotv1api.VolumeSnapshot { + vscName := "fake-vsc" + return &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{Name: "fake-vs", Namespace: "app-ns"}, + Spec: snapshotv1api.VolumeSnapshotSpec{ + Source: snapshotv1api.VolumeSnapshotSource{VolumeSnapshotContentName: &vscName}, + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &vscName, + ReadyToUse: boolptr.True(), + RestoreSize: resource.NewQuantity(1234, ""), + }, + } + } + + param := func() *CSISnapshotExposeParam { + return &CSISnapshotExposeParam{ + SourceNamespace: "app-ns", + SourcePVName: "fake-pv", + SnapshotName: "fake-vs", + StorageClass: "encrypted-sc", + OperationTimeout: time.Millisecond, + ExposeTimeout: time.Second, + } + } + + t.Run("copies secret from source namespace", func(t *testing.T) { + srcSecret := &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "kms-token", Namespace: "app-ns"}, + Data: map[string][]byte{"token": []byte("vault-token")}, + Type: corev1api.SecretTypeOpaque, + } + fakeKubeClient := fake.NewSimpleClientset(srcSecret, scObj) + fakeSnapshotClient := snapshotFake.NewSimpleClientset(readyVS()) + + exposer := csiSnapshotExposer{ + kubeClient: fakeKubeClient, + csiSnapshotClient: fakeSnapshotClient.SnapshotV1(), + log: velerotest.NewLogger(), + } + + p := param() + p.BackupPVCConfig = map[string]velerotypes.BackupPVC{ + "encrypted-sc": {SecretNames: []string{"kms-token"}}, + } + + // Expose will fail later (no VSC exists), but the secret copy should succeed + _ = exposer.Expose(t.Context(), ownerObject, p) + + copied, err := fakeKubeClient.CoreV1().Secrets(ownerObject.Namespace).Get( + t.Context(), "kms-token", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, []byte("vault-token"), copied.Data["token"]) + assert.Equal(t, string(ownerObject.UID), copied.Labels[BackupPVCSecretLabel]) + }) + + t.Run("copies configmap from source namespace", func(t *testing.T) { + srcCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "kms-config", Namespace: "app-ns"}, + Data: map[string]string{"vaultAddress": "https://vault.example.com"}, + } + fakeKubeClient := fake.NewSimpleClientset(srcCM, scObj) + fakeSnapshotClient := snapshotFake.NewSimpleClientset(readyVS()) + + exposer := csiSnapshotExposer{ + kubeClient: fakeKubeClient, + csiSnapshotClient: fakeSnapshotClient.SnapshotV1(), + log: velerotest.NewLogger(), + } + + p := param() + p.BackupPVCConfig = map[string]velerotypes.BackupPVC{ + "encrypted-sc": {ConfigMapNames: []string{"kms-config"}}, + } + + _ = exposer.Expose(t.Context(), ownerObject, p) + + copied, err := fakeKubeClient.CoreV1().ConfigMaps(ownerObject.Namespace).Get( + t.Context(), "kms-config", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, "https://vault.example.com", copied.Data["vaultAddress"]) + assert.Equal(t, string(ownerObject.UID), copied.Labels[BackupPVCSecretLabel]) + }) + + t.Run("returns error when source secret missing", func(t *testing.T) { + fakeKubeClient := fake.NewSimpleClientset(scObj) + fakeSnapshotClient := snapshotFake.NewSimpleClientset(readyVS()) + + exposer := csiSnapshotExposer{ + kubeClient: fakeKubeClient, + csiSnapshotClient: fakeSnapshotClient.SnapshotV1(), + log: velerotest.NewLogger(), + } + + p := param() + p.BackupPVCConfig = map[string]velerotypes.BackupPVC{ + "encrypted-sc": {SecretNames: []string{"missing-secret"}}, + } + + err := exposer.Expose(t.Context(), ownerObject, p) + require.Error(t, err) + assert.Contains(t, err.Error(), "error copying secret") + }) +} + +func TestCleanUp_SecretsAndConfigMaps(t *testing.T) { + ownerObject := corev1api.ObjectReference{ + Kind: "Backup", + Namespace: "velero", + Name: "du-123", + UID: "fake-uid", + APIVersion: "v1", + } + + secret := &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kms-token", Namespace: "velero", + Labels: map[string]string{BackupPVCSecretLabel: string(ownerObject.UID)}, + UID: "secret-uid", + }, + } + cm := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kms-config", Namespace: "velero", + Labels: map[string]string{BackupPVCSecretLabel: string(ownerObject.UID)}, + UID: "cm-uid", + }, + } + unrelatedSecret := &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "other-secret", Namespace: "velero", + Labels: map[string]string{BackupPVCSecretLabel: "other-owner-uid"}, + UID: "other-uid", + }, + } + + fakeKubeClient := fake.NewSimpleClientset(secret, cm, unrelatedSecret) + fakeSnapshotClient := snapshotFake.NewSimpleClientset() + + exposer := csiSnapshotExposer{ + kubeClient: fakeKubeClient, + csiSnapshotClient: fakeSnapshotClient.SnapshotV1(), + log: velerotest.NewLogger(), + } + + exposer.CleanUp(t.Context(), ownerObject, "", "app-ns") + + _, err := fakeKubeClient.CoreV1().Secrets("velero").Get(t.Context(), "kms-token", metav1.GetOptions{}) + require.Error(t, err, "owned secret should be deleted") + + _, err = fakeKubeClient.CoreV1().ConfigMaps("velero").Get(t.Context(), "kms-config", metav1.GetOptions{}) + require.Error(t, err, "owned configmap should be deleted") + + _, err = fakeKubeClient.CoreV1().Secrets("velero").Get(t.Context(), "other-secret", metav1.GetOptions{}) + assert.NoError(t, err, "unrelated secret should not be deleted") +} diff --git a/pkg/exposer/generic_restore.go b/pkg/exposer/generic_restore.go index 0f4b9c5b4..b19720389 100644 --- a/pkg/exposer/generic_restore.go +++ b/pkg/exposer/generic_restore.go @@ -1,5 +1,5 @@ /* -Copyright The Velero Contributors. +Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ import ( "github.com/cockroachdb/errors" "github.com/google/uuid" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -31,18 +32,31 @@ import ( "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" + velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/nodeagent" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/csi" "github.com/vmware-tanzu/velero/pkg/util/datamover" "github.com/vmware-tanzu/velero/pkg/util/kube" ) +// GenericRestoreExposeCSI define the CSI specific input param for Generic Restore Expose +type GenericRestoreExposeCSI struct { + // Snapshot is the CSI snapshot spec + Snapshot *velerov2alpha1api.CSISnapshotSpec + // SnapshotMetadataServiceConfigs is the config for CSI snapshot metadata service + SnapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService +} + // GenericRestoreExposeParam define the input param for Generic Restore Expose type GenericRestoreExposeParam struct { // TargetPVCName is the target volume name to be restored TargetPVCName string + // TargetPVName is the target persistent volume name to be restored + TargetPVName string + // TargetNamespace is the namespace of the volume to be restored TargetNamespace string @@ -84,6 +98,9 @@ type GenericRestoreExposeParam struct { // DataMover is the data mover type, e.g., velero-fs, velero-block DataMover string + + // SnapshotMetadataServiceConfigs is the config for CSI snapshot metadata service + CSI *GenericRestoreExposeCSI } // GenericRestoreRebindVolumeParam define the input param for Generic Restore Rebind Volume @@ -101,6 +118,11 @@ type GenericRestoreRebindVolumeParam struct { TargetFSType string } +// GenericRestoreCleanUpParam define the input param for Generic Restore CleanUp +type GenericRestoreCleanUpParam struct { + Snapshot *velerov2alpha1api.CSISnapshotSpec +} + // GenericRestoreExposer is the interfaces for a generic restore exposer type GenericRestoreExposer interface { // Expose starts the process to a restore expose, the expose process may take long time @@ -124,19 +146,21 @@ type GenericRestoreExposer interface { RebindVolume(context.Context, corev1api.ObjectReference, GenericRestoreRebindVolumeParam) error // CleanUp cleans up any objects generated during the restore expose - CleanUp(context.Context, corev1api.ObjectReference) + CleanUp(context.Context, corev1api.ObjectReference, *GenericRestoreCleanUpParam) } // NewGenericRestoreExposer creates a new instance of generic restore exposer -func NewGenericRestoreExposer(kubeClient kubernetes.Interface, log logrus.FieldLogger) GenericRestoreExposer { +func NewGenericRestoreExposer(kubeClient kubernetes.Interface, ctrlClient client.Client, log logrus.FieldLogger) GenericRestoreExposer { return &genericRestoreExposer{ kubeClient: kubeClient, + ctrlClient: ctrlClient, log: log, } } type genericRestoreExposer struct { kubeClient kubernetes.Interface + ctrlClient client.Client log logrus.FieldLogger } @@ -144,9 +168,11 @@ func (e *genericRestoreExposer) Expose(ctx context.Context, ownerObject corev1ap curLog := e.log.WithFields(logrus.Fields{ "owner": ownerObject.Name, "target PVC": param.TargetPVCName, + "target PV": param.TargetPVName, "target namespace": param.TargetNamespace, }) + curLog.Info("Waiting for target PVC to be consumed") selectedNode, targetPVC, err := kube.WaitPVCConsumed( ctx, e.kubeClient.CoreV1(), @@ -196,7 +222,46 @@ func (e *genericRestoreExposer) Expose(ctx context.Context, ownerObject corev1ap } } - restorePVC, err := e.createRestorePVC(ctx, ownerObject, targetPVC, selectedNode, param.DataMover) + // Copy secrets and configmaps from the target namespace to the Velero namespace if configured. + // These are needed by CSI drivers that require namespace-scoped resources for volume + // provisioning of the restorePVC (e.g., encrypted volumes with KMS tokens and tenant Vault configs). + copyLabels := map[string]string{BackupPVCSecretLabel: string(ownerObject.UID)} + for _, secretName := range param.RestorePVCConfig.SecretNames { + if copyErr := kube.CopySecret(ctx, e.kubeClient.CoreV1(), secretName, + param.TargetNamespace, ownerObject.Namespace, copyLabels, curLog); copyErr != nil { + err = errors.Wrapf(copyErr, "error copying secret %s from %s to %s", + secretName, param.TargetNamespace, ownerObject.Namespace) + return err + } + } + for _, cmName := range param.RestorePVCConfig.ConfigMapNames { + if copyErr := kube.CopyConfigMap(ctx, e.kubeClient.CoreV1(), cmName, + param.TargetNamespace, ownerObject.Namespace, copyLabels, curLog); copyErr != nil { + err = errors.Wrapf(copyErr, "error copying configmap %s from %s to %s", + cmName, param.TargetNamespace, ownerObject.Namespace) + return err + } + } + + defer func() { + if err != nil { + kube.DeleteSecretsWithLabel(ctx, e.kubeClient.CoreV1(), ownerObject.Namespace, + BackupPVCSecretLabel, string(ownerObject.UID), curLog) + kube.DeleteConfigMapsWithLabel(ctx, e.kubeClient.CoreV1(), ownerObject.Namespace, + BackupPVCSecretLabel, string(ownerObject.UID), curLog) + } + }() + + curLog.Info("Creating restore PVC") + + var targetPV *corev1api.PersistentVolume + if len(param.TargetPVName) > 0 { + targetPV, err = e.kubeClient.CoreV1().PersistentVolumes().Get(ctx, param.TargetPVName, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "fail to get the target PV %s", param.TargetPVName) + } + } + restorePVC, err := e.createRestorePVC(ctx, ownerObject, targetPVC, targetPV, selectedNode, param.DataMover, param.ExposeTimeout) if err != nil { return errors.Wrap(err, "error to create restore pvc") } @@ -205,10 +270,44 @@ func (e *genericRestoreExposer) Expose(ctx context.Context, ownerObject corev1ap defer func() { if err != nil { - kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), restorePVC.Name, restorePVC.Namespace, 0, curLog) + if len(param.TargetPVName) == 0 { + kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), restorePVC.Name, restorePVC.Namespace, 0, curLog) + } else { + // cannot delete PV if param.TargetPVName is set because the PV is not created by the Expose process. + // It's the existing PV used for in-place restore. + kube.DeletePVCIfAny(ctx, e.kubeClient.CoreV1(), restorePVC.Name, restorePVC.Namespace, 0, curLog) + } } }() + curLog.Info("Creating restore pod") + var volumeID string + if param.CSI != nil && param.CSI.Snapshot != nil { + vs := &snapshotv1api.VolumeSnapshot{} + if err := e.ctrlClient.Get(ctx, client.ObjectKey{ + Namespace: param.CSI.Snapshot.VolumeSnapshotNamespace, + Name: param.CSI.Snapshot.VolumeSnapshot, + }, vs); err != nil { + return errors.Wrapf(err, "error to get volume snapshot %s/%s", param.CSI.Snapshot.VolumeSnapshotNamespace, param.CSI.Snapshot.VolumeSnapshot) + } + + vsc, err := csi.GetVSCForVS(ctx, vs, e.ctrlClient) + if err != nil { + return errors.Wrapf(err, "error to get volume snapshot content for volume snapshot %s/%s", vs.Namespace, vs.Name) + } + + var cbtInfo csi.CBTInfo + cbtInfo, err = csi.GetCBTInfo(ctx, e.kubeClient, e.log, vs, vsc, param.TargetPVName) + if err != nil { + return errors.Wrap(err, "error to get CBT info") + } + curLog.Debugf("CBT info: %+v", cbtInfo) + volumeID = cbtInfo.VolumeID + } + var csiSnapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService + if param.CSI != nil { + csiSnapshotMetadataServiceConfigs = param.CSI.SnapshotMetadataServiceConfigs + } restorePod, err := e.createRestorePod( ctx, ownerObject, @@ -223,6 +322,9 @@ func (e *genericRestoreExposer) Expose(ctx context.Context, ownerObject corev1ap affinity, param.PriorityClassName, cachePVC, + param.TargetNamespace, + volumeID, + csiSnapshotMetadataServiceConfigs, ) if err != nil { return errors.Wrapf(err, "error to create restore pod") @@ -389,7 +491,7 @@ func (e *genericRestoreExposer) DiagnoseExpose(ctx context.Context, ownerObject return diag } -func (e *genericRestoreExposer) CleanUp(ctx context.Context, ownerObject corev1api.ObjectReference) { +func (e *genericRestoreExposer) CleanUp(ctx context.Context, ownerObject corev1api.ObjectReference, param *GenericRestoreCleanUpParam) { restorePodName := ownerObject.Name restorePVCName := ownerObject.Name cachePVCName := getCachePVCName(ownerObject) @@ -397,6 +499,16 @@ func (e *genericRestoreExposer) CleanUp(ctx context.Context, ownerObject corev1a kube.DeletePodIfAny(ctx, e.kubeClient.CoreV1(), restorePodName, ownerObject.Namespace, e.log) kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), restorePVCName, ownerObject.Namespace, 0, e.log) kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), cachePVCName, ownerObject.Namespace, 0, e.log) + + kube.DeleteSecretsWithLabel(ctx, e.kubeClient.CoreV1(), ownerObject.Namespace, + BackupPVCSecretLabel, string(ownerObject.UID), e.log) + kube.DeleteConfigMapsWithLabel(ctx, e.kubeClient.CoreV1(), ownerObject.Namespace, + BackupPVCSecretLabel, string(ownerObject.UID), e.log) + + if param.Snapshot != nil { + kube.EnsureDeleteVolumeSnapshotIfAny(ctx, e.ctrlClient, param.Snapshot.VolumeSnapshotNamespace, + param.Snapshot.VolumeSnapshot, 0, e.log) + } } func (e *genericRestoreExposer) RebindVolume(ctx context.Context, ownerObject corev1api.ObjectReference, param GenericRestoreRebindVolumeParam) error { @@ -610,6 +722,9 @@ func (e *genericRestoreExposer) createRestorePod( affinity *kube.LoadAffinity, priorityClassName string, cachePVC *corev1api.PersistentVolumeClaim, + volumeSnapshotNamespace string, + volumeID string, + csiSnapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService, ) (*corev1api.Pod, error) { restorePodName := ownerObject.Name restorePVCName := ownerObject.Name @@ -620,7 +735,7 @@ func (e *genericRestoreExposer) createRestorePod( nodeSelector := map[string]string{} if selectedNode != "" { affinity = nil - nodeSelector["kubernetes.io/hostname"] = selectedNode + nodeSelector[corev1api.LabelHostname] = selectedNode e.log.Infof("Selected node for restore pod. Ignore affinity from the node-agent config.") } @@ -628,7 +743,9 @@ func (e *genericRestoreExposer) createRestorePod( affinity = &kube.LoadAffinity{} } - podInfo, err := getInheritedPodInfo(ctx, e.kubeClient, ownerObject.Namespace, nodeOS) + // The restore pod writes the data through the restore PVC only, so the node-agent's host + // path volumes to the kubelet root directory are not inherited. + podInfo, err := getInheritedPodInfo(ctx, e.kubeClient, ownerObject.Namespace, nodeOS, excludeHostPathVolumes) if err != nil { return nil, errors.Wrap(err, "error to get inherited pod info from node-agent") } @@ -688,6 +805,14 @@ func (e *genericRestoreExposer) createRestorePod( fmt.Sprintf("--cache-volume-path=%s", cacheVolumePath), } + if len(volumeID) > 0 { + args = append(args, fmt.Sprintf("--vs-namespace=%s", volumeSnapshotNamespace)) + args = append(args, fmt.Sprintf("--volume-id=%s", volumeID)) + } + if csiSnapshotMetadataServiceConfigs != nil && csiSnapshotMetadataServiceConfigs.SAName != "" { + args = append(args, fmt.Sprintf("--cbt-sa-name=%s", csiSnapshotMetadataServiceConfigs.SAName)) + } + args = append(args, podInfo.logFormatArgs...) args = append(args, podInfo.logLevelArgs...) @@ -760,7 +885,7 @@ func (e *genericRestoreExposer) createRestorePod( TopologySpreadConstraints: []corev1api.TopologySpreadConstraint{ { MaxSkew: 1, - TopologyKey: "kubernetes.io/hostname", + TopologyKey: corev1api.LabelHostname, WhenUnsatisfiable: corev1api.ScheduleAnyway, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -806,7 +931,7 @@ func (e *genericRestoreExposer) createRestorePod( return e.kubeClient.CoreV1().Pods(ownerObject.Namespace).Create(ctx, pod, metav1.CreateOptions{}) } -func (e *genericRestoreExposer) createRestorePVC(ctx context.Context, ownerObject corev1api.ObjectReference, targetPVC *corev1api.PersistentVolumeClaim, selectedNode string, dataMover string) (*corev1api.PersistentVolumeClaim, error) { +func (e *genericRestoreExposer) createRestorePVC(ctx context.Context, ownerObject corev1api.ObjectReference, targetPVC *corev1api.PersistentVolumeClaim, targetPV *corev1api.PersistentVolume, selectedNode string, dataMover string, operationTimeout time.Duration) (*corev1api.PersistentVolumeClaim, error) { restorePVCName := ownerObject.Name pvcObj := &corev1api.PersistentVolumeClaim{ @@ -834,9 +959,10 @@ func (e *genericRestoreExposer) createRestorePVC(ctx context.Context, ownerObjec } if selectedNode != "" { - pvcObj.Annotations = map[string]string{ - kube.KubeAnnSelectedNode: selectedNode, + if pvcObj.Annotations == nil { + pvcObj.Annotations = make(map[string]string) } + pvcObj.Annotations[kube.KubeAnnSelectedNode] = selectedNode } if dataMover == datamover.DataMoverTypeVeleroBlock { @@ -847,5 +973,64 @@ func (e *genericRestoreExposer) createRestorePVC(ctx context.Context, ownerObjec *pvcObj.Spec.VolumeMode = corev1api.PersistentVolumeBlock } - return e.kubeClient.CoreV1().PersistentVolumeClaims(pvcObj.Namespace).Create(ctx, pvcObj, metav1.CreateOptions{}) + volumeName := "" + sameVolumeMode := true + if targetPV != nil { + volumeName = targetPV.Name + sameVolumeMode = kube.GetVolumeModeByPVC(pvcObj) == kube.GetVolumeModeByPV(targetPV) + if !sameVolumeMode { + volumeName = ownerObject.Name + } + pvcObj.Spec.VolumeName = volumeName + } + + restorePVC, err := e.kubeClient.CoreV1().PersistentVolumeClaims(pvcObj.Namespace).Create(ctx, pvcObj, metav1.CreateOptions{}) + if err != nil { + return nil, errors.Wrapf(err, "fail to create the restore PVC %s in namespace %s", pvcObj.Name, pvcObj.Namespace) + } + + defer func() { + if err != nil { + kube.DeletePVCIfAny(ctx, e.kubeClient.CoreV1(), pvcObj.Name, pvcObj.Namespace, 0, e.log) + } + }() + + if targetPV != nil { + if !sameVolumeMode { + tmpPV := &corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: volumeName, + }, + Spec: *targetPV.Spec.DeepCopy(), + } + tmpPV.Spec.VolumeMode = restorePVC.Spec.VolumeMode + e.log.Infof("the volume mode is different, creating temporary PV %s with volume mode %s", tmpPV.Name, tmpPV.Spec.VolumeMode) + tmpPV, err = e.kubeClient.CoreV1().PersistentVolumes().Create(ctx, tmpPV, metav1.CreateOptions{}) + if err != nil { + return nil, errors.Wrapf(err, "fail to create the temporary PV %s", volumeName) + } + + defer func() { + if err != nil { + kube.DeletePVIfAny(ctx, e.kubeClient.CoreV1(), tmpPV.Name, e.log) + } + }() + + e.log.Infof("deleting the target PV %s", targetPV.Name) + if err = e.kubeClient.CoreV1().PersistentVolumes().Delete(ctx, targetPV.Name, metav1.DeleteOptions{}); err != nil { + return nil, errors.Wrapf(err, "fail to delete the target PV %s", targetPV.Name) + } + targetPV = tmpPV + } + + if _, err = kube.ResetPVBinding(ctx, e.kubeClient.CoreV1(), targetPV, nil, restorePVC); err != nil { + return nil, errors.Wrapf(err, "fail to reset PV %s binding to restore PVC %s/%s", targetPV.Name, restorePVC.Namespace, restorePVC.Name) + } + + if _, err = kube.WaitPVCBound(ctx, e.kubeClient.CoreV1(), e.kubeClient.CoreV1(), restorePVC.Name, restorePVC.Namespace, operationTimeout); err != nil { + return nil, errors.Wrapf(err, "fail to wait restore PVC %s/%s bound", restorePVC.Namespace, restorePVC.Name) + } + } + + return restorePVC, nil } diff --git a/pkg/exposer/generic_restore_priority_test.go b/pkg/exposer/generic_restore_priority_test.go index 642e0cc43..c8ca784ee 100644 --- a/pkg/exposer/generic_restore_priority_test.go +++ b/pkg/exposer/generic_restore_priority_test.go @@ -149,6 +149,9 @@ func TestCreateRestorePodWithPriorityClass(t *testing.T) { nil, // affinity tc.expectedPriorityClass, nil, + "", // volumeSnapshotNamespace + "", // volumeID + nil, ) require.NoError(t, err, tc.description) @@ -229,6 +232,9 @@ func TestCreateRestorePodWithMissingConfigMap(t *testing.T) { nil, // affinity "", // empty priority class since config map is missing nil, + "", // volumeSnapshotNamespace + "", // volumeID + nil, ) // Should succeed even when config map is missing diff --git a/pkg/exposer/generic_restore_test.go b/pkg/exposer/generic_restore_test.go index b65863318..c08c16b60 100644 --- a/pkg/exposer/generic_restore_test.go +++ b/pkg/exposer/generic_restore_test.go @@ -34,6 +34,7 @@ import ( velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" + velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util/datamover" "github.com/vmware-tanzu/velero/pkg/util/kube" ) @@ -61,6 +62,21 @@ func TestRestoreExpose(t *testing.T) { StorageClassName: &scName, }, } + targetPVObj := &corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-target-pv", + }, + } + + modeBlock := corev1api.PersistentVolumeBlock + targetPVObjWithDifferentVolumeMode := &corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-target-pv", + }, + Spec: corev1api.PersistentVolumeSpec{ + VolumeMode: &modeBlock, + }, + } modeFilesystem := corev1api.PersistentVolumeFilesystem targetPVCObjWithVolumeMode := &corev1api.PersistentVolumeClaim{ @@ -118,12 +134,14 @@ func TestRestoreExpose(t *testing.T) { ownerRestore *velerov1.Restore targetPVCName string targetNamespace string + targetPVName string kubeReactors []reactor cacheVolume *CacheConfigs dataMover string expectBackupPod bool expectBackupPVC bool expectCachePVC bool + expectBackupPV bool err string }{ { @@ -184,7 +202,7 @@ func TestRestoreExpose(t *testing.T) { }, }, }, - err: "error to create restore pvc: fake-create-error", + err: "error to create restore pvc: fail to create the restore PVC fake-restore in namespace velero: fake-create-error", }, { name: "succeed", @@ -199,6 +217,135 @@ func TestRestoreExpose(t *testing.T) { expectBackupPod: true, expectBackupPVC: true, }, + { + name: "succeed with target PV set", + targetPVCName: "fake-target-pvc", + targetNamespace: "fake-ns", + targetPVName: "fake-target-pv", + ownerRestore: restore, + kubeClientObj: []runtime.Object{ + targetPVCObj, + targetPVObj, + daemonSet, + storageClass, + }, + kubeReactors: []reactor{ + { + verb: "get", + resource: "persistentvolumeclaims", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + getAction := action.(clientTesting.GetAction) + if getAction.GetName() == "fake-restore" { + return true, &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-restore", + Namespace: velerov1.DefaultNamespace, + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeName: "fake-target-pv", + }, + Status: corev1api.PersistentVolumeClaimStatus{ + Phase: corev1api.ClaimBound, + }, + }, nil + } + return false, nil, nil + }, + }, + }, + expectBackupPod: true, + expectBackupPVC: true, + }, + { + name: "create temporary PV fail", + targetPVCName: "fake-target-pvc", + targetNamespace: "fake-ns", + targetPVName: "fake-target-pv", + ownerRestore: restore, + kubeClientObj: []runtime.Object{ + targetPVCObj, + targetPVObjWithDifferentVolumeMode, + daemonSet, + storageClass, + }, + kubeReactors: []reactor{ + { + verb: "create", + resource: "persistentvolumes", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, errors.New("fake-create-pv-error") + }, + }, + }, + err: "error to create restore pvc: fail to create the temporary PV fake-restore: fake-create-pv-error", + }, + { + name: "delete original PV fail", + targetPVCName: "fake-target-pvc", + targetNamespace: "fake-ns", + targetPVName: "fake-target-pv", + ownerRestore: restore, + kubeClientObj: []runtime.Object{ + targetPVCObj, + targetPVObjWithDifferentVolumeMode, + daemonSet, + storageClass, + }, + kubeReactors: []reactor{ + { + verb: "delete", + resource: "persistentvolumes", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + deleteAction := action.(clientTesting.DeleteAction) + if deleteAction.GetName() == "fake-target-pv" { + return true, nil, errors.New("fake-delete-pv-error") + } + return false, nil, nil + }, + }, + }, + err: "error to create restore pvc: fail to delete the target PV fake-target-pv: fake-delete-pv-error", + }, + { + name: "succeed with target PV set and different volume mode", + targetPVCName: "fake-target-pvc", + targetNamespace: "fake-ns", + targetPVName: "fake-target-pv", + ownerRestore: restore, + kubeClientObj: []runtime.Object{ + targetPVCObj, + targetPVObjWithDifferentVolumeMode, + daemonSet, + storageClass, + }, + kubeReactors: []reactor{ + { + verb: "get", + resource: "persistentvolumeclaims", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + getAction := action.(clientTesting.GetAction) + if getAction.GetName() == "fake-restore" { + return true, &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-restore", + Namespace: velerov1.DefaultNamespace, + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeName: "fake-restore", + }, + Status: corev1api.PersistentVolumeClaimStatus{ + Phase: corev1api.ClaimBound, + }, + }, nil + } + return false, nil, nil + }, + }, + }, + expectBackupPod: true, + expectBackupPVC: true, + expectBackupPV: true, + }, { name: "succeed, cache config, no cache volume", targetPVCName: "fake-target-pvc", @@ -310,6 +457,7 @@ func TestRestoreExpose(t *testing.T) { GenericRestoreExposeParam{ TargetPVCName: test.targetPVCName, TargetNamespace: test.targetNamespace, + TargetPVName: test.targetPVName, HostingPodLabels: map[string]string{}, Resources: corev1api.ResourceRequirements{}, ExposeTimeout: time.Millisecond, @@ -329,7 +477,7 @@ func TestRestoreExpose(t *testing.T) { if test.expectBackupPod { require.NoError(t, err) } else { - require.True(t, apierrors.IsNotFound(err)) + require.True(t, apierrors.IsNotFound(err), "expected IsNotFound, got %v", err) } pvc, err := exposer.kubeClient.CoreV1().PersistentVolumeClaims(ownerObject.Namespace).Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) @@ -340,19 +488,118 @@ func TestRestoreExpose(t *testing.T) { require.Equal(t, corev1api.PersistentVolumeBlock, *pvc.Spec.VolumeMode) } } else { - require.True(t, apierrors.IsNotFound(err)) + require.True(t, apierrors.IsNotFound(err), "expected IsNotFound, got %v", err) } _, err = exposer.kubeClient.CoreV1().PersistentVolumeClaims(ownerObject.Namespace).Get(t.Context(), getCachePVCName(ownerObject), metav1.GetOptions{}) if test.expectCachePVC { require.NoError(t, err) } else { - require.True(t, apierrors.IsNotFound(err)) + require.True(t, apierrors.IsNotFound(err), "expected IsNotFound, got %v", err) + } + + _, err = exposer.kubeClient.CoreV1().PersistentVolumes().Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) + if test.expectBackupPV { + require.NoError(t, err) + } else { + require.True(t, apierrors.IsNotFound(err), "expected IsNotFound, got %v", err) + } + + if test.targetPVName != "" && !test.expectBackupPV && test.err == "" { + // if targetPVName was provided, and sameVolumeMode was true, the original PV should still exist + _, err = exposer.kubeClient.CoreV1().PersistentVolumes().Get(t.Context(), test.targetPVName, metav1.GetOptions{}) + require.NoError(t, err) + } else if test.targetPVName != "" && test.expectBackupPV { + // if targetPVName was provided, and sameVolumeMode was false (expectBackupPV is true), the original PV should be deleted + _, err = exposer.kubeClient.CoreV1().PersistentVolumes().Get(t.Context(), test.targetPVName, metav1.GetOptions{}) + require.True(t, apierrors.IsNotFound(err), "expected original PV %s to be deleted, but it still exists", test.targetPVName) } }) } } +func TestRestoreExpose_SecretCopy(t *testing.T) { + scName := "fake-sc" + restore := &velerov1.Restore{ + TypeMeta: metav1.TypeMeta{APIVersion: velerov1.SchemeGroupVersion.String(), Kind: "Restore"}, + ObjectMeta: metav1.ObjectMeta{Namespace: velerov1.DefaultNamespace, Name: "fake-restore", UID: "fake-uid"}, + } + ownerObject := corev1api.ObjectReference{ + Kind: restore.Kind, + Namespace: restore.Namespace, + Name: restore.Name, + UID: restore.UID, + APIVersion: restore.APIVersion, + } + targetPVCObj := &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Namespace: "fake-ns", Name: "fake-target-pvc"}, + Spec: corev1api.PersistentVolumeClaimSpec{StorageClassName: &scName}, + } + storageClass := &storagev1api.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "fake-sc"}} + daemonSet := &appsv1api.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Namespace: "velero", Name: "node-agent"}, + TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: appsv1api.SchemeGroupVersion.String()}, + Spec: appsv1api.DaemonSetSpec{ + Template: corev1api.PodTemplateSpec{ + Spec: corev1api.PodSpec{Containers: []corev1api.Container{{Image: "fake-image"}}}, + }, + }, + } + + t.Run("copies secret and configmap from target namespace", func(t *testing.T) { + srcSecret := &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "kms-token", Namespace: "fake-ns"}, + Data: map[string][]byte{"token": []byte("vault-token")}, + Type: corev1api.SecretTypeOpaque, + } + srcCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "kms-config", Namespace: "fake-ns"}, + Data: map[string]string{"vaultAddress": "https://vault.example.com"}, + } + fakeKubeClient := fake.NewSimpleClientset(targetPVCObj, storageClass, daemonSet, srcSecret, srcCM) + exposer := genericRestoreExposer{kubeClient: fakeKubeClient, log: velerotest.NewLogger()} + + err := exposer.Expose(t.Context(), ownerObject, GenericRestoreExposeParam{ + TargetPVCName: "fake-target-pvc", + TargetNamespace: "fake-ns", + HostingPodLabels: map[string]string{}, + Resources: corev1api.ResourceRequirements{}, + ExposeTimeout: time.Millisecond, + RestorePVCConfig: velerotypes.RestorePVC{ + SecretNames: []string{"kms-token"}, + ConfigMapNames: []string{"kms-config"}, + }, + }) + require.NoError(t, err) + + copiedSecret, err := fakeKubeClient.CoreV1().Secrets(ownerObject.Namespace).Get(t.Context(), "kms-token", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, []byte("vault-token"), copiedSecret.Data["token"]) + assert.Equal(t, string(ownerObject.UID), copiedSecret.Labels[BackupPVCSecretLabel]) + + copiedCM, err := fakeKubeClient.CoreV1().ConfigMaps(ownerObject.Namespace).Get(t.Context(), "kms-config", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, "https://vault.example.com", copiedCM.Data["vaultAddress"]) + assert.Equal(t, string(ownerObject.UID), copiedCM.Labels[BackupPVCSecretLabel]) + }) + + t.Run("returns error when source secret missing", func(t *testing.T) { + fakeKubeClient := fake.NewSimpleClientset(targetPVCObj, storageClass, daemonSet) + exposer := genericRestoreExposer{kubeClient: fakeKubeClient, log: velerotest.NewLogger()} + + err := exposer.Expose(t.Context(), ownerObject, GenericRestoreExposeParam{ + TargetPVCName: "fake-target-pvc", + TargetNamespace: "fake-ns", + HostingPodLabels: map[string]string{}, + Resources: corev1api.ResourceRequirements{}, + ExposeTimeout: time.Millisecond, + RestorePVCConfig: velerotypes.RestorePVC{SecretNames: []string{"missing-secret"}}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "error copying secret") + }) +} + func TestRebindVolume(t *testing.T) { restore := &velerov1.Restore{ TypeMeta: metav1.TypeMeta{ @@ -397,6 +644,9 @@ func TestRebindVolume(t *testing.T) { Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "fake-restore-pv", }, + Status: corev1api.PersistentVolumeClaimStatus{ + Phase: corev1api.ClaimBound, + }, } restorePVObj := &corev1api.PersistentVolume{ @@ -1330,12 +1580,13 @@ func TestCreateRestorePod(t *testing.T) { } tests := []struct { - name string - kubeClientObj []runtime.Object - selectedNode string - affinity *kube.LoadAffinity - nodeOS string - expectedPod *corev1api.Pod + name string + kubeClientObj []runtime.Object + selectedNode string + affinity *kube.LoadAffinity + nodeOS string + expectedPod *corev1api.Pod + expectedNodeSelector map[string]string }{ { name: "linux", @@ -1345,7 +1596,7 @@ func TestCreateRestorePod(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"linux"}, }, @@ -1363,7 +1614,7 @@ func TestCreateRestorePod(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"windows"}, }, @@ -1373,6 +1624,29 @@ func TestCreateRestorePod(t *testing.T) { }, nodeOS: "windows", }, + { + // A selected node is pinned through the node selector, and the + // affinity from the node-agent config is ignored. + name: "selected node", + kubeClientObj: []runtime.Object{daemonSet, daemonSetWin, targetPVCObj}, + selectedNode: "fake-selected-node", + affinity: &kube.LoadAffinity{ + NodeSelector: metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: corev1api.LabelOSStable, + Operator: metav1.LabelSelectorOpIn, + Values: []string{"linux"}, + }, + }, + }, + StorageClass: scName, + }, + nodeOS: "linux", + expectedNodeSelector: map[string]string{ + corev1api.LabelHostname: "fake-selected-node", + }, + }, } for _, test := range tests { @@ -1401,12 +1675,18 @@ func TestCreateRestorePod(t *testing.T) { test.affinity, "", // priority class name nil, + "", // volumeSnapshotNamespace + "", // volumeID + nil, ) require.NoError(t, err) if test.expectedPod != nil { assert.Equal(t, test.expectedPod, pod) } + if test.expectedNodeSelector != nil { + assert.Equal(t, test.expectedNodeSelector, pod.Spec.NodeSelector) + } }) } } diff --git a/pkg/exposer/image.go b/pkg/exposer/image.go index 2157d8175..396303d4f 100644 --- a/pkg/exposer/image.go +++ b/pkg/exposer/image.go @@ -1,5 +1,5 @@ /* -Copyright The Velero Contributors. +Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -27,6 +27,18 @@ import ( "github.com/vmware-tanzu/velero/pkg/nodeagent" ) +const ( + // excludeHostPathVolumes indicates that the volumes backed by a host path are not + // inherited from the node-agent. The exposers accessing data through PVCs use it so + // that the hosting pods don't get unnecessary access to the host file system. + excludeHostPathVolumes = true + + // inheritHostPathVolumes indicates that the volumes backed by a host path are + // inherited from the node-agent. fs-backup uses it because it resolves and accesses + // the pod volume data through the kubelet pod directory on the host. + inheritHostPathVolumes = false +) + type inheritedPodInfo struct { image string serviceAccount string @@ -41,7 +53,12 @@ type inheritedPodInfo struct { imagePullSecrets []corev1api.LocalObjectReference } -func getInheritedPodInfo(ctx context.Context, client kubernetes.Interface, veleroNamespace string, osType string) (inheritedPodInfo, error) { +// getInheritedPodInfo collects the pod info to be inherited by the hosting pods from the +// node-agent pod template. When excludeHostPath is true, the volumes backed by a host path, +// together with their volume mounts, are dropped from the result. The volumes are detected +// by their source instead of their name, so the ones customized in the node-agent daemonset +// are covered as well. +func getInheritedPodInfo(ctx context.Context, client kubernetes.Interface, veleroNamespace string, osType string, excludeHostPath bool) (inheritedPodInfo, error) { podInfo := inheritedPodInfo{} podSpec, err := nodeagent.GetPodSpec(ctx, client, veleroNamespace, osType) @@ -58,8 +75,7 @@ func getInheritedPodInfo(ctx context.Context, client kubernetes.Interface, veler podInfo.env = podSpec.Containers[0].Env podInfo.envFrom = podSpec.Containers[0].EnvFrom - podInfo.volumeMounts = podSpec.Containers[0].VolumeMounts - podInfo.volumes = podSpec.Volumes + podInfo.volumeMounts, podInfo.volumes = filterVolumes(podSpec.Containers[0].VolumeMounts, podSpec.Volumes, excludeHostPath) podInfo.dnsPolicy = podSpec.DNSPolicy podInfo.dnsConfig = podSpec.DNSConfig @@ -81,3 +97,35 @@ func getInheritedPodInfo(ctx context.Context, client kubernetes.Interface, veler return podInfo, nil } + +// filterVolumes removes the volumes backed by a host path, as well as the volume mounts +// referring to them, when excludeHostPath is true. The volumes are recognized by their +// source, so the host path volumes customized in the node-agent daemonset are removed as +// well. The other volumes, including the ones customized by users, are kept as is. +func filterVolumes(volumeMounts []corev1api.VolumeMount, volumes []corev1api.Volume, excludeHostPath bool) ([]corev1api.VolumeMount, []corev1api.Volume) { + if !excludeHostPath { + return volumeMounts, volumes + } + + excluded := make(map[string]struct{}) + retainedVolumes := make([]corev1api.Volume, 0, len(volumes)) + for _, volume := range volumes { + if volume.HostPath != nil { + excluded[volume.Name] = struct{}{} + continue + } + + retainedVolumes = append(retainedVolumes, volume) + } + + retainedMounts := make([]corev1api.VolumeMount, 0, len(volumeMounts)) + for _, volumeMount := range volumeMounts { + if _, found := excluded[volumeMount.Name]; found { + continue + } + + retainedMounts = append(retainedMounts, volumeMount) + } + + return retainedMounts, retainedVolumes +} diff --git a/pkg/exposer/image_daemonset_test.go b/pkg/exposer/image_daemonset_test.go new file mode 100644 index 000000000..21ae104df --- /dev/null +++ b/pkg/exposer/image_daemonset_test.go @@ -0,0 +1,91 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package exposer + +import ( + "context" + "testing" + + appsv1api "k8s.io/api/apps/v1" + corev1api "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/vmware-tanzu/velero/pkg/install" +) + +// TestInheritedPodInfoAgainstRealDaemonSet guards the exclusion against the node-agent +// daemonset that is actually installed, so that a host path volume added to the daemonset +// later is not silently inherited by the data mover pods. +func TestInheritedPodInfoAgainstRealDaemonSet(t *testing.T) { + nodeAgent := install.DaemonSet("velero") + client := fake.NewSimpleClientset(&appsv1api.DaemonSet{ + ObjectMeta: nodeAgent.ObjectMeta, + Spec: nodeAgent.Spec, + }) + + hostPathVolumes := func(volumes []corev1api.Volume) []string { + names := []string{} + for _, volume := range volumes { + if volume.HostPath != nil { + names = append(names, volume.Name) + } + } + return names + } + + // The installed daemonset must carry host path volumes, otherwise this test is vacuous. + if len(hostPathVolumes(nodeAgent.Spec.Template.Spec.Volumes)) == 0 { + t.Fatal("the installed node-agent daemonset is expected to have host path volumes") + } + + // fs-backup resolves pod volume data through the kubelet pod directory, so it keeps them. + fsBackupInfo, err := getInheritedPodInfo(context.Background(), client, "velero", "linux", inheritHostPathVolumes) + if err != nil { + t.Fatalf("error to get inherited pod info for fs-backup: %v", err) + } + + if len(hostPathVolumes(fsBackupInfo.volumes)) == 0 { + t.Error("fs-backup is expected to inherit the host path volumes") + } + + // The data mover pods access data through PVCs, so they must not get any host path. + dataMoverInfo, err := getInheritedPodInfo(context.Background(), client, "velero", "linux", excludeHostPathVolumes) + if err != nil { + t.Fatalf("error to get inherited pod info for data mover: %v", err) + } + + if inherited := hostPathVolumes(dataMoverInfo.volumes); len(inherited) > 0 { + t.Errorf("data mover pods are not expected to inherit host path volumes, but got %v", inherited) + } + + // The other volumes, e.g., the scratch volume, are still required. + if len(dataMoverInfo.volumes) == 0 { + t.Error("data mover pods are expected to inherit the volumes other than the host path ones") + } + + // Every remaining mount must still have its backing volume. + volumeNames := map[string]struct{}{} + for _, volume := range dataMoverInfo.volumes { + volumeNames[volume.Name] = struct{}{} + } + + for _, volumeMount := range dataMoverInfo.volumeMounts { + if _, exist := volumeNames[volumeMount.Name]; !exist { + t.Errorf("volume mount %q doesn't have a backing volume", volumeMount.Name) + } + } +} diff --git a/pkg/exposer/image_test.go b/pkg/exposer/image_test.go index 5c47f5c04..a7672b344 100644 --- a/pkg/exposer/image_test.go +++ b/pkg/exposer/image_test.go @@ -1,5 +1,5 @@ /* -Copyright The Velero Contributors. +Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" + "github.com/vmware-tanzu/velero/pkg/nodeagent" "github.com/vmware-tanzu/velero/pkg/util/kube" appsv1api "k8s.io/api/apps/v1" @@ -187,16 +188,132 @@ func TestGetInheritedPodInfo(t *testing.T) { }, } + daemonSetWithHostPath := &appsv1api.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "fake-ns", + Name: "node-agent", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + }, + Spec: appsv1api.DaemonSetSpec{ + Template: corev1api.PodTemplateSpec{ + Spec: corev1api.PodSpec{ + Containers: []corev1api.Container{ + { + Name: "container-1", + Image: "image-1", + VolumeMounts: []corev1api.VolumeMount{ + { + Name: nodeagent.HostPodVolumeMount, + MountPath: "/host_pods", + }, + { + Name: "host-plugins", + MountPath: "/var/lib/kubelet/plugins", + }, + { + Name: "customized-host-path", + MountPath: "/customized", + }, + { + Name: "scratch", + MountPath: "/scratch", + }, + { + Name: "user-credentials", + MountPath: "/credentials", + }, + }, + }, + }, + Volumes: []corev1api.Volume{ + { + Name: nodeagent.HostPodVolumeMount, + VolumeSource: corev1api.VolumeSource{ + HostPath: &corev1api.HostPathVolumeSource{ + Path: "/var/lib/kubelet/pods", + }, + }, + }, + { + Name: "host-plugins", + VolumeSource: corev1api.VolumeSource{ + HostPath: &corev1api.HostPathVolumeSource{ + Path: "/var/lib/kubelet/plugins", + }, + }, + }, + { + // A host path volume added by users. It's not named after any + // well-known volume, so it can only be recognized by its source. + Name: "customized-host-path", + VolumeSource: corev1api.VolumeSource{ + HostPath: &corev1api.HostPathVolumeSource{ + Path: "/mnt/customized", + }, + }, + }, + { + Name: "scratch", + VolumeSource: corev1api.VolumeSource{ + EmptyDir: new(corev1api.EmptyDirVolumeSource), + }, + }, + { + Name: "user-credentials", + VolumeSource: corev1api.VolumeSource{ + Secret: &corev1api.SecretVolumeSource{ + SecretName: "user-credentials", + }, + }, + }, + }, + ServiceAccountName: "sa-1", + }, + }, + }, + } + + scratchAndCredentialMounts := []corev1api.VolumeMount{ + { + Name: "scratch", + MountPath: "/scratch", + }, + { + Name: "user-credentials", + MountPath: "/credentials", + }, + } + + scratchAndCredentialVolumes := []corev1api.Volume{ + { + Name: "scratch", + VolumeSource: corev1api.VolumeSource{ + EmptyDir: new(corev1api.EmptyDirVolumeSource), + }, + }, + { + Name: "user-credentials", + VolumeSource: corev1api.VolumeSource{ + Secret: &corev1api.SecretVolumeSource{ + SecretName: "user-credentials", + }, + }, + }, + } + scheme := runtime.NewScheme() appsv1api.AddToScheme(scheme) tests := []struct { - name string - namespace string - client kubernetes.Interface - kubeClientObj []runtime.Object - result inheritedPodInfo - expectErr string + name string + namespace string + client kubernetes.Interface + kubeClientObj []runtime.Object + excludeHostPath bool + result inheritedPodInfo + expectErr string }{ { name: "ds is not found", @@ -329,12 +446,93 @@ func TestGetInheritedPodInfo(t *testing.T) { }, }, }, + { + name: "host path volumes are inherited by default", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + daemonSetWithHostPath, + }, + result: inheritedPodInfo{ + image: "image-1", + serviceAccount: "sa-1", + volumeMounts: daemonSetWithHostPath.Spec.Template.Spec.Containers[0].VolumeMounts, + volumes: daemonSetWithHostPath.Spec.Template.Spec.Volumes, + }, + }, + { + name: "host path volumes and their mounts are excluded, no matter how they are named", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + daemonSetWithHostPath, + }, + excludeHostPath: true, + result: inheritedPodInfo{ + image: "image-1", + serviceAccount: "sa-1", + volumeMounts: scratchAndCredentialMounts, + volumes: scratchAndCredentialVolumes, + }, + }, + { + name: "excluding host path volumes keeps the others when there is none", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + daemonSetWithNoLog, + }, + excludeHostPath: true, + result: inheritedPodInfo{ + image: "image-1", + serviceAccount: "sa-1", + env: []corev1api.EnvVar{ + { + Name: "env-1", + Value: "value-1", + }, + { + Name: "env-2", + Value: "value-2", + }, + }, + envFrom: []corev1api.EnvFromSource{ + { + ConfigMapRef: &corev1api.ConfigMapEnvSource{ + LocalObjectReference: corev1api.LocalObjectReference{ + Name: "test-configmap", + }, + }, + }, + { + SecretRef: &corev1api.SecretEnvSource{ + LocalObjectReference: corev1api.LocalObjectReference{ + Name: "test-secret", + }, + }, + }, + }, + volumeMounts: []corev1api.VolumeMount{ + { + Name: "volume-1", + }, + { + Name: "volume-2", + }, + }, + volumes: []corev1api.Volume{ + { + Name: "volume-1", + }, + { + Name: "volume-2", + }, + }, + }, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) - info, err := getInheritedPodInfo(t.Context(), fakeKubeClient, test.namespace, kube.NodeOSLinux) + info, err := getInheritedPodInfo(t.Context(), fakeKubeClient, test.namespace, kube.NodeOSLinux, test.excludeHostPath) if test.expectErr == "" { require.NoError(t, err) diff --git a/pkg/exposer/mocks/GenericRestoreExposer.go b/pkg/exposer/mocks/GenericRestoreExposer.go index a1d8943d4..30639b6a8 100644 --- a/pkg/exposer/mocks/GenericRestoreExposer.go +++ b/pkg/exposer/mocks/GenericRestoreExposer.go @@ -42,8 +42,8 @@ func (_m *GenericRestoreExposer) EXPECT() *GenericRestoreExposer_Expecter { } // CleanUp provides a mock function for the type GenericRestoreExposer -func (_mock *GenericRestoreExposer) CleanUp(context1 context.Context, objectReference v1.ObjectReference) { - _mock.Called(context1, objectReference) +func (_mock *GenericRestoreExposer) CleanUp(context1 context.Context, objectReference v1.ObjectReference, param *exposer.GenericRestoreCleanUpParam) { + _mock.Called(context1, objectReference, param) return } @@ -55,11 +55,12 @@ type GenericRestoreExposer_CleanUp_Call struct { // CleanUp is a helper method to define mock.On call // - context1 context.Context // - objectReference v1.ObjectReference -func (_e *GenericRestoreExposer_Expecter) CleanUp(context1 interface{}, objectReference interface{}) *GenericRestoreExposer_CleanUp_Call { - return &GenericRestoreExposer_CleanUp_Call{Call: _e.mock.On("CleanUp", context1, objectReference)} +// - param *exposer.GenericRestoreCleanUpParam +func (_e *GenericRestoreExposer_Expecter) CleanUp(context1 interface{}, objectReference interface{}, param interface{}) *GenericRestoreExposer_CleanUp_Call { + return &GenericRestoreExposer_CleanUp_Call{Call: _e.mock.On("CleanUp", context1, objectReference, param)} } -func (_c *GenericRestoreExposer_CleanUp_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference)) *GenericRestoreExposer_CleanUp_Call { +func (_c *GenericRestoreExposer_CleanUp_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference, param *exposer.GenericRestoreCleanUpParam)) *GenericRestoreExposer_CleanUp_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -69,9 +70,14 @@ func (_c *GenericRestoreExposer_CleanUp_Call) Run(run func(context1 context.Cont if args[1] != nil { arg1 = args[1].(v1.ObjectReference) } + var arg2 *exposer.GenericRestoreCleanUpParam + if args[2] != nil { + arg2 = args[2].(*exposer.GenericRestoreCleanUpParam) + } run( arg0, arg1, + arg2, ) }) return _c @@ -82,7 +88,7 @@ func (_c *GenericRestoreExposer_CleanUp_Call) Return() *GenericRestoreExposer_Cl return _c } -func (_c *GenericRestoreExposer_CleanUp_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference)) *GenericRestoreExposer_CleanUp_Call { +func (_c *GenericRestoreExposer_CleanUp_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference, param *exposer.GenericRestoreCleanUpParam)) *GenericRestoreExposer_CleanUp_Call { _c.Run(run) return _c } diff --git a/pkg/exposer/pod_volume.go b/pkg/exposer/pod_volume.go index 0526b2c5e..5d6de1831 100644 --- a/pkg/exposer/pod_volume.go +++ b/pkg/exposer/pod_volume.go @@ -1,5 +1,5 @@ /* -Copyright The Velero Contributors. +Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -365,7 +365,7 @@ func (e *podVolumeExposer) createHostingPod( clientVolumeName := string(ownerObject.UID) clientVolumePath := "/" + clientVolumeName - podInfo, err := getInheritedPodInfo(ctx, e.kubeClient, ownerObject.Namespace, nodeOS) + podInfo, err := getInheritedPodInfo(ctx, e.kubeClient, ownerObject.Namespace, nodeOS, inheritHostPathVolumes) if err != nil { return nil, errors.Wrap(err, "error to get inherited pod info from node-agent") } diff --git a/pkg/install/daemonset.go b/pkg/install/daemonset.go index 190e785d8..6ef1139d2 100644 --- a/pkg/install/daemonset.go +++ b/pkg/install/daemonset.go @@ -247,7 +247,7 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1api.DaemonSet { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Values: []string{"windows"}, Operator: corev1api.NodeSelectorOpIn, }, @@ -280,7 +280,7 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1api.DaemonSet { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Values: []string{"windows"}, Operator: corev1api.NodeSelectorOpNotIn, }, diff --git a/pkg/install/daemonset_test.go b/pkg/install/daemonset_test.go index 6cab7f063..2c7e201e4 100644 --- a/pkg/install/daemonset_test.go +++ b/pkg/install/daemonset_test.go @@ -41,7 +41,7 @@ func TestDaemonSet(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Values: []string{"windows"}, Operator: corev1api.NodeSelectorOpNotIn, }, @@ -107,7 +107,7 @@ func TestDaemonSet(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Values: []string{"windows"}, Operator: corev1api.NodeSelectorOpIn, }, diff --git a/pkg/install/deployment.go b/pkg/install/deployment.go index 4ce4b5a4f..642a51321 100644 --- a/pkg/install/deployment.go +++ b/pkg/install/deployment.go @@ -34,37 +34,38 @@ import ( type podTemplateOption func(*podTemplateConfig) type podTemplateConfig struct { - image string - envVars []corev1api.EnvVar - restoreOnly bool - annotations map[string]string - labels map[string]string - resources corev1api.ResourceRequirements - withSecret bool - defaultRepoMaintenanceFrequency time.Duration - garbageCollectionFrequency time.Duration - podVolumeOperationTimeout time.Duration - plugins []string - features []string - defaultVolumesToFsBackup bool - serviceAccountName string - uploaderType string - defaultSnapshotMoveData bool - csiSnapshotEarlyFrequentPolling bool - privilegedNodeAgent bool - disableInformerCache bool - scheduleSkipImmediately bool - podResources kube.PodResources - keepLatestMaintenanceJobs int - backupRepoConfigMap string - repoMaintenanceJobConfigMap string - nodeAgentConfigMap string - itemBlockWorkerCount int - concurrentBackups int - forWindows bool - kubeletRootDir string - nodeAgentDisableHostPath bool - priorityClassName string + image string + envVars []corev1api.EnvVar + restoreOnly bool + annotations map[string]string + labels map[string]string + resources corev1api.ResourceRequirements + withSecret bool + defaultRepoMaintenanceFrequency time.Duration + garbageCollectionFrequency time.Duration + podVolumeOperationTimeout time.Duration + plugins []string + features []string + defaultVolumesToFsBackup bool + serviceAccountName string + uploaderType string + defaultSnapshotMoveData bool + csiSnapshotEarlyFrequentPolling bool + privilegedNodeAgent bool + disableInformerCache bool + scheduleSkipImmediately bool + podResources kube.PodResources + keepLatestMaintenanceJobs int + backupRepoConfigMap string + repoMaintenanceJobConfigMap string + defaultResourceModifierConfigMap string + nodeAgentConfigMap string + itemBlockWorkerCount int + concurrentBackups int + forWindows bool + kubeletRootDir string + nodeAgentDisableHostPath bool + priorityClassName string } func WithImage(image string) podTemplateOption { @@ -139,7 +140,10 @@ func WithPodVolumeOperationTimeout(val time.Duration) podTemplateOption { func WithPlugins(plugins []string) podTemplateOption { return func(c *podTemplateConfig) { - c.plugins = plugins + c.plugins = make([]string, 0, len(plugins)) + for _, plugin := range plugins { + c.plugins = append(c.plugins, strings.TrimSpace(plugin)) + } } } @@ -226,6 +230,12 @@ func WithRepoMaintenanceJobConfigMap(repoMaintenanceJobConfigMap string) podTemp } } +func WithDefaultResourceModifierConfigMap(name string) podTemplateOption { + return func(c *podTemplateConfig) { + c.defaultResourceModifierConfigMap = name + } +} + func WithItemBlockWorkerCount(itemBlockWorkerCount int) podTemplateOption { return func(c *podTemplateConfig) { c.itemBlockWorkerCount = itemBlockWorkerCount @@ -347,6 +357,10 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1api.Deployme args = append(args, fmt.Sprintf("--repo-maintenance-job-configmap=%s", c.repoMaintenanceJobConfigMap)) } + if len(c.defaultResourceModifierConfigMap) > 0 { + args = append(args, fmt.Sprintf("--default-resource-modifier-configmap=%s", c.defaultResourceModifierConfigMap)) + } + if c.itemBlockWorkerCount > 0 { args = append(args, fmt.Sprintf("--item-block-worker-count=%d", c.itemBlockWorkerCount)) } @@ -381,7 +395,7 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1api.Deployme { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Values: []string{"windows"}, Operator: corev1api.NodeSelectorOpNotIn, }, @@ -430,6 +444,15 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1api.Deployme }, }, Resources: c.resources, + SecurityContext: &corev1api.SecurityContext{ + Capabilities: &corev1api.Capabilities{ + Drop: []corev1api.Capability{"ALL"}, + }, + AllowPrivilegeEscalation: ptr.To(false), + SeccompProfile: &corev1api.SeccompProfile{ + Type: corev1api.SeccompProfileTypeRuntimeDefault, + }, + }, }, }, Volumes: []corev1api.Volume{ @@ -509,7 +532,7 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1api.Deployme if len(c.plugins) > 0 { for _, image := range c.plugins { - container := *builder.ForPluginContainer(image, pullPolicy).Result() + container := *builder.ForPluginContainer(image, pullPolicy, deployment.Spec.Template.Spec.InitContainers).Result() deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers, container) } } diff --git a/pkg/install/deployment_test.go b/pkg/install/deployment_test.go index 53b696f72..c2d582b8e 100644 --- a/pkg/install/deployment_test.go +++ b/pkg/install/deployment_test.go @@ -60,6 +60,15 @@ func TestDeployment(t *testing.T) { assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2) assert.Equal(t, "--features=EnableCSI,foo,bar,baz", deploy.Spec.Template.Spec.Containers[0].Args[1]) + deploy = Deployment("velero", WithPlugins([]string{ + "harbor-repo.vmware.com/harbor-ci/velero/velero-plugin-for-aws:v1.2.0", + " \n vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1 ", + })) + assert.Len(t, deploy.Spec.Template.Spec.InitContainers, 2) + assert.Equal(t, "harbor-repo.vmware.com/harbor-ci/velero/velero-plugin-for-aws:v1.2.0", deploy.Spec.Template.Spec.InitContainers[0].Image) + assert.Equal(t, "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1", deploy.Spec.Template.Spec.InitContainers[1].Image) + assert.Equal(t, "vsphereveleroplugin-velero-plugin-for-vsphere", deploy.Spec.Template.Spec.InitContainers[1].Name) + deploy = Deployment("velero", WithUploaderType("kopia")) assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2) assert.Equal(t, "--uploader-type=kopia", deploy.Spec.Template.Spec.Containers[0].Args[1]) @@ -100,6 +109,10 @@ func TestDeployment(t *testing.T) { assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2) assert.Equal(t, "--repo-maintenance-job-configmap=test-repo-maintenance-config", deploy.Spec.Template.Spec.Containers[0].Args[1]) + deploy = Deployment("velero", WithDefaultResourceModifierConfigMap("default-restore-modifiers")) + assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2) + assert.Equal(t, "--default-resource-modifier-configmap=default-restore-modifiers", deploy.Spec.Template.Spec.Containers[0].Args[1]) + assert.Equal(t, &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ @@ -107,7 +120,7 @@ func TestDeployment(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Values: []string{"windows"}, Operator: corev1api.NodeSelectorOpNotIn, }, diff --git a/pkg/install/install_test.go b/pkg/install/install_test.go index 47a9aa273..c09250ef2 100644 --- a/pkg/install/install_test.go +++ b/pkg/install/install_test.go @@ -21,7 +21,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client/fake" - v1crds "github.com/vmware-tanzu/velero/config/crd/v1/crds" + v1crds "github.com/vmware-tanzu/velero/config/crd/v1" "github.com/vmware-tanzu/velero/pkg/test" ) diff --git a/pkg/install/resources.go b/pkg/install/resources.go index c4ec6f1bc..3869c1969 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -27,8 +27,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - v1crds "github.com/vmware-tanzu/velero/config/crd/v1/crds" - v2alpha1crds "github.com/vmware-tanzu/velero/config/crd/v2alpha1/crds" + v1crds "github.com/vmware-tanzu/velero/config/crd/v1" + v2alpha1crds "github.com/vmware-tanzu/velero/config/crd/v2alpha1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/util/kube" ) @@ -234,49 +234,50 @@ func appendUnstructured(list *unstructured.UnstructuredList, obj runtime.Object) } type VeleroOptions struct { - Namespace string - Image string - ProviderName string - Bucket string - Prefix string - PodAnnotations map[string]string - PodLabels map[string]string - ServiceAccountAnnotations map[string]string - ServiceAccountName string - VeleroPodResources corev1api.ResourceRequirements - NodeAgentPodResources corev1api.ResourceRequirements - SecretData []byte - RestoreOnly bool - UseNodeAgent bool - UseNodeAgentWindows bool - PrivilegedNodeAgent bool - UseVolumeSnapshots bool - BSLConfig map[string]string - VSLConfig map[string]string - DefaultRepoMaintenanceFrequency time.Duration - GarbageCollectionFrequency time.Duration - PodVolumeOperationTimeout time.Duration - Plugins []string - NoDefaultBackupLocation bool - CACertData []byte - Features []string - DefaultVolumesToFsBackup bool - UploaderType string - DefaultSnapshotMoveData bool - CSISnapshotEarlyFrequentPolling bool - DisableInformerCache bool - ScheduleSkipImmediately bool - PodResources kube.PodResources - KeepLatestMaintenanceJobs int - BackupRepoConfigMap string - RepoMaintenanceJobConfigMap string - NodeAgentConfigMap string - ItemBlockWorkerCount int - ConcurrentBackups int - KubeletRootDir string - NodeAgentDisableHostPath bool - ServerPriorityClassName string - NodeAgentPriorityClassName string + Namespace string + Image string + ProviderName string + Bucket string + Prefix string + PodAnnotations map[string]string + PodLabels map[string]string + ServiceAccountAnnotations map[string]string + ServiceAccountName string + VeleroPodResources corev1api.ResourceRequirements + NodeAgentPodResources corev1api.ResourceRequirements + SecretData []byte + RestoreOnly bool + UseNodeAgent bool + UseNodeAgentWindows bool + PrivilegedNodeAgent bool + UseVolumeSnapshots bool + BSLConfig map[string]string + VSLConfig map[string]string + DefaultRepoMaintenanceFrequency time.Duration + GarbageCollectionFrequency time.Duration + PodVolumeOperationTimeout time.Duration + Plugins []string + NoDefaultBackupLocation bool + CACertData []byte + Features []string + DefaultVolumesToFsBackup bool + UploaderType string + DefaultSnapshotMoveData bool + CSISnapshotEarlyFrequentPolling bool + DisableInformerCache bool + ScheduleSkipImmediately bool + PodResources kube.PodResources + KeepLatestMaintenanceJobs int + BackupRepoConfigMap string + RepoMaintenanceJobConfigMap string + DefaultResourceModifierConfigMap string + NodeAgentConfigMap string + ItemBlockWorkerCount int + ConcurrentBackups int + KubeletRootDir string + NodeAgentDisableHostPath bool + ServerPriorityClassName string + NodeAgentPriorityClassName string } func AllCRDs() *unstructured.UnstructuredList { @@ -407,6 +408,10 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { deployOpts = append(deployOpts, WithRepoMaintenanceJobConfigMap(o.RepoMaintenanceJobConfigMap)) } + if len(o.DefaultResourceModifierConfigMap) > 0 { + deployOpts = append(deployOpts, WithDefaultResourceModifierConfigMap(o.DefaultResourceModifierConfigMap)) + } + deploy := Deployment(o.Namespace, deployOpts...) if err := appendUnstructured(resources, deploy); err != nil { diff --git a/pkg/install/resources_test.go b/pkg/install/resources_test.go index bafa3a684..29b99ed41 100644 --- a/pkg/install/resources_test.go +++ b/pkg/install/resources_test.go @@ -118,6 +118,32 @@ func TestAllResources(t *testing.T) { assert.Len(t, ds, 2) } +func TestAllResourcesWithDefaultResourceModifierConfigMap(t *testing.T) { + option := &VeleroOptions{ + Namespace: "velero", + SecretData: []byte{'a'}, + DefaultResourceModifierConfigMap: "default-rm", + } + list := AllResources(option) + + for _, item := range list.Items { + if item.GetKind() == "Deployment" && item.GetName() == "velero" { + containers, _, _ := unstructured.NestedSlice(item.Object, "spec", "template", "spec", "containers") + args, _, _ := unstructured.NestedStringSlice(containers[0].(map[string]any), "args") + found := false + for _, arg := range args { + if arg == "--default-resource-modifier-configmap=default-rm" { + found = true + break + } + } + assert.True(t, found, "expected --default-resource-modifier-configmap=default-rm in deployment args") + return + } + } + t.Fatal("velero deployment not found in AllResources output") +} + func TestAllResourcesWithPriorityClassName(t *testing.T) { testCases := []struct { name string diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 86d78028c..4661eaec8 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -758,6 +758,14 @@ func (m *ServerMetrics) RegisterPodVolumeOpLatencyGauge(node, pvbName, opName, b } } +// DeleteBackupLastSuccessfulTimestamp removes the backupLastSuccessfulTimestamp +// metric for a single schedule. +func (m *ServerMetrics) DeleteBackupLastSuccessfulTimestamp(scheduleName string) { + if g, ok := m.metrics[backupLastSuccessfulTimestamp].(*prometheus.GaugeVec); ok { + g.DeleteLabelValues(scheduleName) + } +} + // SetBackupTarballSizeBytesGauge records the size, in bytes, of a backup tarball. func (m *ServerMetrics) SetBackupTarballSizeBytesGauge(backupSchedule string, size int64) { if g, ok := m.metrics[backupTarballSizeBytesGauge].(*prometheus.GaugeVec); ok { diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index a24f2bf33..d7f070298 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -21,6 +21,7 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -457,6 +458,32 @@ func getHistogramCount(t *testing.T, vec *prometheus.HistogramVec, scheduleLabel return 0 } +// TestDeleteBackupLastSuccessfulTimestamp verifies that DeleteBackupLastSuccessfulTimestamp +// removes only the specified schedule's metric. +func TestDeleteBackupLastSuccessfulTimestamp(t *testing.T) { + m := NewServerMetrics() + + now := time.Now() + m.SetBackupLastSuccessfulTimestamp("schedule-1", now) + m.SetBackupLastSuccessfulTimestamp("schedule-2", now.Add(-time.Hour)) + m.SetBackupLastSuccessfulTimestamp("", now.Add(-2*time.Hour)) + + g := m.metrics[backupLastSuccessfulTimestamp].(*prometheus.GaugeVec) + assert.Equal(t, 3, testutil.CollectAndCount(g)) + + m.DeleteBackupLastSuccessfulTimestamp("schedule-1") + assert.Equal(t, 2, testutil.CollectAndCount(g)) + assert.Equal(t, float64(now.Add(-time.Hour).Unix()), testutil.ToFloat64(g.WithLabelValues("schedule-2"))) + assert.Equal(t, float64(now.Add(-2*time.Hour).Unix()), testutil.ToFloat64(g.WithLabelValues(""))) + + m.DeleteBackupLastSuccessfulTimestamp("schedule-2") + assert.Equal(t, 1, testutil.CollectAndCount(g)) + assert.Equal(t, float64(now.Add(-2*time.Hour).Unix()), testutil.ToFloat64(g.WithLabelValues(""))) + + m.DeleteBackupLastSuccessfulTimestamp("") + assert.Equal(t, 0, testutil.CollectAndCount(g)) +} + // TestRepoMaintenanceMetrics verifies that repo maintenance metrics are properly recorded. func TestRepoMaintenanceMetrics(t *testing.T) { tests := []struct { diff --git a/pkg/nodeagent/node_agent.go b/pkg/nodeagent/node_agent.go index 61720c99d..b449a91f4 100644 --- a/pkg/nodeagent/node_agent.go +++ b/pkg/nodeagent/node_agent.go @@ -22,6 +22,7 @@ import ( "fmt" "github.com/cockroachdb/errors" + appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -80,6 +81,36 @@ func KbClientIsRunningInNode(ctx context.Context, namespace string, nodeName str return isRunningInNode(ctx, namespace, nodeName, nil, kubeClient) } +// IsReady checks whether the node-agent daemonset has at least one ready pod +// by inspecting the DaemonSet status. +func IsReady(ctx context.Context, namespace string, crClient ctrlclient.Client) error { + dsLinux := new(appsv1api.DaemonSet) + if err := crClient.Get(ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: daemonSet}, dsLinux); err != nil { + dsLinux = nil + if !apierrors.IsNotFound(err) { + return errors.Wrap(err, "failed to get linux node-agent daemonset") + } + } + + dsWindows := new(appsv1api.DaemonSet) + if err := crClient.Get(ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: daemonsetWindows}, dsWindows); err != nil { + dsWindows = nil + if !apierrors.IsNotFound(err) { + return errors.Wrap(err, "failed to get windows node-agent daemonset") + } + } + + if dsLinux != nil && dsLinux.Status.NumberReady > 0 { + return nil + } + + if dsWindows != nil && dsWindows.Status.NumberReady > 0 { + return nil + } + + return errors.New("node-agent is not ready: no ready pods found") +} + // IsRunningInNode checks if the node agent pod is running properly in a specified node through controller client. If not, return the error found func IsRunningInNode(ctx context.Context, namespace string, nodeName string, crClient ctrlclient.Client) error { return isRunningInNode(ctx, namespace, nodeName, crClient, nil) diff --git a/pkg/nodeagent/node_agent_test.go b/pkg/nodeagent/node_agent_test.go index 36b154a75..9bba67ec4 100644 --- a/pkg/nodeagent/node_agent_test.go +++ b/pkg/nodeagent/node_agent_test.go @@ -17,6 +17,7 @@ limitations under the License. package nodeagent import ( + "context" "testing" "github.com/cockroachdb/errors" @@ -28,7 +29,9 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" clientTesting "k8s.io/client-go/testing" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "github.com/vmware-tanzu/velero/pkg/builder" velerotypes "github.com/vmware-tanzu/velero/pkg/types" @@ -213,6 +216,152 @@ func TestIsRunningInNode(t *testing.T) { } } +func TestIsReady(t *testing.T) { + scheme := runtime.NewScheme() + appsv1api.AddToScheme(scheme) + + dsLinuxNotReady := &appsv1api.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Namespace: "fake-ns", Name: "node-agent"}, + Status: appsv1api.DaemonSetStatus{NumberReady: 0}, + } + dsLinuxReady := &appsv1api.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Namespace: "fake-ns", Name: "node-agent"}, + Status: appsv1api.DaemonSetStatus{NumberReady: 3}, + } + dsWindowsNotReady := &appsv1api.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Namespace: "fake-ns", Name: "node-agent-windows"}, + Status: appsv1api.DaemonSetStatus{NumberReady: 0}, + } + dsWindowsReady := &appsv1api.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Namespace: "fake-ns", Name: "node-agent-windows"}, + Status: appsv1api.DaemonSetStatus{NumberReady: 2}, + } + + tests := []struct { + name string + kubeClientObj []runtime.Object + namespace string + interceptor *interceptor.Funcs + expectErr string + }{ + { + name: "both daemonsets not found", + namespace: "fake-ns", + expectErr: "node-agent is not ready: no ready pods found", + }, + { + name: "linux daemonset get error", + namespace: "fake-ns", + interceptor: &interceptor.Funcs{ + Get: func(ctx context.Context, c ctrlclient.WithWatch, key ctrlclient.ObjectKey, obj ctrlclient.Object, opts ...ctrlclient.GetOption) error { + if key.Name == "node-agent" { + return errors.New("fake-get-error") + } + return c.Get(ctx, key, obj, opts...) + }, + }, + expectErr: "failed to get linux node-agent daemonset: fake-get-error", + }, + { + name: "windows daemonset get error", + namespace: "fake-ns", + interceptor: &interceptor.Funcs{ + Get: func(ctx context.Context, c ctrlclient.WithWatch, key ctrlclient.ObjectKey, obj ctrlclient.Object, opts ...ctrlclient.GetOption) error { + if key.Name == "node-agent-windows" { + return errors.New("fake-get-error") + } + return c.Get(ctx, key, obj, opts...) + }, + }, + expectErr: "failed to get windows node-agent daemonset: fake-get-error", + }, + { + name: "linux ds exist but no ready pods", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + dsLinuxNotReady, + }, + expectErr: "node-agent is not ready: no ready pods found", + }, + { + name: "linux ds with ready pods", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + dsLinuxReady, + }, + }, + { + name: "windows ds exist but no ready pods", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + dsWindowsNotReady, + }, + expectErr: "node-agent is not ready: no ready pods found", + }, + { + name: "windows ds with ready pods", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + dsWindowsReady, + }, + }, + { + name: "both daemonsets exist but no ready pods", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + dsLinuxNotReady, + dsWindowsNotReady, + }, + expectErr: "node-agent is not ready: no ready pods found", + }, + { + name: "both daemonsets exist, linux ready", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + dsLinuxReady, + dsWindowsNotReady, + }, + }, + { + name: "both daemonsets exist, windows ready", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + dsLinuxNotReady, + dsWindowsReady, + }, + }, + { + name: "both daemonsets exist, both ready", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + dsLinuxReady, + dsWindowsReady, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + builder := clientFake.NewClientBuilder(). + WithScheme(scheme). + WithRuntimeObjects(test.kubeClientObj...) + + if test.interceptor != nil { + builder = builder.WithInterceptorFuncs(*test.interceptor) + } + + fakeClient := builder.Build() + + err := IsReady(t.Context(), test.namespace, fakeClient) + if test.expectErr == "" { + assert.NoError(t, err) + } else { + assert.ErrorContains(t, err, test.expectErr) + } + }) + } +} + func TestGetPodSpec(t *testing.T) { podSpec := corev1api.PodSpec{ NodeName: "fake-node", diff --git a/pkg/persistence/object_store.go b/pkg/persistence/object_store.go index 440ca8756..338ee016e 100644 --- a/pkg/persistence/object_store.go +++ b/pkg/persistence/object_store.go @@ -94,6 +94,7 @@ type BackupStore interface { // DownloadURLTTL is how long a download URL is valid for. const DownloadURLTTL = 10 * time.Minute +const maxDecompressedSize = 1024 * 1024 * 1024 // 1 GB type objectBackupStore struct { objectStore velero.ObjectStore @@ -164,6 +165,9 @@ func (b *objectBackupStoreGetter) Get(location *velerov1api.BackupStorageLocatio } } + // Delete any user-provided credentialsFile to prevent path traversal vulnerabilities + delete(objectStoreConfig, "credentialsFile") + // add the bucket name and prefix to the config map so that object stores // can use them when initializing. The AWS object store uses the bucket // name to determine the bucket's region when setting up its client. @@ -320,7 +324,8 @@ func (s *objectBackupStore) GetBackupMetadata(name string) (*velerov1api.Backup, } defer res.Close() - data, err := io.ReadAll(res) + limitReader := io.LimitReader(res, maxDecompressedSize) + data, err := io.ReadAll(limitReader) if err != nil { return nil, errors.WithStack(err) } @@ -431,7 +436,9 @@ func decode(jsongzReader io.Reader, into any) error { } defer gzr.Close() - if err := json.NewDecoder(gzr).Decode(into); err != nil { + limitReader := io.LimitReader(gzr, maxDecompressedSize) + + if err := json.NewDecoder(limitReader).Decode(into); err != nil { return errors.Wrap(err, "error decoding object data") } diff --git a/pkg/plugin/generated/BackupItemAction.pb.go b/pkg/plugin/generated/BackupItemAction.pb.go index 5d3d3cb7e..f9f363d1e 100644 --- a/pkg/plugin/generated/BackupItemAction.pb.go +++ b/pkg/plugin/generated/BackupItemAction.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 +// protoc-gen-go v1.36.11-devel // protoc v4.25.2 // source: BackupItemAction.proto @@ -11,6 +11,7 @@ import ( protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" + unsafe "unsafe" ) const ( @@ -21,22 +22,19 @@ const ( ) type ExecuteRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + Item []byte `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` + Backup []byte `protobuf:"bytes,3,opt,name=backup,proto3" json:"backup,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - Item []byte `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` - Backup []byte `protobuf:"bytes,3,opt,name=backup,proto3" json:"backup,omitempty"` + sizeCache protoimpl.SizeCache } func (x *ExecuteRequest) Reset() { *x = ExecuteRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_BackupItemAction_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_BackupItemAction_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ExecuteRequest) String() string { @@ -47,7 +45,7 @@ func (*ExecuteRequest) ProtoMessage() {} func (x *ExecuteRequest) ProtoReflect() protoreflect.Message { mi := &file_BackupItemAction_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -84,21 +82,18 @@ func (x *ExecuteRequest) GetBackup() []byte { } type ExecuteResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Item []byte `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"` - AdditionalItems []*ResourceIdentifier `protobuf:"bytes,2,rep,name=additionalItems,proto3" json:"additionalItems,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Item []byte `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"` + AdditionalItems []*ResourceIdentifier `protobuf:"bytes,2,rep,name=additionalItems,proto3" json:"additionalItems,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ExecuteResponse) Reset() { *x = ExecuteResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_BackupItemAction_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_BackupItemAction_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ExecuteResponse) String() string { @@ -109,7 +104,7 @@ func (*ExecuteResponse) ProtoMessage() {} func (x *ExecuteResponse) ProtoReflect() protoreflect.Message { mi := &file_BackupItemAction_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -139,20 +134,17 @@ func (x *ExecuteResponse) GetAdditionalItems() []*ResourceIdentifier { } type BackupItemActionAppliesToRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + sizeCache protoimpl.SizeCache } func (x *BackupItemActionAppliesToRequest) Reset() { *x = BackupItemActionAppliesToRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_BackupItemAction_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_BackupItemAction_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *BackupItemActionAppliesToRequest) String() string { @@ -163,7 +155,7 @@ func (*BackupItemActionAppliesToRequest) ProtoMessage() {} func (x *BackupItemActionAppliesToRequest) ProtoReflect() protoreflect.Message { mi := &file_BackupItemAction_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -186,20 +178,17 @@ func (x *BackupItemActionAppliesToRequest) GetPlugin() string { } type BackupItemActionAppliesToResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - ResourceSelector *ResourceSelector `protobuf:"bytes,1,opt,name=ResourceSelector,proto3" json:"ResourceSelector,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + ResourceSelector *ResourceSelector `protobuf:"bytes,1,opt,name=ResourceSelector,proto3" json:"ResourceSelector,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *BackupItemActionAppliesToResponse) Reset() { *x = BackupItemActionAppliesToResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_BackupItemAction_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_BackupItemAction_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *BackupItemActionAppliesToResponse) String() string { @@ -210,7 +199,7 @@ func (*BackupItemActionAppliesToResponse) ProtoMessage() {} func (x *BackupItemActionAppliesToResponse) ProtoReflect() protoreflect.Message { mi := &file_BackupItemAction_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -234,65 +223,38 @@ func (x *BackupItemActionAppliesToResponse) GetResourceSelector() *ResourceSelec var File_BackupItemAction_proto protoreflect.FileDescriptor -var file_BackupItemAction_proto_rawDesc = []byte{ - 0x0a, 0x16, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, - 0x74, 0x65, 0x64, 0x1a, 0x0c, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x22, 0x54, 0x0a, 0x0e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x69, - 0x74, 0x65, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, - 0x16, 0x0a, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x22, 0x6e, 0x0a, 0x0f, 0x45, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, - 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x47, - 0x0a, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, - 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, - 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x65, 0x6e, - 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x52, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, - 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x3a, 0x0a, 0x20, 0x42, 0x61, 0x63, 0x6b, 0x75, - 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, - 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, - 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, - 0x67, 0x69, 0x6e, 0x22, 0x6c, 0x0a, 0x21, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, - 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, - 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, - 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x32, 0xbc, 0x01, 0x0a, 0x10, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, - 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x66, 0x0a, 0x09, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, - 0x73, 0x54, 0x6f, 0x12, 0x2b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, - 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x2c, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x42, 0x61, 0x63, - 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, - 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x40, - 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x12, 0x19, 0x2e, 0x67, 0x65, 0x6e, 0x65, - 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, - 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, - 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, 0x6e, 0x7a, 0x75, 0x2f, 0x76, 0x65, 0x6c, 0x65, - 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, 0x67, 0x65, - 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} +const file_BackupItemAction_proto_rawDesc = "" + + "\n" + + "\x16BackupItemAction.proto\x12\tgenerated\x1a\fShared.proto\"T\n" + + "\x0eExecuteRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12\x12\n" + + "\x04item\x18\x02 \x01(\fR\x04item\x12\x16\n" + + "\x06backup\x18\x03 \x01(\fR\x06backup\"n\n" + + "\x0fExecuteResponse\x12\x12\n" + + "\x04item\x18\x01 \x01(\fR\x04item\x12G\n" + + "\x0fadditionalItems\x18\x02 \x03(\v2\x1d.generated.ResourceIdentifierR\x0fadditionalItems\":\n" + + " BackupItemActionAppliesToRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\"l\n" + + "!BackupItemActionAppliesToResponse\x12G\n" + + "\x10ResourceSelector\x18\x01 \x01(\v2\x1b.generated.ResourceSelectorR\x10ResourceSelector2\xbc\x01\n" + + "\x10BackupItemAction\x12f\n" + + "\tAppliesTo\x12+.generated.BackupItemActionAppliesToRequest\x1a,.generated.BackupItemActionAppliesToResponse\x12@\n" + + "\aExecute\x12\x19.generated.ExecuteRequest\x1a\x1a.generated.ExecuteResponseB5Z3github.com/vmware-tanzu/velero/pkg/plugin/generatedb\x06proto3" var ( file_BackupItemAction_proto_rawDescOnce sync.Once - file_BackupItemAction_proto_rawDescData = file_BackupItemAction_proto_rawDesc + file_BackupItemAction_proto_rawDescData []byte ) func file_BackupItemAction_proto_rawDescGZIP() []byte { file_BackupItemAction_proto_rawDescOnce.Do(func() { - file_BackupItemAction_proto_rawDescData = protoimpl.X.CompressGZIP(file_BackupItemAction_proto_rawDescData) + file_BackupItemAction_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_BackupItemAction_proto_rawDesc), len(file_BackupItemAction_proto_rawDesc))) }) return file_BackupItemAction_proto_rawDescData } var file_BackupItemAction_proto_msgTypes = make([]protoimpl.MessageInfo, 4) -var file_BackupItemAction_proto_goTypes = []interface{}{ +var file_BackupItemAction_proto_goTypes = []any{ (*ExecuteRequest)(nil), // 0: generated.ExecuteRequest (*ExecuteResponse)(nil), // 1: generated.ExecuteResponse (*BackupItemActionAppliesToRequest)(nil), // 2: generated.BackupItemActionAppliesToRequest @@ -320,61 +282,11 @@ func file_BackupItemAction_proto_init() { return } file_Shared_proto_init() - if !protoimpl.UnsafeEnabled { - file_BackupItemAction_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ExecuteRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_BackupItemAction_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ExecuteResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_BackupItemAction_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BackupItemActionAppliesToRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_BackupItemAction_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BackupItemActionAppliesToResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_BackupItemAction_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_BackupItemAction_proto_rawDesc), len(file_BackupItemAction_proto_rawDesc)), NumEnums: 0, NumMessages: 4, NumExtensions: 0, @@ -385,7 +297,6 @@ func file_BackupItemAction_proto_init() { MessageInfos: file_BackupItemAction_proto_msgTypes, }.Build() File_BackupItemAction_proto = out.File - file_BackupItemAction_proto_rawDesc = nil file_BackupItemAction_proto_goTypes = nil file_BackupItemAction_proto_depIdxs = nil } diff --git a/pkg/plugin/generated/DeleteItemAction.pb.go b/pkg/plugin/generated/DeleteItemAction.pb.go index 871b63889..3935cf216 100644 --- a/pkg/plugin/generated/DeleteItemAction.pb.go +++ b/pkg/plugin/generated/DeleteItemAction.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 +// protoc-gen-go v1.36.11-devel // protoc v4.25.2 // source: DeleteItemAction.proto @@ -11,6 +11,7 @@ import ( protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" + unsafe "unsafe" ) const ( @@ -21,22 +22,19 @@ const ( ) type DeleteItemActionExecuteRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + Item []byte `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` + Backup []byte `protobuf:"bytes,3,opt,name=backup,proto3" json:"backup,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - Item []byte `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` - Backup []byte `protobuf:"bytes,3,opt,name=backup,proto3" json:"backup,omitempty"` + sizeCache protoimpl.SizeCache } func (x *DeleteItemActionExecuteRequest) Reset() { *x = DeleteItemActionExecuteRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_DeleteItemAction_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_DeleteItemAction_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *DeleteItemActionExecuteRequest) String() string { @@ -47,7 +45,7 @@ func (*DeleteItemActionExecuteRequest) ProtoMessage() {} func (x *DeleteItemActionExecuteRequest) ProtoReflect() protoreflect.Message { mi := &file_DeleteItemAction_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -84,20 +82,17 @@ func (x *DeleteItemActionExecuteRequest) GetBackup() []byte { } type DeleteItemActionAppliesToRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + sizeCache protoimpl.SizeCache } func (x *DeleteItemActionAppliesToRequest) Reset() { *x = DeleteItemActionAppliesToRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_DeleteItemAction_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_DeleteItemAction_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *DeleteItemActionAppliesToRequest) String() string { @@ -108,7 +103,7 @@ func (*DeleteItemActionAppliesToRequest) ProtoMessage() {} func (x *DeleteItemActionAppliesToRequest) ProtoReflect() protoreflect.Message { mi := &file_DeleteItemAction_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -131,20 +126,17 @@ func (x *DeleteItemActionAppliesToRequest) GetPlugin() string { } type DeleteItemActionAppliesToResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - ResourceSelector *ResourceSelector `protobuf:"bytes,1,opt,name=ResourceSelector,proto3" json:"ResourceSelector,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + ResourceSelector *ResourceSelector `protobuf:"bytes,1,opt,name=ResourceSelector,proto3" json:"ResourceSelector,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *DeleteItemActionAppliesToResponse) Reset() { *x = DeleteItemActionAppliesToResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_DeleteItemAction_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_DeleteItemAction_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *DeleteItemActionAppliesToResponse) String() string { @@ -155,7 +147,7 @@ func (*DeleteItemActionAppliesToResponse) ProtoMessage() {} func (x *DeleteItemActionAppliesToResponse) ProtoReflect() protoreflect.Message { mi := &file_DeleteItemAction_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -179,60 +171,35 @@ func (x *DeleteItemActionAppliesToResponse) GetResourceSelector() *ResourceSelec var File_DeleteItemAction_proto protoreflect.FileDescriptor -var file_DeleteItemAction_proto_rawDesc = []byte{ - 0x0a, 0x16, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, - 0x74, 0x65, 0x64, 0x1a, 0x0c, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x22, 0x64, 0x0a, 0x1e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x69, - 0x74, 0x65, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, - 0x16, 0x0a, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x22, 0x3a, 0x0a, 0x20, 0x44, 0x65, 0x6c, 0x65, 0x74, - 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, - 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, - 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, - 0x67, 0x69, 0x6e, 0x22, 0x6c, 0x0a, 0x21, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x74, 0x65, - 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, - 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, - 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x32, 0xc2, 0x01, 0x0a, 0x10, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, - 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x66, 0x0a, 0x09, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, - 0x73, 0x54, 0x6f, 0x12, 0x2b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, - 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x2c, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x44, 0x65, 0x6c, - 0x65, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, - 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, - 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x12, 0x29, 0x2e, 0x67, 0x65, 0x6e, 0x65, - 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, - 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, 0x6e, 0x7a, - 0x75, 0x2f, 0x76, 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, - 0x67, 0x69, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, -} +const file_DeleteItemAction_proto_rawDesc = "" + + "\n" + + "\x16DeleteItemAction.proto\x12\tgenerated\x1a\fShared.proto\"d\n" + + "\x1eDeleteItemActionExecuteRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12\x12\n" + + "\x04item\x18\x02 \x01(\fR\x04item\x12\x16\n" + + "\x06backup\x18\x03 \x01(\fR\x06backup\":\n" + + " DeleteItemActionAppliesToRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\"l\n" + + "!DeleteItemActionAppliesToResponse\x12G\n" + + "\x10ResourceSelector\x18\x01 \x01(\v2\x1b.generated.ResourceSelectorR\x10ResourceSelector2\xc2\x01\n" + + "\x10DeleteItemAction\x12f\n" + + "\tAppliesTo\x12+.generated.DeleteItemActionAppliesToRequest\x1a,.generated.DeleteItemActionAppliesToResponse\x12F\n" + + "\aExecute\x12).generated.DeleteItemActionExecuteRequest\x1a\x10.generated.EmptyB5Z3github.com/vmware-tanzu/velero/pkg/plugin/generatedb\x06proto3" var ( file_DeleteItemAction_proto_rawDescOnce sync.Once - file_DeleteItemAction_proto_rawDescData = file_DeleteItemAction_proto_rawDesc + file_DeleteItemAction_proto_rawDescData []byte ) func file_DeleteItemAction_proto_rawDescGZIP() []byte { file_DeleteItemAction_proto_rawDescOnce.Do(func() { - file_DeleteItemAction_proto_rawDescData = protoimpl.X.CompressGZIP(file_DeleteItemAction_proto_rawDescData) + file_DeleteItemAction_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_DeleteItemAction_proto_rawDesc), len(file_DeleteItemAction_proto_rawDesc))) }) return file_DeleteItemAction_proto_rawDescData } var file_DeleteItemAction_proto_msgTypes = make([]protoimpl.MessageInfo, 3) -var file_DeleteItemAction_proto_goTypes = []interface{}{ +var file_DeleteItemAction_proto_goTypes = []any{ (*DeleteItemActionExecuteRequest)(nil), // 0: generated.DeleteItemActionExecuteRequest (*DeleteItemActionAppliesToRequest)(nil), // 1: generated.DeleteItemActionAppliesToRequest (*DeleteItemActionAppliesToResponse)(nil), // 2: generated.DeleteItemActionAppliesToResponse @@ -258,49 +225,11 @@ func file_DeleteItemAction_proto_init() { return } file_Shared_proto_init() - if !protoimpl.UnsafeEnabled { - file_DeleteItemAction_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DeleteItemActionExecuteRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_DeleteItemAction_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DeleteItemActionAppliesToRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_DeleteItemAction_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DeleteItemActionAppliesToResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_DeleteItemAction_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_DeleteItemAction_proto_rawDesc), len(file_DeleteItemAction_proto_rawDesc)), NumEnums: 0, NumMessages: 3, NumExtensions: 0, @@ -311,7 +240,6 @@ func file_DeleteItemAction_proto_init() { MessageInfos: file_DeleteItemAction_proto_msgTypes, }.Build() File_DeleteItemAction_proto = out.File - file_DeleteItemAction_proto_rawDesc = nil file_DeleteItemAction_proto_goTypes = nil file_DeleteItemAction_proto_depIdxs = nil } diff --git a/pkg/plugin/generated/ObjectStore.pb.go b/pkg/plugin/generated/ObjectStore.pb.go index 563849355..c960f159c 100644 --- a/pkg/plugin/generated/ObjectStore.pb.go +++ b/pkg/plugin/generated/ObjectStore.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 +// protoc-gen-go v1.36.11-devel // protoc v4.25.2 // source: ObjectStore.proto @@ -11,6 +11,7 @@ import ( protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" + unsafe "unsafe" ) const ( @@ -21,23 +22,20 @@ const ( ) type PutObjectRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` + Key string `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` + Body []byte `protobuf:"bytes,4,opt,name=body,proto3" json:"body,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` - Key string `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` - Body []byte `protobuf:"bytes,4,opt,name=body,proto3" json:"body,omitempty"` + sizeCache protoimpl.SizeCache } func (x *PutObjectRequest) Reset() { *x = PutObjectRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_ObjectStore_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_ObjectStore_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *PutObjectRequest) String() string { @@ -48,7 +46,7 @@ func (*PutObjectRequest) ProtoMessage() {} func (x *PutObjectRequest) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -92,22 +90,19 @@ func (x *PutObjectRequest) GetBody() []byte { } type ObjectExistsRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` + Key string `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` - Key string `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` + sizeCache protoimpl.SizeCache } func (x *ObjectExistsRequest) Reset() { *x = ObjectExistsRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_ObjectStore_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_ObjectStore_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ObjectExistsRequest) String() string { @@ -118,7 +113,7 @@ func (*ObjectExistsRequest) ProtoMessage() {} func (x *ObjectExistsRequest) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -155,20 +150,17 @@ func (x *ObjectExistsRequest) GetKey() string { } type ObjectExistsResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Exists bool `protobuf:"varint,1,opt,name=exists,proto3" json:"exists,omitempty"` unknownFields protoimpl.UnknownFields - - Exists bool `protobuf:"varint,1,opt,name=exists,proto3" json:"exists,omitempty"` + sizeCache protoimpl.SizeCache } func (x *ObjectExistsResponse) Reset() { *x = ObjectExistsResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_ObjectStore_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_ObjectStore_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ObjectExistsResponse) String() string { @@ -179,7 +171,7 @@ func (*ObjectExistsResponse) ProtoMessage() {} func (x *ObjectExistsResponse) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -202,22 +194,19 @@ func (x *ObjectExistsResponse) GetExists() bool { } type GetObjectRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` + Key string `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` - Key string `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` + sizeCache protoimpl.SizeCache } func (x *GetObjectRequest) Reset() { *x = GetObjectRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_ObjectStore_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_ObjectStore_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *GetObjectRequest) String() string { @@ -228,7 +217,7 @@ func (*GetObjectRequest) ProtoMessage() {} func (x *GetObjectRequest) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -265,20 +254,17 @@ func (x *GetObjectRequest) GetKey() string { } type Bytes struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` unknownFields protoimpl.UnknownFields - - Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + sizeCache protoimpl.SizeCache } func (x *Bytes) Reset() { *x = Bytes{} - if protoimpl.UnsafeEnabled { - mi := &file_ObjectStore_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_ObjectStore_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *Bytes) String() string { @@ -289,7 +275,7 @@ func (*Bytes) ProtoMessage() {} func (x *Bytes) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -312,23 +298,20 @@ func (x *Bytes) GetData() []byte { } type ListCommonPrefixesRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` + Delimiter string `protobuf:"bytes,3,opt,name=delimiter,proto3" json:"delimiter,omitempty"` + Prefix string `protobuf:"bytes,4,opt,name=prefix,proto3" json:"prefix,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` - Delimiter string `protobuf:"bytes,3,opt,name=delimiter,proto3" json:"delimiter,omitempty"` - Prefix string `protobuf:"bytes,4,opt,name=prefix,proto3" json:"prefix,omitempty"` + sizeCache protoimpl.SizeCache } func (x *ListCommonPrefixesRequest) Reset() { *x = ListCommonPrefixesRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_ObjectStore_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_ObjectStore_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ListCommonPrefixesRequest) String() string { @@ -339,7 +322,7 @@ func (*ListCommonPrefixesRequest) ProtoMessage() {} func (x *ListCommonPrefixesRequest) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -383,20 +366,17 @@ func (x *ListCommonPrefixesRequest) GetPrefix() string { } type ListCommonPrefixesResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Prefixes []string `protobuf:"bytes,1,rep,name=prefixes,proto3" json:"prefixes,omitempty"` unknownFields protoimpl.UnknownFields - - Prefixes []string `protobuf:"bytes,1,rep,name=prefixes,proto3" json:"prefixes,omitempty"` + sizeCache protoimpl.SizeCache } func (x *ListCommonPrefixesResponse) Reset() { *x = ListCommonPrefixesResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_ObjectStore_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_ObjectStore_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ListCommonPrefixesResponse) String() string { @@ -407,7 +387,7 @@ func (*ListCommonPrefixesResponse) ProtoMessage() {} func (x *ListCommonPrefixesResponse) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -430,22 +410,19 @@ func (x *ListCommonPrefixesResponse) GetPrefixes() []string { } type ListObjectsRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` + Prefix string `protobuf:"bytes,3,opt,name=prefix,proto3" json:"prefix,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` - Prefix string `protobuf:"bytes,3,opt,name=prefix,proto3" json:"prefix,omitempty"` + sizeCache protoimpl.SizeCache } func (x *ListObjectsRequest) Reset() { *x = ListObjectsRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_ObjectStore_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_ObjectStore_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ListObjectsRequest) String() string { @@ -456,7 +433,7 @@ func (*ListObjectsRequest) ProtoMessage() {} func (x *ListObjectsRequest) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[7] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -493,20 +470,17 @@ func (x *ListObjectsRequest) GetPrefix() string { } type ListObjectsResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` unknownFields protoimpl.UnknownFields - - Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` + sizeCache protoimpl.SizeCache } func (x *ListObjectsResponse) Reset() { *x = ListObjectsResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_ObjectStore_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_ObjectStore_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ListObjectsResponse) String() string { @@ -517,7 +491,7 @@ func (*ListObjectsResponse) ProtoMessage() {} func (x *ListObjectsResponse) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[8] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -540,22 +514,19 @@ func (x *ListObjectsResponse) GetKeys() []string { } type DeleteObjectRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` + Key string `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` - Key string `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` + sizeCache protoimpl.SizeCache } func (x *DeleteObjectRequest) Reset() { *x = DeleteObjectRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_ObjectStore_proto_msgTypes[9] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_ObjectStore_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *DeleteObjectRequest) String() string { @@ -566,7 +537,7 @@ func (*DeleteObjectRequest) ProtoMessage() {} func (x *DeleteObjectRequest) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[9] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -603,23 +574,20 @@ func (x *DeleteObjectRequest) GetKey() string { } type CreateSignedURLRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` + Key string `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` + Ttl int64 `protobuf:"varint,4,opt,name=ttl,proto3" json:"ttl,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - Bucket string `protobuf:"bytes,2,opt,name=bucket,proto3" json:"bucket,omitempty"` - Key string `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` - Ttl int64 `protobuf:"varint,4,opt,name=ttl,proto3" json:"ttl,omitempty"` + sizeCache protoimpl.SizeCache } func (x *CreateSignedURLRequest) Reset() { *x = CreateSignedURLRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_ObjectStore_proto_msgTypes[10] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_ObjectStore_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *CreateSignedURLRequest) String() string { @@ -630,7 +598,7 @@ func (*CreateSignedURLRequest) ProtoMessage() {} func (x *CreateSignedURLRequest) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[10] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -674,20 +642,17 @@ func (x *CreateSignedURLRequest) GetTtl() int64 { } type CreateSignedURLResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` unknownFields protoimpl.UnknownFields - - Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` + sizeCache protoimpl.SizeCache } func (x *CreateSignedURLResponse) Reset() { *x = CreateSignedURLResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_ObjectStore_proto_msgTypes[11] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_ObjectStore_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *CreateSignedURLResponse) String() string { @@ -698,7 +663,7 @@ func (*CreateSignedURLResponse) ProtoMessage() {} func (x *CreateSignedURLResponse) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[11] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -721,21 +686,18 @@ func (x *CreateSignedURLResponse) GetUrl() string { } type ObjectStoreInitRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + Config map[string]string `protobuf:"bytes,2,rep,name=config,proto3" json:"config,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - Config map[string]string `protobuf:"bytes,2,rep,name=config,proto3" json:"config,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + sizeCache protoimpl.SizeCache } func (x *ObjectStoreInitRequest) Reset() { *x = ObjectStoreInitRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_ObjectStore_proto_msgTypes[12] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_ObjectStore_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ObjectStoreInitRequest) String() string { @@ -746,7 +708,7 @@ func (*ObjectStoreInitRequest) ProtoMessage() {} func (x *ObjectStoreInitRequest) ProtoReflect() protoreflect.Message { mi := &file_ObjectStore_proto_msgTypes[12] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -777,138 +739,80 @@ func (x *ObjectStoreInitRequest) GetConfig() map[string]string { var File_ObjectStore_proto protoreflect.FileDescriptor -var file_ObjectStore_proto_rawDesc = []byte{ - 0x0a, 0x11, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x1a, 0x0c, - 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x68, 0x0a, 0x10, - 0x50, 0x75, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, - 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x22, 0x57, 0x0a, 0x13, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x45, 0x78, 0x69, 0x73, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, - 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, - 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, - 0x2e, 0x0a, 0x14, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x45, 0x78, 0x69, 0x73, 0x74, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x65, 0x78, 0x69, 0x73, 0x74, - 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x65, 0x78, 0x69, 0x73, 0x74, 0x73, 0x22, - 0x54, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x62, - 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, - 0x6b, 0x65, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x1b, 0x0a, 0x05, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x12, - 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, - 0x74, 0x61, 0x22, 0x81, 0x01, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x6f, - 0x6e, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, - 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, - 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x12, 0x16, - 0x0a, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, - 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x22, 0x38, 0x0a, 0x1a, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, - 0x6d, 0x6d, 0x6f, 0x6e, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, - 0x22, 0x5c, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x16, - 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, - 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x22, 0x29, - 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x22, 0x57, 0x0a, 0x13, 0x44, 0x65, 0x6c, - 0x65, 0x74, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, - 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x22, 0x6c, 0x0a, 0x16, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x69, 0x67, 0x6e, - 0x65, 0x64, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, - 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, - 0x75, 0x67, 0x69, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x74, 0x74, 0x6c, - 0x22, 0x2b, 0x0a, 0x17, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x64, - 0x55, 0x52, 0x4c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, - 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0xb2, 0x01, - 0x0a, 0x16, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x6e, 0x69, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, - 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, - 0x12, 0x45, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x2d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4f, 0x62, 0x6a, - 0x65, 0x63, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a, 0x39, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x32, 0xe4, 0x04, 0x0a, 0x0b, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x74, 0x6f, - 0x72, 0x65, 0x12, 0x3b, 0x0a, 0x04, 0x49, 0x6e, 0x69, 0x74, 0x12, 0x21, 0x2e, 0x67, 0x65, 0x6e, - 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x74, 0x6f, - 0x72, 0x65, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, - 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, - 0x3c, 0x0a, 0x09, 0x50, 0x75, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1b, 0x2e, 0x67, - 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x50, 0x75, 0x74, 0x4f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x67, 0x65, 0x6e, 0x65, - 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x28, 0x01, 0x12, 0x4f, 0x0a, - 0x0c, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x45, 0x78, 0x69, 0x73, 0x74, 0x73, 0x12, 0x1e, 0x2e, - 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x45, 0x78, 0x69, 0x73, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, - 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x45, 0x78, 0x69, 0x73, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, - 0x0a, 0x09, 0x47, 0x65, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1b, 0x2e, 0x67, 0x65, - 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x65, 0x64, 0x2e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x30, 0x01, 0x12, 0x61, 0x0a, 0x12, - 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, - 0x65, 0x73, 0x12, 0x24, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x50, - 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x4c, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1d, - 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4f, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, - 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x40, 0x0a, - 0x0c, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1e, 0x2e, - 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, - 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, - 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, - 0x58, 0x0a, 0x0f, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x55, - 0x52, 0x4c, 0x12, 0x21, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x43, - 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x55, 0x52, 0x4c, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, - 0x64, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x55, 0x52, - 0x4c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, - 0x61, 0x6e, 0x7a, 0x75, 0x2f, 0x76, 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, - 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} +const file_ObjectStore_proto_rawDesc = "" + + "\n" + + "\x11ObjectStore.proto\x12\tgenerated\x1a\fShared.proto\"h\n" + + "\x10PutObjectRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12\x16\n" + + "\x06bucket\x18\x02 \x01(\tR\x06bucket\x12\x10\n" + + "\x03key\x18\x03 \x01(\tR\x03key\x12\x12\n" + + "\x04body\x18\x04 \x01(\fR\x04body\"W\n" + + "\x13ObjectExistsRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12\x16\n" + + "\x06bucket\x18\x02 \x01(\tR\x06bucket\x12\x10\n" + + "\x03key\x18\x03 \x01(\tR\x03key\".\n" + + "\x14ObjectExistsResponse\x12\x16\n" + + "\x06exists\x18\x01 \x01(\bR\x06exists\"T\n" + + "\x10GetObjectRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12\x16\n" + + "\x06bucket\x18\x02 \x01(\tR\x06bucket\x12\x10\n" + + "\x03key\x18\x03 \x01(\tR\x03key\"\x1b\n" + + "\x05Bytes\x12\x12\n" + + "\x04data\x18\x01 \x01(\fR\x04data\"\x81\x01\n" + + "\x19ListCommonPrefixesRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12\x16\n" + + "\x06bucket\x18\x02 \x01(\tR\x06bucket\x12\x1c\n" + + "\tdelimiter\x18\x03 \x01(\tR\tdelimiter\x12\x16\n" + + "\x06prefix\x18\x04 \x01(\tR\x06prefix\"8\n" + + "\x1aListCommonPrefixesResponse\x12\x1a\n" + + "\bprefixes\x18\x01 \x03(\tR\bprefixes\"\\\n" + + "\x12ListObjectsRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12\x16\n" + + "\x06bucket\x18\x02 \x01(\tR\x06bucket\x12\x16\n" + + "\x06prefix\x18\x03 \x01(\tR\x06prefix\")\n" + + "\x13ListObjectsResponse\x12\x12\n" + + "\x04keys\x18\x01 \x03(\tR\x04keys\"W\n" + + "\x13DeleteObjectRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12\x16\n" + + "\x06bucket\x18\x02 \x01(\tR\x06bucket\x12\x10\n" + + "\x03key\x18\x03 \x01(\tR\x03key\"l\n" + + "\x16CreateSignedURLRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12\x16\n" + + "\x06bucket\x18\x02 \x01(\tR\x06bucket\x12\x10\n" + + "\x03key\x18\x03 \x01(\tR\x03key\x12\x10\n" + + "\x03ttl\x18\x04 \x01(\x03R\x03ttl\"+\n" + + "\x17CreateSignedURLResponse\x12\x10\n" + + "\x03url\x18\x01 \x01(\tR\x03url\"\xb2\x01\n" + + "\x16ObjectStoreInitRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12E\n" + + "\x06config\x18\x02 \x03(\v2-.generated.ObjectStoreInitRequest.ConfigEntryR\x06config\x1a9\n" + + "\vConfigEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x012\xe4\x04\n" + + "\vObjectStore\x12;\n" + + "\x04Init\x12!.generated.ObjectStoreInitRequest\x1a\x10.generated.Empty\x12<\n" + + "\tPutObject\x12\x1b.generated.PutObjectRequest\x1a\x10.generated.Empty(\x01\x12O\n" + + "\fObjectExists\x12\x1e.generated.ObjectExistsRequest\x1a\x1f.generated.ObjectExistsResponse\x12<\n" + + "\tGetObject\x12\x1b.generated.GetObjectRequest\x1a\x10.generated.Bytes0\x01\x12a\n" + + "\x12ListCommonPrefixes\x12$.generated.ListCommonPrefixesRequest\x1a%.generated.ListCommonPrefixesResponse\x12L\n" + + "\vListObjects\x12\x1d.generated.ListObjectsRequest\x1a\x1e.generated.ListObjectsResponse\x12@\n" + + "\fDeleteObject\x12\x1e.generated.DeleteObjectRequest\x1a\x10.generated.Empty\x12X\n" + + "\x0fCreateSignedURL\x12!.generated.CreateSignedURLRequest\x1a\".generated.CreateSignedURLResponseB5Z3github.com/vmware-tanzu/velero/pkg/plugin/generatedb\x06proto3" var ( file_ObjectStore_proto_rawDescOnce sync.Once - file_ObjectStore_proto_rawDescData = file_ObjectStore_proto_rawDesc + file_ObjectStore_proto_rawDescData []byte ) func file_ObjectStore_proto_rawDescGZIP() []byte { file_ObjectStore_proto_rawDescOnce.Do(func() { - file_ObjectStore_proto_rawDescData = protoimpl.X.CompressGZIP(file_ObjectStore_proto_rawDescData) + file_ObjectStore_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_ObjectStore_proto_rawDesc), len(file_ObjectStore_proto_rawDesc))) }) return file_ObjectStore_proto_rawDescData } var file_ObjectStore_proto_msgTypes = make([]protoimpl.MessageInfo, 14) -var file_ObjectStore_proto_goTypes = []interface{}{ +var file_ObjectStore_proto_goTypes = []any{ (*PutObjectRequest)(nil), // 0: generated.PutObjectRequest (*ObjectExistsRequest)(nil), // 1: generated.ObjectExistsRequest (*ObjectExistsResponse)(nil), // 2: generated.ObjectExistsResponse @@ -956,169 +860,11 @@ func file_ObjectStore_proto_init() { return } file_Shared_proto_init() - if !protoimpl.UnsafeEnabled { - file_ObjectStore_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutObjectRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ObjectStore_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ObjectExistsRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ObjectStore_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ObjectExistsResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ObjectStore_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetObjectRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ObjectStore_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Bytes); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ObjectStore_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListCommonPrefixesRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ObjectStore_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListCommonPrefixesResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ObjectStore_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListObjectsRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ObjectStore_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListObjectsResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ObjectStore_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DeleteObjectRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ObjectStore_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateSignedURLRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ObjectStore_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateSignedURLResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_ObjectStore_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ObjectStoreInitRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_ObjectStore_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_ObjectStore_proto_rawDesc), len(file_ObjectStore_proto_rawDesc)), NumEnums: 0, NumMessages: 14, NumExtensions: 0, @@ -1129,7 +875,6 @@ func file_ObjectStore_proto_init() { MessageInfos: file_ObjectStore_proto_msgTypes, }.Build() File_ObjectStore_proto = out.File - file_ObjectStore_proto_rawDesc = nil file_ObjectStore_proto_goTypes = nil file_ObjectStore_proto_depIdxs = nil } diff --git a/pkg/plugin/generated/PluginLister.pb.go b/pkg/plugin/generated/PluginLister.pb.go index 590265750..239e57266 100644 --- a/pkg/plugin/generated/PluginLister.pb.go +++ b/pkg/plugin/generated/PluginLister.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 +// protoc-gen-go v1.36.11-devel // protoc v4.25.2 // source: PluginLister.proto @@ -11,6 +11,7 @@ import ( protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" + unsafe "unsafe" ) const ( @@ -21,22 +22,19 @@ const ( ) type PluginIdentifier struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Command string `protobuf:"bytes,1,opt,name=command,proto3" json:"command,omitempty"` + Kind string `protobuf:"bytes,2,opt,name=kind,proto3" json:"kind,omitempty"` + Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` unknownFields protoimpl.UnknownFields - - Command string `protobuf:"bytes,1,opt,name=command,proto3" json:"command,omitempty"` - Kind string `protobuf:"bytes,2,opt,name=kind,proto3" json:"kind,omitempty"` - Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` + sizeCache protoimpl.SizeCache } func (x *PluginIdentifier) Reset() { *x = PluginIdentifier{} - if protoimpl.UnsafeEnabled { - mi := &file_PluginLister_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_PluginLister_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *PluginIdentifier) String() string { @@ -47,7 +45,7 @@ func (*PluginIdentifier) ProtoMessage() {} func (x *PluginIdentifier) ProtoReflect() protoreflect.Message { mi := &file_PluginLister_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -84,20 +82,17 @@ func (x *PluginIdentifier) GetName() string { } type ListPluginsResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugins []*PluginIdentifier `protobuf:"bytes,1,rep,name=plugins,proto3" json:"plugins,omitempty"` unknownFields protoimpl.UnknownFields - - Plugins []*PluginIdentifier `protobuf:"bytes,1,rep,name=plugins,proto3" json:"plugins,omitempty"` + sizeCache protoimpl.SizeCache } func (x *ListPluginsResponse) Reset() { *x = ListPluginsResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_PluginLister_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_PluginLister_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ListPluginsResponse) String() string { @@ -108,7 +103,7 @@ func (*ListPluginsResponse) ProtoMessage() {} func (x *ListPluginsResponse) ProtoReflect() protoreflect.Message { mi := &file_PluginLister_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -132,46 +127,32 @@ func (x *ListPluginsResponse) GetPlugins() []*PluginIdentifier { var File_PluginLister_proto protoreflect.FileDescriptor -var file_PluginLister_proto_rawDesc = []byte{ - 0x0a, 0x12, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x1a, - 0x0c, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x54, 0x0a, - 0x10, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, - 0x72, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6b, - 0x69, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x12, - 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x22, 0x4c, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x6c, 0x75, 0x67, 0x69, - 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, 0x07, 0x70, 0x6c, - 0x75, 0x67, 0x69, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x65, - 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x49, 0x64, - 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x52, 0x07, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, - 0x73, 0x32, 0x4f, 0x0a, 0x0c, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x65, - 0x72, 0x12, 0x3f, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x73, - 0x12, 0x10, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x1a, 0x1e, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, 0x6e, 0x7a, 0x75, 0x2f, 0x76, 0x65, - 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, - 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, -} +const file_PluginLister_proto_rawDesc = "" + + "\n" + + "\x12PluginLister.proto\x12\tgenerated\x1a\fShared.proto\"T\n" + + "\x10PluginIdentifier\x12\x18\n" + + "\acommand\x18\x01 \x01(\tR\acommand\x12\x12\n" + + "\x04kind\x18\x02 \x01(\tR\x04kind\x12\x12\n" + + "\x04name\x18\x03 \x01(\tR\x04name\"L\n" + + "\x13ListPluginsResponse\x125\n" + + "\aplugins\x18\x01 \x03(\v2\x1b.generated.PluginIdentifierR\aplugins2O\n" + + "\fPluginLister\x12?\n" + + "\vListPlugins\x12\x10.generated.Empty\x1a\x1e.generated.ListPluginsResponseB5Z3github.com/vmware-tanzu/velero/pkg/plugin/generatedb\x06proto3" var ( file_PluginLister_proto_rawDescOnce sync.Once - file_PluginLister_proto_rawDescData = file_PluginLister_proto_rawDesc + file_PluginLister_proto_rawDescData []byte ) func file_PluginLister_proto_rawDescGZIP() []byte { file_PluginLister_proto_rawDescOnce.Do(func() { - file_PluginLister_proto_rawDescData = protoimpl.X.CompressGZIP(file_PluginLister_proto_rawDescData) + file_PluginLister_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_PluginLister_proto_rawDesc), len(file_PluginLister_proto_rawDesc))) }) return file_PluginLister_proto_rawDescData } var file_PluginLister_proto_msgTypes = make([]protoimpl.MessageInfo, 2) -var file_PluginLister_proto_goTypes = []interface{}{ +var file_PluginLister_proto_goTypes = []any{ (*PluginIdentifier)(nil), // 0: generated.PluginIdentifier (*ListPluginsResponse)(nil), // 1: generated.ListPluginsResponse (*Empty)(nil), // 2: generated.Empty @@ -193,37 +174,11 @@ func file_PluginLister_proto_init() { return } file_Shared_proto_init() - if !protoimpl.UnsafeEnabled { - file_PluginLister_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PluginIdentifier); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_PluginLister_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListPluginsResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_PluginLister_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_PluginLister_proto_rawDesc), len(file_PluginLister_proto_rawDesc)), NumEnums: 0, NumMessages: 2, NumExtensions: 0, @@ -234,7 +189,6 @@ func file_PluginLister_proto_init() { MessageInfos: file_PluginLister_proto_msgTypes, }.Build() File_PluginLister_proto = out.File - file_PluginLister_proto_rawDesc = nil file_PluginLister_proto_goTypes = nil file_PluginLister_proto_depIdxs = nil } diff --git a/pkg/plugin/generated/RestoreItemAction.pb.go b/pkg/plugin/generated/RestoreItemAction.pb.go index 9489af476..f0d6dd3b7 100644 --- a/pkg/plugin/generated/RestoreItemAction.pb.go +++ b/pkg/plugin/generated/RestoreItemAction.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 +// protoc-gen-go v1.36.11-devel // protoc v4.25.2 // source: RestoreItemAction.proto @@ -11,6 +11,7 @@ import ( protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" + unsafe "unsafe" ) const ( @@ -21,23 +22,20 @@ const ( ) type RestoreItemActionExecuteRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - Item []byte `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` - Restore []byte `protobuf:"bytes,3,opt,name=restore,proto3" json:"restore,omitempty"` - ItemFromBackup []byte `protobuf:"bytes,4,opt,name=itemFromBackup,proto3" json:"itemFromBackup,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + Item []byte `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` + Restore []byte `protobuf:"bytes,3,opt,name=restore,proto3" json:"restore,omitempty"` + ItemFromBackup []byte `protobuf:"bytes,4,opt,name=itemFromBackup,proto3" json:"itemFromBackup,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RestoreItemActionExecuteRequest) Reset() { *x = RestoreItemActionExecuteRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_RestoreItemAction_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_RestoreItemAction_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *RestoreItemActionExecuteRequest) String() string { @@ -48,7 +46,7 @@ func (*RestoreItemActionExecuteRequest) ProtoMessage() {} func (x *RestoreItemActionExecuteRequest) ProtoReflect() protoreflect.Message { mi := &file_RestoreItemAction_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -92,22 +90,19 @@ func (x *RestoreItemActionExecuteRequest) GetItemFromBackup() []byte { } type RestoreItemActionExecuteResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Item []byte `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"` - AdditionalItems []*ResourceIdentifier `protobuf:"bytes,2,rep,name=additionalItems,proto3" json:"additionalItems,omitempty"` - SkipRestore bool `protobuf:"varint,3,opt,name=skipRestore,proto3" json:"skipRestore,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Item []byte `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"` + AdditionalItems []*ResourceIdentifier `protobuf:"bytes,2,rep,name=additionalItems,proto3" json:"additionalItems,omitempty"` + SkipRestore bool `protobuf:"varint,3,opt,name=skipRestore,proto3" json:"skipRestore,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RestoreItemActionExecuteResponse) Reset() { *x = RestoreItemActionExecuteResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_RestoreItemAction_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_RestoreItemAction_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *RestoreItemActionExecuteResponse) String() string { @@ -118,7 +113,7 @@ func (*RestoreItemActionExecuteResponse) ProtoMessage() {} func (x *RestoreItemActionExecuteResponse) ProtoReflect() protoreflect.Message { mi := &file_RestoreItemAction_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -155,20 +150,17 @@ func (x *RestoreItemActionExecuteResponse) GetSkipRestore() bool { } type RestoreItemActionAppliesToRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + sizeCache protoimpl.SizeCache } func (x *RestoreItemActionAppliesToRequest) Reset() { *x = RestoreItemActionAppliesToRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_RestoreItemAction_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_RestoreItemAction_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *RestoreItemActionAppliesToRequest) String() string { @@ -179,7 +171,7 @@ func (*RestoreItemActionAppliesToRequest) ProtoMessage() {} func (x *RestoreItemActionAppliesToRequest) ProtoReflect() protoreflect.Message { mi := &file_RestoreItemAction_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -202,20 +194,17 @@ func (x *RestoreItemActionAppliesToRequest) GetPlugin() string { } type RestoreItemActionAppliesToResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - ResourceSelector *ResourceSelector `protobuf:"bytes,1,opt,name=ResourceSelector,proto3" json:"ResourceSelector,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + ResourceSelector *ResourceSelector `protobuf:"bytes,1,opt,name=ResourceSelector,proto3" json:"ResourceSelector,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RestoreItemActionAppliesToResponse) Reset() { *x = RestoreItemActionAppliesToResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_RestoreItemAction_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_RestoreItemAction_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *RestoreItemActionAppliesToResponse) String() string { @@ -226,7 +215,7 @@ func (*RestoreItemActionAppliesToResponse) ProtoMessage() {} func (x *RestoreItemActionAppliesToResponse) ProtoReflect() protoreflect.Message { mi := &file_RestoreItemAction_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -250,75 +239,40 @@ func (x *RestoreItemActionAppliesToResponse) GetResourceSelector() *ResourceSele var File_RestoreItemAction_proto protoreflect.FileDescriptor -var file_RestoreItemAction_proto_rawDesc = []byte{ - 0x0a, 0x17, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x67, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x65, 0x64, 0x1a, 0x0c, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x22, 0x8f, 0x01, 0x0a, 0x1f, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, - 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x12, - 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, - 0x65, 0x6d, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x07, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x26, 0x0a, 0x0e, - 0x69, 0x74, 0x65, 0x6d, 0x46, 0x72, 0x6f, 0x6d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0e, 0x69, 0x74, 0x65, 0x6d, 0x46, 0x72, 0x6f, 0x6d, 0x42, 0x61, - 0x63, 0x6b, 0x75, 0x70, 0x22, 0xa1, 0x01, 0x0a, 0x20, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, - 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, - 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x47, 0x0a, - 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, - 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, - 0x69, 0x66, 0x69, 0x65, 0x72, 0x52, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, - 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x73, 0x6b, 0x69, 0x70, 0x52, 0x65, - 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x73, 0x6b, 0x69, - 0x70, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x22, 0x3b, 0x0a, 0x21, 0x52, 0x65, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, - 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, - 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, - 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x22, 0x6d, 0x0a, 0x22, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, - 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, - 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x10, 0x52, - 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, - 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, - 0x6f, 0x72, 0x52, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x32, 0xe1, 0x01, 0x0a, 0x11, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, - 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x68, 0x0a, 0x09, 0x41, 0x70, - 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x12, 0x2c, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, - 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, - 0x64, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x62, 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x12, - 0x2a, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x65, - 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x67, 0x65, - 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, - 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, - 0x6e, 0x7a, 0x75, 0x2f, 0x76, 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, - 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} +const file_RestoreItemAction_proto_rawDesc = "" + + "\n" + + "\x17RestoreItemAction.proto\x12\tgenerated\x1a\fShared.proto\"\x8f\x01\n" + + "\x1fRestoreItemActionExecuteRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12\x12\n" + + "\x04item\x18\x02 \x01(\fR\x04item\x12\x18\n" + + "\arestore\x18\x03 \x01(\fR\arestore\x12&\n" + + "\x0eitemFromBackup\x18\x04 \x01(\fR\x0eitemFromBackup\"\xa1\x01\n" + + " RestoreItemActionExecuteResponse\x12\x12\n" + + "\x04item\x18\x01 \x01(\fR\x04item\x12G\n" + + "\x0fadditionalItems\x18\x02 \x03(\v2\x1d.generated.ResourceIdentifierR\x0fadditionalItems\x12 \n" + + "\vskipRestore\x18\x03 \x01(\bR\vskipRestore\";\n" + + "!RestoreItemActionAppliesToRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\"m\n" + + "\"RestoreItemActionAppliesToResponse\x12G\n" + + "\x10ResourceSelector\x18\x01 \x01(\v2\x1b.generated.ResourceSelectorR\x10ResourceSelector2\xe1\x01\n" + + "\x11RestoreItemAction\x12h\n" + + "\tAppliesTo\x12,.generated.RestoreItemActionAppliesToRequest\x1a-.generated.RestoreItemActionAppliesToResponse\x12b\n" + + "\aExecute\x12*.generated.RestoreItemActionExecuteRequest\x1a+.generated.RestoreItemActionExecuteResponseB5Z3github.com/vmware-tanzu/velero/pkg/plugin/generatedb\x06proto3" var ( file_RestoreItemAction_proto_rawDescOnce sync.Once - file_RestoreItemAction_proto_rawDescData = file_RestoreItemAction_proto_rawDesc + file_RestoreItemAction_proto_rawDescData []byte ) func file_RestoreItemAction_proto_rawDescGZIP() []byte { file_RestoreItemAction_proto_rawDescOnce.Do(func() { - file_RestoreItemAction_proto_rawDescData = protoimpl.X.CompressGZIP(file_RestoreItemAction_proto_rawDescData) + file_RestoreItemAction_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_RestoreItemAction_proto_rawDesc), len(file_RestoreItemAction_proto_rawDesc))) }) return file_RestoreItemAction_proto_rawDescData } var file_RestoreItemAction_proto_msgTypes = make([]protoimpl.MessageInfo, 4) -var file_RestoreItemAction_proto_goTypes = []interface{}{ +var file_RestoreItemAction_proto_goTypes = []any{ (*RestoreItemActionExecuteRequest)(nil), // 0: generated.RestoreItemActionExecuteRequest (*RestoreItemActionExecuteResponse)(nil), // 1: generated.RestoreItemActionExecuteResponse (*RestoreItemActionAppliesToRequest)(nil), // 2: generated.RestoreItemActionAppliesToRequest @@ -346,61 +300,11 @@ func file_RestoreItemAction_proto_init() { return } file_Shared_proto_init() - if !protoimpl.UnsafeEnabled { - file_RestoreItemAction_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RestoreItemActionExecuteRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_RestoreItemAction_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RestoreItemActionExecuteResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_RestoreItemAction_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RestoreItemActionAppliesToRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_RestoreItemAction_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RestoreItemActionAppliesToResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_RestoreItemAction_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_RestoreItemAction_proto_rawDesc), len(file_RestoreItemAction_proto_rawDesc)), NumEnums: 0, NumMessages: 4, NumExtensions: 0, @@ -411,7 +315,6 @@ func file_RestoreItemAction_proto_init() { MessageInfos: file_RestoreItemAction_proto_msgTypes, }.Build() File_RestoreItemAction_proto = out.File - file_RestoreItemAction_proto_rawDesc = nil file_RestoreItemAction_proto_goTypes = nil file_RestoreItemAction_proto_depIdxs = nil } diff --git a/pkg/plugin/generated/Shared.pb.go b/pkg/plugin/generated/Shared.pb.go index 07af30089..7c458579b 100644 --- a/pkg/plugin/generated/Shared.pb.go +++ b/pkg/plugin/generated/Shared.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 +// protoc-gen-go v1.36.11-devel // protoc v4.25.2 // source: Shared.proto @@ -12,6 +12,7 @@ import ( timestamppb "google.golang.org/protobuf/types/known/timestamppb" reflect "reflect" sync "sync" + unsafe "unsafe" ) const ( @@ -22,18 +23,16 @@ const ( ) type Empty struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Empty) Reset() { *x = Empty{} - if protoimpl.UnsafeEnabled { - mi := &file_Shared_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_Shared_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *Empty) String() string { @@ -44,7 +43,7 @@ func (*Empty) ProtoMessage() {} func (x *Empty) ProtoReflect() protoreflect.Message { mi := &file_Shared_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -60,20 +59,17 @@ func (*Empty) Descriptor() ([]byte, []int) { } type Stack struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Frames []*StackFrame `protobuf:"bytes,1,rep,name=frames,proto3" json:"frames,omitempty"` unknownFields protoimpl.UnknownFields - - Frames []*StackFrame `protobuf:"bytes,1,rep,name=frames,proto3" json:"frames,omitempty"` + sizeCache protoimpl.SizeCache } func (x *Stack) Reset() { *x = Stack{} - if protoimpl.UnsafeEnabled { - mi := &file_Shared_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_Shared_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *Stack) String() string { @@ -84,7 +80,7 @@ func (*Stack) ProtoMessage() {} func (x *Stack) ProtoReflect() protoreflect.Message { mi := &file_Shared_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -107,22 +103,19 @@ func (x *Stack) GetFrames() []*StackFrame { } type StackFrame struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + File string `protobuf:"bytes,1,opt,name=file,proto3" json:"file,omitempty"` + Line int32 `protobuf:"varint,2,opt,name=line,proto3" json:"line,omitempty"` + Function string `protobuf:"bytes,3,opt,name=function,proto3" json:"function,omitempty"` unknownFields protoimpl.UnknownFields - - File string `protobuf:"bytes,1,opt,name=file,proto3" json:"file,omitempty"` - Line int32 `protobuf:"varint,2,opt,name=line,proto3" json:"line,omitempty"` - Function string `protobuf:"bytes,3,opt,name=function,proto3" json:"function,omitempty"` + sizeCache protoimpl.SizeCache } func (x *StackFrame) Reset() { *x = StackFrame{} - if protoimpl.UnsafeEnabled { - mi := &file_Shared_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_Shared_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *StackFrame) String() string { @@ -133,7 +126,7 @@ func (*StackFrame) ProtoMessage() {} func (x *StackFrame) ProtoReflect() protoreflect.Message { mi := &file_Shared_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -170,23 +163,20 @@ func (x *StackFrame) GetFunction() string { } type ResourceIdentifier struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Group string `protobuf:"bytes,1,opt,name=group,proto3" json:"group,omitempty"` + Resource string `protobuf:"bytes,2,opt,name=resource,proto3" json:"resource,omitempty"` + Namespace string `protobuf:"bytes,3,opt,name=namespace,proto3" json:"namespace,omitempty"` + Name string `protobuf:"bytes,4,opt,name=name,proto3" json:"name,omitempty"` unknownFields protoimpl.UnknownFields - - Group string `protobuf:"bytes,1,opt,name=group,proto3" json:"group,omitempty"` - Resource string `protobuf:"bytes,2,opt,name=resource,proto3" json:"resource,omitempty"` - Namespace string `protobuf:"bytes,3,opt,name=namespace,proto3" json:"namespace,omitempty"` - Name string `protobuf:"bytes,4,opt,name=name,proto3" json:"name,omitempty"` + sizeCache protoimpl.SizeCache } func (x *ResourceIdentifier) Reset() { *x = ResourceIdentifier{} - if protoimpl.UnsafeEnabled { - mi := &file_Shared_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_Shared_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ResourceIdentifier) String() string { @@ -197,7 +187,7 @@ func (*ResourceIdentifier) ProtoMessage() {} func (x *ResourceIdentifier) ProtoReflect() protoreflect.Message { mi := &file_Shared_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -241,24 +231,21 @@ func (x *ResourceIdentifier) GetName() string { } type ResourceSelector struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - IncludedNamespaces []string `protobuf:"bytes,1,rep,name=includedNamespaces,proto3" json:"includedNamespaces,omitempty"` - ExcludedNamespaces []string `protobuf:"bytes,2,rep,name=excludedNamespaces,proto3" json:"excludedNamespaces,omitempty"` - IncludedResources []string `protobuf:"bytes,3,rep,name=includedResources,proto3" json:"includedResources,omitempty"` - ExcludedResources []string `protobuf:"bytes,4,rep,name=excludedResources,proto3" json:"excludedResources,omitempty"` - Selector string `protobuf:"bytes,5,opt,name=selector,proto3" json:"selector,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + IncludedNamespaces []string `protobuf:"bytes,1,rep,name=includedNamespaces,proto3" json:"includedNamespaces,omitempty"` + ExcludedNamespaces []string `protobuf:"bytes,2,rep,name=excludedNamespaces,proto3" json:"excludedNamespaces,omitempty"` + IncludedResources []string `protobuf:"bytes,3,rep,name=includedResources,proto3" json:"includedResources,omitempty"` + ExcludedResources []string `protobuf:"bytes,4,rep,name=excludedResources,proto3" json:"excludedResources,omitempty"` + Selector string `protobuf:"bytes,5,opt,name=selector,proto3" json:"selector,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ResourceSelector) Reset() { *x = ResourceSelector{} - if protoimpl.UnsafeEnabled { - mi := &file_Shared_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_Shared_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ResourceSelector) String() string { @@ -269,7 +256,7 @@ func (*ResourceSelector) ProtoMessage() {} func (x *ResourceSelector) ProtoReflect() protoreflect.Message { mi := &file_Shared_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -320,10 +307,7 @@ func (x *ResourceSelector) GetSelector() string { } type OperationProgress struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` Completed bool `protobuf:"varint,1,opt,name=completed,proto3" json:"completed,omitempty"` Err string `protobuf:"bytes,2,opt,name=err,proto3" json:"err,omitempty"` NCompleted int64 `protobuf:"varint,3,opt,name=nCompleted,proto3" json:"nCompleted,omitempty"` @@ -332,15 +316,15 @@ type OperationProgress struct { Description string `protobuf:"bytes,6,opt,name=description,proto3" json:"description,omitempty"` Started *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=started,proto3" json:"started,omitempty"` Updated *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=updated,proto3" json:"updated,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *OperationProgress) Reset() { *x = OperationProgress{} - if protoimpl.UnsafeEnabled { - mi := &file_Shared_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_Shared_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *OperationProgress) String() string { @@ -351,7 +335,7 @@ func (*OperationProgress) ProtoMessage() {} func (x *OperationProgress) ProtoReflect() protoreflect.Message { mi := &file_Shared_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -424,82 +408,54 @@ func (x *OperationProgress) GetUpdated() *timestamppb.Timestamp { var File_Shared_proto protoreflect.FileDescriptor -var file_Shared_proto_rawDesc = []byte{ - 0x0a, 0x0c, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, - 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x07, 0x0a, 0x05, 0x45, 0x6d, - 0x70, 0x74, 0x79, 0x22, 0x36, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x63, 0x6b, 0x12, 0x2d, 0x0a, 0x06, - 0x66, 0x72, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67, - 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x53, 0x74, 0x61, 0x63, 0x6b, 0x46, 0x72, - 0x61, 0x6d, 0x65, 0x52, 0x06, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x73, 0x22, 0x50, 0x0a, 0x0a, 0x53, - 0x74, 0x61, 0x63, 0x6b, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x12, 0x0a, - 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x6c, 0x69, 0x6e, - 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x78, 0x0a, - 0x12, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, - 0x69, 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, - 0x61, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0xea, 0x01, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x2e, 0x0a, 0x12, - 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, - 0x65, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x2e, 0x0a, 0x12, - 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x12, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, - 0x65, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x2c, 0x0a, 0x11, - 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x64, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, - 0x64, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x65, 0x78, - 0x63, 0x6c, 0x75, 0x64, 0x65, 0x64, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x64, 0x52, - 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x65, 0x6c, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x65, 0x6c, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x22, 0xb1, 0x02, 0x0a, 0x11, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, - 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, - 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x72, 0x72, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x65, 0x72, 0x72, 0x12, 0x1e, 0x0a, 0x0a, 0x6e, 0x43, - 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, - 0x6e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x6e, 0x54, - 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6e, 0x54, 0x6f, 0x74, - 0x61, 0x6c, 0x12, 0x26, 0x0a, 0x0e, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, - 0x6e, 0x69, 0x74, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x6e, 0x69, 0x74, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, - 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x34, 0x0a, 0x07, - 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x73, 0x74, 0x61, 0x72, 0x74, - 0x65, 0x64, 0x12, 0x34, 0x0a, 0x07, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x18, 0x08, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, - 0x07, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, - 0x6e, 0x7a, 0x75, 0x2f, 0x76, 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, - 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} +const file_Shared_proto_rawDesc = "" + + "\n" + + "\fShared.proto\x12\tgenerated\x1a\x1fgoogle/protobuf/timestamp.proto\"\a\n" + + "\x05Empty\"6\n" + + "\x05Stack\x12-\n" + + "\x06frames\x18\x01 \x03(\v2\x15.generated.StackFrameR\x06frames\"P\n" + + "\n" + + "StackFrame\x12\x12\n" + + "\x04file\x18\x01 \x01(\tR\x04file\x12\x12\n" + + "\x04line\x18\x02 \x01(\x05R\x04line\x12\x1a\n" + + "\bfunction\x18\x03 \x01(\tR\bfunction\"x\n" + + "\x12ResourceIdentifier\x12\x14\n" + + "\x05group\x18\x01 \x01(\tR\x05group\x12\x1a\n" + + "\bresource\x18\x02 \x01(\tR\bresource\x12\x1c\n" + + "\tnamespace\x18\x03 \x01(\tR\tnamespace\x12\x12\n" + + "\x04name\x18\x04 \x01(\tR\x04name\"\xea\x01\n" + + "\x10ResourceSelector\x12.\n" + + "\x12includedNamespaces\x18\x01 \x03(\tR\x12includedNamespaces\x12.\n" + + "\x12excludedNamespaces\x18\x02 \x03(\tR\x12excludedNamespaces\x12,\n" + + "\x11includedResources\x18\x03 \x03(\tR\x11includedResources\x12,\n" + + "\x11excludedResources\x18\x04 \x03(\tR\x11excludedResources\x12\x1a\n" + + "\bselector\x18\x05 \x01(\tR\bselector\"\xb1\x02\n" + + "\x11OperationProgress\x12\x1c\n" + + "\tcompleted\x18\x01 \x01(\bR\tcompleted\x12\x10\n" + + "\x03err\x18\x02 \x01(\tR\x03err\x12\x1e\n" + + "\n" + + "nCompleted\x18\x03 \x01(\x03R\n" + + "nCompleted\x12\x16\n" + + "\x06nTotal\x18\x04 \x01(\x03R\x06nTotal\x12&\n" + + "\x0eoperationUnits\x18\x05 \x01(\tR\x0eoperationUnits\x12 \n" + + "\vdescription\x18\x06 \x01(\tR\vdescription\x124\n" + + "\astarted\x18\a \x01(\v2\x1a.google.protobuf.TimestampR\astarted\x124\n" + + "\aupdated\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\aupdatedB5Z3github.com/vmware-tanzu/velero/pkg/plugin/generatedb\x06proto3" var ( file_Shared_proto_rawDescOnce sync.Once - file_Shared_proto_rawDescData = file_Shared_proto_rawDesc + file_Shared_proto_rawDescData []byte ) func file_Shared_proto_rawDescGZIP() []byte { file_Shared_proto_rawDescOnce.Do(func() { - file_Shared_proto_rawDescData = protoimpl.X.CompressGZIP(file_Shared_proto_rawDescData) + file_Shared_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_Shared_proto_rawDesc), len(file_Shared_proto_rawDesc))) }) return file_Shared_proto_rawDescData } var file_Shared_proto_msgTypes = make([]protoimpl.MessageInfo, 6) -var file_Shared_proto_goTypes = []interface{}{ +var file_Shared_proto_goTypes = []any{ (*Empty)(nil), // 0: generated.Empty (*Stack)(nil), // 1: generated.Stack (*StackFrame)(nil), // 2: generated.StackFrame @@ -524,85 +480,11 @@ func file_Shared_proto_init() { if File_Shared_proto != nil { return } - if !protoimpl.UnsafeEnabled { - file_Shared_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Empty); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_Shared_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Stack); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_Shared_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StackFrame); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_Shared_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ResourceIdentifier); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_Shared_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ResourceSelector); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_Shared_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*OperationProgress); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_Shared_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_Shared_proto_rawDesc), len(file_Shared_proto_rawDesc)), NumEnums: 0, NumMessages: 6, NumExtensions: 0, @@ -613,7 +495,6 @@ func file_Shared_proto_init() { MessageInfos: file_Shared_proto_msgTypes, }.Build() File_Shared_proto = out.File - file_Shared_proto_rawDesc = nil file_Shared_proto_goTypes = nil file_Shared_proto_depIdxs = nil } diff --git a/pkg/plugin/generated/VolumeSnapshotter.pb.go b/pkg/plugin/generated/VolumeSnapshotter.pb.go index 673ad9739..2b5f9a86e 100644 --- a/pkg/plugin/generated/VolumeSnapshotter.pb.go +++ b/pkg/plugin/generated/VolumeSnapshotter.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 +// protoc-gen-go v1.36.11-devel // protoc v4.25.2 // source: VolumeSnapshotter.proto @@ -11,6 +11,7 @@ import ( protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" + unsafe "unsafe" ) const ( @@ -21,24 +22,21 @@ const ( ) type CreateVolumeRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + SnapshotID string `protobuf:"bytes,2,opt,name=snapshotID,proto3" json:"snapshotID,omitempty"` + VolumeType string `protobuf:"bytes,3,opt,name=volumeType,proto3" json:"volumeType,omitempty"` + VolumeAZ string `protobuf:"bytes,4,opt,name=volumeAZ,proto3" json:"volumeAZ,omitempty"` + Iops int64 `protobuf:"varint,5,opt,name=iops,proto3" json:"iops,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - SnapshotID string `protobuf:"bytes,2,opt,name=snapshotID,proto3" json:"snapshotID,omitempty"` - VolumeType string `protobuf:"bytes,3,opt,name=volumeType,proto3" json:"volumeType,omitempty"` - VolumeAZ string `protobuf:"bytes,4,opt,name=volumeAZ,proto3" json:"volumeAZ,omitempty"` - Iops int64 `protobuf:"varint,5,opt,name=iops,proto3" json:"iops,omitempty"` + sizeCache protoimpl.SizeCache } func (x *CreateVolumeRequest) Reset() { *x = CreateVolumeRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_VolumeSnapshotter_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_VolumeSnapshotter_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *CreateVolumeRequest) String() string { @@ -49,7 +47,7 @@ func (*CreateVolumeRequest) ProtoMessage() {} func (x *CreateVolumeRequest) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -100,20 +98,17 @@ func (x *CreateVolumeRequest) GetIops() int64 { } type CreateVolumeResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + VolumeID string `protobuf:"bytes,1,opt,name=volumeID,proto3" json:"volumeID,omitempty"` unknownFields protoimpl.UnknownFields - - VolumeID string `protobuf:"bytes,1,opt,name=volumeID,proto3" json:"volumeID,omitempty"` + sizeCache protoimpl.SizeCache } func (x *CreateVolumeResponse) Reset() { *x = CreateVolumeResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_VolumeSnapshotter_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_VolumeSnapshotter_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *CreateVolumeResponse) String() string { @@ -124,7 +119,7 @@ func (*CreateVolumeResponse) ProtoMessage() {} func (x *CreateVolumeResponse) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -147,22 +142,19 @@ func (x *CreateVolumeResponse) GetVolumeID() string { } type GetVolumeInfoRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + VolumeID string `protobuf:"bytes,2,opt,name=volumeID,proto3" json:"volumeID,omitempty"` + VolumeAZ string `protobuf:"bytes,3,opt,name=volumeAZ,proto3" json:"volumeAZ,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - VolumeID string `protobuf:"bytes,2,opt,name=volumeID,proto3" json:"volumeID,omitempty"` - VolumeAZ string `protobuf:"bytes,3,opt,name=volumeAZ,proto3" json:"volumeAZ,omitempty"` + sizeCache protoimpl.SizeCache } func (x *GetVolumeInfoRequest) Reset() { *x = GetVolumeInfoRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_VolumeSnapshotter_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_VolumeSnapshotter_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *GetVolumeInfoRequest) String() string { @@ -173,7 +165,7 @@ func (*GetVolumeInfoRequest) ProtoMessage() {} func (x *GetVolumeInfoRequest) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -210,21 +202,18 @@ func (x *GetVolumeInfoRequest) GetVolumeAZ() string { } type GetVolumeInfoResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + VolumeType string `protobuf:"bytes,1,opt,name=volumeType,proto3" json:"volumeType,omitempty"` + Iops int64 `protobuf:"varint,2,opt,name=iops,proto3" json:"iops,omitempty"` unknownFields protoimpl.UnknownFields - - VolumeType string `protobuf:"bytes,1,opt,name=volumeType,proto3" json:"volumeType,omitempty"` - Iops int64 `protobuf:"varint,2,opt,name=iops,proto3" json:"iops,omitempty"` + sizeCache protoimpl.SizeCache } func (x *GetVolumeInfoResponse) Reset() { *x = GetVolumeInfoResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_VolumeSnapshotter_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_VolumeSnapshotter_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *GetVolumeInfoResponse) String() string { @@ -235,7 +224,7 @@ func (*GetVolumeInfoResponse) ProtoMessage() {} func (x *GetVolumeInfoResponse) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -265,23 +254,20 @@ func (x *GetVolumeInfoResponse) GetIops() int64 { } type CreateSnapshotRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + VolumeID string `protobuf:"bytes,2,opt,name=volumeID,proto3" json:"volumeID,omitempty"` + VolumeAZ string `protobuf:"bytes,3,opt,name=volumeAZ,proto3" json:"volumeAZ,omitempty"` + Tags map[string]string `protobuf:"bytes,4,rep,name=tags,proto3" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - VolumeID string `protobuf:"bytes,2,opt,name=volumeID,proto3" json:"volumeID,omitempty"` - VolumeAZ string `protobuf:"bytes,3,opt,name=volumeAZ,proto3" json:"volumeAZ,omitempty"` - Tags map[string]string `protobuf:"bytes,4,rep,name=tags,proto3" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + sizeCache protoimpl.SizeCache } func (x *CreateSnapshotRequest) Reset() { *x = CreateSnapshotRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_VolumeSnapshotter_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_VolumeSnapshotter_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *CreateSnapshotRequest) String() string { @@ -292,7 +278,7 @@ func (*CreateSnapshotRequest) ProtoMessage() {} func (x *CreateSnapshotRequest) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -336,20 +322,17 @@ func (x *CreateSnapshotRequest) GetTags() map[string]string { } type CreateSnapshotResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + SnapshotID string `protobuf:"bytes,1,opt,name=snapshotID,proto3" json:"snapshotID,omitempty"` unknownFields protoimpl.UnknownFields - - SnapshotID string `protobuf:"bytes,1,opt,name=snapshotID,proto3" json:"snapshotID,omitempty"` + sizeCache protoimpl.SizeCache } func (x *CreateSnapshotResponse) Reset() { *x = CreateSnapshotResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_VolumeSnapshotter_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_VolumeSnapshotter_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *CreateSnapshotResponse) String() string { @@ -360,7 +343,7 @@ func (*CreateSnapshotResponse) ProtoMessage() {} func (x *CreateSnapshotResponse) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -383,21 +366,18 @@ func (x *CreateSnapshotResponse) GetSnapshotID() string { } type DeleteSnapshotRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + SnapshotID string `protobuf:"bytes,2,opt,name=snapshotID,proto3" json:"snapshotID,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - SnapshotID string `protobuf:"bytes,2,opt,name=snapshotID,proto3" json:"snapshotID,omitempty"` + sizeCache protoimpl.SizeCache } func (x *DeleteSnapshotRequest) Reset() { *x = DeleteSnapshotRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_VolumeSnapshotter_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_VolumeSnapshotter_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *DeleteSnapshotRequest) String() string { @@ -408,7 +388,7 @@ func (*DeleteSnapshotRequest) ProtoMessage() {} func (x *DeleteSnapshotRequest) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -438,21 +418,18 @@ func (x *DeleteSnapshotRequest) GetSnapshotID() string { } type GetVolumeIDRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - PersistentVolume []byte `protobuf:"bytes,2,opt,name=persistentVolume,proto3" json:"persistentVolume,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + PersistentVolume []byte `protobuf:"bytes,2,opt,name=persistentVolume,proto3" json:"persistentVolume,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GetVolumeIDRequest) Reset() { *x = GetVolumeIDRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_VolumeSnapshotter_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_VolumeSnapshotter_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *GetVolumeIDRequest) String() string { @@ -463,7 +440,7 @@ func (*GetVolumeIDRequest) ProtoMessage() {} func (x *GetVolumeIDRequest) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[7] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -493,20 +470,17 @@ func (x *GetVolumeIDRequest) GetPersistentVolume() []byte { } type GetVolumeIDResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + VolumeID string `protobuf:"bytes,1,opt,name=volumeID,proto3" json:"volumeID,omitempty"` unknownFields protoimpl.UnknownFields - - VolumeID string `protobuf:"bytes,1,opt,name=volumeID,proto3" json:"volumeID,omitempty"` + sizeCache protoimpl.SizeCache } func (x *GetVolumeIDResponse) Reset() { *x = GetVolumeIDResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_VolumeSnapshotter_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_VolumeSnapshotter_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *GetVolumeIDResponse) String() string { @@ -517,7 +491,7 @@ func (*GetVolumeIDResponse) ProtoMessage() {} func (x *GetVolumeIDResponse) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[8] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -540,22 +514,19 @@ func (x *GetVolumeIDResponse) GetVolumeID() string { } type SetVolumeIDRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - PersistentVolume []byte `protobuf:"bytes,2,opt,name=persistentVolume,proto3" json:"persistentVolume,omitempty"` - VolumeID string `protobuf:"bytes,3,opt,name=volumeID,proto3" json:"volumeID,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + PersistentVolume []byte `protobuf:"bytes,2,opt,name=persistentVolume,proto3" json:"persistentVolume,omitempty"` + VolumeID string `protobuf:"bytes,3,opt,name=volumeID,proto3" json:"volumeID,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SetVolumeIDRequest) Reset() { *x = SetVolumeIDRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_VolumeSnapshotter_proto_msgTypes[9] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_VolumeSnapshotter_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *SetVolumeIDRequest) String() string { @@ -566,7 +537,7 @@ func (*SetVolumeIDRequest) ProtoMessage() {} func (x *SetVolumeIDRequest) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[9] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -603,20 +574,17 @@ func (x *SetVolumeIDRequest) GetVolumeID() string { } type SetVolumeIDResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - PersistentVolume []byte `protobuf:"bytes,1,opt,name=persistentVolume,proto3" json:"persistentVolume,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + PersistentVolume []byte `protobuf:"bytes,1,opt,name=persistentVolume,proto3" json:"persistentVolume,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SetVolumeIDResponse) Reset() { *x = SetVolumeIDResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_VolumeSnapshotter_proto_msgTypes[10] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_VolumeSnapshotter_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *SetVolumeIDResponse) String() string { @@ -627,7 +595,7 @@ func (*SetVolumeIDResponse) ProtoMessage() {} func (x *SetVolumeIDResponse) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[10] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -650,21 +618,18 @@ func (x *SetVolumeIDResponse) GetPersistentVolume() []byte { } type VolumeSnapshotterInitRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + Config map[string]string `protobuf:"bytes,2,rep,name=config,proto3" json:"config,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - Config map[string]string `protobuf:"bytes,2,rep,name=config,proto3" json:"config,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + sizeCache protoimpl.SizeCache } func (x *VolumeSnapshotterInitRequest) Reset() { *x = VolumeSnapshotterInitRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_VolumeSnapshotter_proto_msgTypes[11] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_VolumeSnapshotter_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *VolumeSnapshotterInitRequest) String() string { @@ -675,7 +640,7 @@ func (*VolumeSnapshotterInitRequest) ProtoMessage() {} func (x *VolumeSnapshotterInitRequest) ProtoReflect() protoreflect.Message { mi := &file_VolumeSnapshotter_proto_msgTypes[11] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -706,147 +671,87 @@ func (x *VolumeSnapshotterInitRequest) GetConfig() map[string]string { var File_VolumeSnapshotter_proto protoreflect.FileDescriptor -var file_VolumeSnapshotter_proto_rawDesc = []byte{ - 0x0a, 0x17, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, - 0x74, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x67, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x65, 0x64, 0x1a, 0x0c, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x22, 0x9d, 0x01, 0x0a, 0x13, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x56, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, - 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, - 0x69, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x44, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, - 0x49, 0x44, 0x12, 0x1e, 0x0a, 0x0a, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x79, 0x70, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x41, 0x5a, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x41, 0x5a, 0x12, 0x12, - 0x0a, 0x04, 0x69, 0x6f, 0x70, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x69, 0x6f, - 0x70, 0x73, 0x22, 0x32, 0x0a, 0x14, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x76, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x22, 0x66, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, - 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, - 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x49, 0x44, 0x12, 0x1a, 0x0a, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x41, 0x5a, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x41, 0x5a, 0x22, 0x4b, - 0x0a, 0x15, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x76, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x76, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x6f, 0x70, 0x73, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x69, 0x6f, 0x70, 0x73, 0x22, 0xe0, 0x01, 0x0a, 0x15, - 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x1a, 0x0a, - 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x12, 0x1a, 0x0a, 0x08, 0x76, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x41, 0x5a, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x41, 0x5a, 0x12, 0x3e, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x04, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, - 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x04, 0x74, 0x61, 0x67, 0x73, 0x1a, 0x37, 0x0a, 0x09, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x38, - 0x0a, 0x16, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x6e, 0x61, 0x70, - 0x73, 0x68, 0x6f, 0x74, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x6e, - 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x44, 0x22, 0x4f, 0x0a, 0x15, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x6e, 0x61, - 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, - 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x44, 0x22, 0x58, 0x0a, 0x12, 0x47, 0x65, 0x74, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x2a, 0x0a, 0x10, 0x70, 0x65, 0x72, 0x73, 0x69, - 0x73, 0x74, 0x65, 0x6e, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x10, 0x70, 0x65, 0x72, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x74, 0x56, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x22, 0x31, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x76, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x22, 0x74, 0x0a, 0x12, 0x53, 0x65, 0x74, 0x56, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, - 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, - 0x75, 0x67, 0x69, 0x6e, 0x12, 0x2a, 0x0a, 0x10, 0x70, 0x65, 0x72, 0x73, 0x69, 0x73, 0x74, 0x65, - 0x6e, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x10, - 0x70, 0x65, 0x72, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, - 0x12, 0x1a, 0x0a, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x22, 0x41, 0x0a, 0x13, - 0x53, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x10, 0x70, 0x65, 0x72, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, - 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x10, 0x70, - 0x65, 0x72, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x22, - 0xbe, 0x01, 0x0a, 0x1c, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, - 0x6f, 0x74, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x4b, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x65, 0x64, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, - 0x68, 0x6f, 0x74, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a, 0x39, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x32, 0xc0, 0x04, 0x0a, 0x11, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, - 0x68, 0x6f, 0x74, 0x74, 0x65, 0x72, 0x12, 0x41, 0x0a, 0x04, 0x49, 0x6e, 0x69, 0x74, 0x12, 0x27, - 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x69, 0x74, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, - 0x74, 0x65, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5b, 0x0a, 0x18, 0x43, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x46, 0x72, 0x6f, 0x6d, 0x53, 0x6e, 0x61, - 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x1e, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, - 0x64, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, - 0x64, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x52, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1f, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, - 0x74, 0x65, 0x64, 0x2e, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x66, - 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x65, 0x64, 0x2e, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x6e, - 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x55, 0x0a, 0x0e, 0x43, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x20, 0x2e, 0x67, - 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, - 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, - 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x44, 0x0a, 0x0e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, - 0x68, 0x6f, 0x74, 0x12, 0x20, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, - 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, - 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x4c, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x56, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x12, 0x1d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, - 0x65, 0x64, 0x2e, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, - 0x64, 0x2e, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4c, 0x0a, 0x0b, 0x53, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x49, 0x44, 0x12, 0x1d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, - 0x2e, 0x53, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, - 0x53, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, 0x6e, 0x7a, 0x75, 0x2f, 0x76, - 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, - 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, -} +const file_VolumeSnapshotter_proto_rawDesc = "" + + "\n" + + "\x17VolumeSnapshotter.proto\x12\tgenerated\x1a\fShared.proto\"\x9d\x01\n" + + "\x13CreateVolumeRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12\x1e\n" + + "\n" + + "snapshotID\x18\x02 \x01(\tR\n" + + "snapshotID\x12\x1e\n" + + "\n" + + "volumeType\x18\x03 \x01(\tR\n" + + "volumeType\x12\x1a\n" + + "\bvolumeAZ\x18\x04 \x01(\tR\bvolumeAZ\x12\x12\n" + + "\x04iops\x18\x05 \x01(\x03R\x04iops\"2\n" + + "\x14CreateVolumeResponse\x12\x1a\n" + + "\bvolumeID\x18\x01 \x01(\tR\bvolumeID\"f\n" + + "\x14GetVolumeInfoRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12\x1a\n" + + "\bvolumeID\x18\x02 \x01(\tR\bvolumeID\x12\x1a\n" + + "\bvolumeAZ\x18\x03 \x01(\tR\bvolumeAZ\"K\n" + + "\x15GetVolumeInfoResponse\x12\x1e\n" + + "\n" + + "volumeType\x18\x01 \x01(\tR\n" + + "volumeType\x12\x12\n" + + "\x04iops\x18\x02 \x01(\x03R\x04iops\"\xe0\x01\n" + + "\x15CreateSnapshotRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12\x1a\n" + + "\bvolumeID\x18\x02 \x01(\tR\bvolumeID\x12\x1a\n" + + "\bvolumeAZ\x18\x03 \x01(\tR\bvolumeAZ\x12>\n" + + "\x04tags\x18\x04 \x03(\v2*.generated.CreateSnapshotRequest.TagsEntryR\x04tags\x1a7\n" + + "\tTagsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"8\n" + + "\x16CreateSnapshotResponse\x12\x1e\n" + + "\n" + + "snapshotID\x18\x01 \x01(\tR\n" + + "snapshotID\"O\n" + + "\x15DeleteSnapshotRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12\x1e\n" + + "\n" + + "snapshotID\x18\x02 \x01(\tR\n" + + "snapshotID\"X\n" + + "\x12GetVolumeIDRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12*\n" + + "\x10persistentVolume\x18\x02 \x01(\fR\x10persistentVolume\"1\n" + + "\x13GetVolumeIDResponse\x12\x1a\n" + + "\bvolumeID\x18\x01 \x01(\tR\bvolumeID\"t\n" + + "\x12SetVolumeIDRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12*\n" + + "\x10persistentVolume\x18\x02 \x01(\fR\x10persistentVolume\x12\x1a\n" + + "\bvolumeID\x18\x03 \x01(\tR\bvolumeID\"A\n" + + "\x13SetVolumeIDResponse\x12*\n" + + "\x10persistentVolume\x18\x01 \x01(\fR\x10persistentVolume\"\xbe\x01\n" + + "\x1cVolumeSnapshotterInitRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12K\n" + + "\x06config\x18\x02 \x03(\v23.generated.VolumeSnapshotterInitRequest.ConfigEntryR\x06config\x1a9\n" + + "\vConfigEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x012\xc0\x04\n" + + "\x11VolumeSnapshotter\x12A\n" + + "\x04Init\x12'.generated.VolumeSnapshotterInitRequest\x1a\x10.generated.Empty\x12[\n" + + "\x18CreateVolumeFromSnapshot\x12\x1e.generated.CreateVolumeRequest\x1a\x1f.generated.CreateVolumeResponse\x12R\n" + + "\rGetVolumeInfo\x12\x1f.generated.GetVolumeInfoRequest\x1a .generated.GetVolumeInfoResponse\x12U\n" + + "\x0eCreateSnapshot\x12 .generated.CreateSnapshotRequest\x1a!.generated.CreateSnapshotResponse\x12D\n" + + "\x0eDeleteSnapshot\x12 .generated.DeleteSnapshotRequest\x1a\x10.generated.Empty\x12L\n" + + "\vGetVolumeID\x12\x1d.generated.GetVolumeIDRequest\x1a\x1e.generated.GetVolumeIDResponse\x12L\n" + + "\vSetVolumeID\x12\x1d.generated.SetVolumeIDRequest\x1a\x1e.generated.SetVolumeIDResponseB5Z3github.com/vmware-tanzu/velero/pkg/plugin/generatedb\x06proto3" var ( file_VolumeSnapshotter_proto_rawDescOnce sync.Once - file_VolumeSnapshotter_proto_rawDescData = file_VolumeSnapshotter_proto_rawDesc + file_VolumeSnapshotter_proto_rawDescData []byte ) func file_VolumeSnapshotter_proto_rawDescGZIP() []byte { file_VolumeSnapshotter_proto_rawDescOnce.Do(func() { - file_VolumeSnapshotter_proto_rawDescData = protoimpl.X.CompressGZIP(file_VolumeSnapshotter_proto_rawDescData) + file_VolumeSnapshotter_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_VolumeSnapshotter_proto_rawDesc), len(file_VolumeSnapshotter_proto_rawDesc))) }) return file_VolumeSnapshotter_proto_rawDescData } var file_VolumeSnapshotter_proto_msgTypes = make([]protoimpl.MessageInfo, 14) -var file_VolumeSnapshotter_proto_goTypes = []interface{}{ +var file_VolumeSnapshotter_proto_goTypes = []any{ (*CreateVolumeRequest)(nil), // 0: generated.CreateVolumeRequest (*CreateVolumeResponse)(nil), // 1: generated.CreateVolumeResponse (*GetVolumeInfoRequest)(nil), // 2: generated.GetVolumeInfoRequest @@ -893,157 +798,11 @@ func file_VolumeSnapshotter_proto_init() { return } file_Shared_proto_init() - if !protoimpl.UnsafeEnabled { - file_VolumeSnapshotter_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateVolumeRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_VolumeSnapshotter_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateVolumeResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_VolumeSnapshotter_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetVolumeInfoRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_VolumeSnapshotter_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetVolumeInfoResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_VolumeSnapshotter_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateSnapshotRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_VolumeSnapshotter_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateSnapshotResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_VolumeSnapshotter_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DeleteSnapshotRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_VolumeSnapshotter_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetVolumeIDRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_VolumeSnapshotter_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetVolumeIDResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_VolumeSnapshotter_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetVolumeIDRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_VolumeSnapshotter_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetVolumeIDResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_VolumeSnapshotter_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeSnapshotterInitRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_VolumeSnapshotter_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_VolumeSnapshotter_proto_rawDesc), len(file_VolumeSnapshotter_proto_rawDesc)), NumEnums: 0, NumMessages: 14, NumExtensions: 0, @@ -1054,7 +813,6 @@ func file_VolumeSnapshotter_proto_init() { MessageInfos: file_VolumeSnapshotter_proto_msgTypes, }.Build() File_VolumeSnapshotter_proto = out.File - file_VolumeSnapshotter_proto_rawDesc = nil file_VolumeSnapshotter_proto_goTypes = nil file_VolumeSnapshotter_proto_depIdxs = nil } diff --git a/pkg/plugin/generated/backupitemaction/v2/BackupItemAction.pb.go b/pkg/plugin/generated/backupitemaction/v2/BackupItemAction.pb.go index 5eb2c852b..097dfc721 100644 --- a/pkg/plugin/generated/backupitemaction/v2/BackupItemAction.pb.go +++ b/pkg/plugin/generated/backupitemaction/v2/BackupItemAction.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 +// protoc-gen-go v1.36.11-devel // protoc v4.25.2 // source: backupitemaction/v2/BackupItemAction.proto @@ -13,6 +13,7 @@ import ( emptypb "google.golang.org/protobuf/types/known/emptypb" reflect "reflect" sync "sync" + unsafe "unsafe" ) const ( @@ -23,22 +24,19 @@ const ( ) type ExecuteRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + Item []byte `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` + Backup []byte `protobuf:"bytes,3,opt,name=backup,proto3" json:"backup,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - Item []byte `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` - Backup []byte `protobuf:"bytes,3,opt,name=backup,proto3" json:"backup,omitempty"` + sizeCache protoimpl.SizeCache } func (x *ExecuteRequest) Reset() { *x = ExecuteRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ExecuteRequest) String() string { @@ -49,7 +47,7 @@ func (*ExecuteRequest) ProtoMessage() {} func (x *ExecuteRequest) ProtoReflect() protoreflect.Message { mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -86,23 +84,20 @@ func (x *ExecuteRequest) GetBackup() []byte { } type ExecuteResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` Item []byte `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"` AdditionalItems []*generated.ResourceIdentifier `protobuf:"bytes,2,rep,name=additionalItems,proto3" json:"additionalItems,omitempty"` OperationID string `protobuf:"bytes,3,opt,name=operationID,proto3" json:"operationID,omitempty"` PostOperationItems []*generated.ResourceIdentifier `protobuf:"bytes,4,rep,name=postOperationItems,proto3" json:"postOperationItems,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ExecuteResponse) Reset() { *x = ExecuteResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ExecuteResponse) String() string { @@ -113,7 +108,7 @@ func (*ExecuteResponse) ProtoMessage() {} func (x *ExecuteResponse) ProtoReflect() protoreflect.Message { mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -157,20 +152,17 @@ func (x *ExecuteResponse) GetPostOperationItems() []*generated.ResourceIdentifie } type BackupItemActionAppliesToRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + sizeCache protoimpl.SizeCache } func (x *BackupItemActionAppliesToRequest) Reset() { *x = BackupItemActionAppliesToRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *BackupItemActionAppliesToRequest) String() string { @@ -181,7 +173,7 @@ func (*BackupItemActionAppliesToRequest) ProtoMessage() {} func (x *BackupItemActionAppliesToRequest) ProtoReflect() protoreflect.Message { mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -204,20 +196,17 @@ func (x *BackupItemActionAppliesToRequest) GetPlugin() string { } type BackupItemActionAppliesToResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` ResourceSelector *generated.ResourceSelector `protobuf:"bytes,1,opt,name=ResourceSelector,proto3" json:"ResourceSelector,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *BackupItemActionAppliesToResponse) Reset() { *x = BackupItemActionAppliesToResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *BackupItemActionAppliesToResponse) String() string { @@ -228,7 +217,7 @@ func (*BackupItemActionAppliesToResponse) ProtoMessage() {} func (x *BackupItemActionAppliesToResponse) ProtoReflect() protoreflect.Message { mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -251,22 +240,19 @@ func (x *BackupItemActionAppliesToResponse) GetResourceSelector() *generated.Res } type BackupItemActionProgressRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + OperationID string `protobuf:"bytes,2,opt,name=operationID,proto3" json:"operationID,omitempty"` + Backup []byte `protobuf:"bytes,3,opt,name=backup,proto3" json:"backup,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - OperationID string `protobuf:"bytes,2,opt,name=operationID,proto3" json:"operationID,omitempty"` - Backup []byte `protobuf:"bytes,3,opt,name=backup,proto3" json:"backup,omitempty"` + sizeCache protoimpl.SizeCache } func (x *BackupItemActionProgressRequest) Reset() { *x = BackupItemActionProgressRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *BackupItemActionProgressRequest) String() string { @@ -277,7 +263,7 @@ func (*BackupItemActionProgressRequest) ProtoMessage() {} func (x *BackupItemActionProgressRequest) ProtoReflect() protoreflect.Message { mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -314,20 +300,17 @@ func (x *BackupItemActionProgressRequest) GetBackup() []byte { } type BackupItemActionProgressResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Progress *generated.OperationProgress `protobuf:"bytes,1,opt,name=progress,proto3" json:"progress,omitempty"` unknownFields protoimpl.UnknownFields - - Progress *generated.OperationProgress `protobuf:"bytes,1,opt,name=progress,proto3" json:"progress,omitempty"` + sizeCache protoimpl.SizeCache } func (x *BackupItemActionProgressResponse) Reset() { *x = BackupItemActionProgressResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *BackupItemActionProgressResponse) String() string { @@ -338,7 +321,7 @@ func (*BackupItemActionProgressResponse) ProtoMessage() {} func (x *BackupItemActionProgressResponse) ProtoReflect() protoreflect.Message { mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -361,22 +344,19 @@ func (x *BackupItemActionProgressResponse) GetProgress() *generated.OperationPro } type BackupItemActionCancelRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + OperationID string `protobuf:"bytes,2,opt,name=operationID,proto3" json:"operationID,omitempty"` + Backup []byte `protobuf:"bytes,3,opt,name=backup,proto3" json:"backup,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - OperationID string `protobuf:"bytes,2,opt,name=operationID,proto3" json:"operationID,omitempty"` - Backup []byte `protobuf:"bytes,3,opt,name=backup,proto3" json:"backup,omitempty"` + sizeCache protoimpl.SizeCache } func (x *BackupItemActionCancelRequest) Reset() { *x = BackupItemActionCancelRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *BackupItemActionCancelRequest) String() string { @@ -387,7 +367,7 @@ func (*BackupItemActionCancelRequest) ProtoMessage() {} func (x *BackupItemActionCancelRequest) ProtoReflect() protoreflect.Message { mi := &file_backupitemaction_v2_BackupItemAction_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -425,105 +405,52 @@ func (x *BackupItemActionCancelRequest) GetBackup() []byte { var File_backupitemaction_v2_BackupItemAction_proto protoreflect.FileDescriptor -var file_backupitemaction_v2_BackupItemAction_proto_rawDesc = []byte{ - 0x0a, 0x2a, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x69, 0x74, 0x65, 0x6d, 0x61, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x2f, 0x76, 0x32, 0x2f, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, - 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x02, 0x76, 0x32, - 0x1a, 0x0c, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, - 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x54, 0x0a, 0x0e, 0x45, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, - 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, - 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x61, 0x63, - 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, - 0x70, 0x22, 0xdf, 0x01, 0x0a, 0x0f, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x47, 0x0a, 0x0f, 0x61, 0x64, 0x64, - 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, - 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, - 0x72, 0x52, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, - 0x6d, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, - 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x4d, 0x0a, 0x12, 0x70, 0x6f, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x1d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x52, - 0x12, 0x70, 0x6f, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x74, - 0x65, 0x6d, 0x73, 0x22, 0x3a, 0x0a, 0x20, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, - 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, - 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x22, - 0x6c, 0x0a, 0x21, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, - 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x10, 0x52, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x73, 0x0a, - 0x1f, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, - 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x61, - 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x62, 0x61, 0x63, 0x6b, - 0x75, 0x70, 0x22, 0x5c, 0x0a, 0x20, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, - 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, - 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, - 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, - 0x22, 0x71, 0x0a, 0x1d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, 0x62, - 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x62, 0x61, 0x63, - 0x6b, 0x75, 0x70, 0x32, 0xbc, 0x02, 0x0a, 0x10, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, - 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x58, 0x0a, 0x09, 0x41, 0x70, 0x70, 0x6c, - 0x69, 0x65, 0x73, 0x54, 0x6f, 0x12, 0x24, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, - 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, - 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x76, 0x32, - 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x32, 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x12, 0x12, 0x2e, - 0x76, 0x32, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x13, 0x2e, 0x76, 0x32, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x55, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, - 0x73, 0x73, 0x12, 0x23, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, - 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, - 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, - 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, - 0x06, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x12, 0x21, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, - 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x61, 0x6e, - 0x63, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x42, 0x49, 0x5a, 0x47, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, 0x6e, 0x7a, 0x75, 0x2f, 0x76, 0x65, - 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, - 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, - 0x69, 0x74, 0x65, 0x6d, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x32, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, -} +const file_backupitemaction_v2_BackupItemAction_proto_rawDesc = "" + + "\n" + + "*backupitemaction/v2/BackupItemAction.proto\x12\x02v2\x1a\fShared.proto\x1a\x1bgoogle/protobuf/empty.proto\"T\n" + + "\x0eExecuteRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12\x12\n" + + "\x04item\x18\x02 \x01(\fR\x04item\x12\x16\n" + + "\x06backup\x18\x03 \x01(\fR\x06backup\"\xdf\x01\n" + + "\x0fExecuteResponse\x12\x12\n" + + "\x04item\x18\x01 \x01(\fR\x04item\x12G\n" + + "\x0fadditionalItems\x18\x02 \x03(\v2\x1d.generated.ResourceIdentifierR\x0fadditionalItems\x12 \n" + + "\voperationID\x18\x03 \x01(\tR\voperationID\x12M\n" + + "\x12postOperationItems\x18\x04 \x03(\v2\x1d.generated.ResourceIdentifierR\x12postOperationItems\":\n" + + " BackupItemActionAppliesToRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\"l\n" + + "!BackupItemActionAppliesToResponse\x12G\n" + + "\x10ResourceSelector\x18\x01 \x01(\v2\x1b.generated.ResourceSelectorR\x10ResourceSelector\"s\n" + + "\x1fBackupItemActionProgressRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12 \n" + + "\voperationID\x18\x02 \x01(\tR\voperationID\x12\x16\n" + + "\x06backup\x18\x03 \x01(\fR\x06backup\"\\\n" + + " BackupItemActionProgressResponse\x128\n" + + "\bprogress\x18\x01 \x01(\v2\x1c.generated.OperationProgressR\bprogress\"q\n" + + "\x1dBackupItemActionCancelRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12 \n" + + "\voperationID\x18\x02 \x01(\tR\voperationID\x12\x16\n" + + "\x06backup\x18\x03 \x01(\fR\x06backup2\xbc\x02\n" + + "\x10BackupItemAction\x12X\n" + + "\tAppliesTo\x12$.v2.BackupItemActionAppliesToRequest\x1a%.v2.BackupItemActionAppliesToResponse\x122\n" + + "\aExecute\x12\x12.v2.ExecuteRequest\x1a\x13.v2.ExecuteResponse\x12U\n" + + "\bProgress\x12#.v2.BackupItemActionProgressRequest\x1a$.v2.BackupItemActionProgressResponse\x12C\n" + + "\x06Cancel\x12!.v2.BackupItemActionCancelRequest\x1a\x16.google.protobuf.EmptyBIZGgithub.com/vmware-tanzu/velero/pkg/plugin/generated/backupitemaction/v2b\x06proto3" var ( file_backupitemaction_v2_BackupItemAction_proto_rawDescOnce sync.Once - file_backupitemaction_v2_BackupItemAction_proto_rawDescData = file_backupitemaction_v2_BackupItemAction_proto_rawDesc + file_backupitemaction_v2_BackupItemAction_proto_rawDescData []byte ) func file_backupitemaction_v2_BackupItemAction_proto_rawDescGZIP() []byte { file_backupitemaction_v2_BackupItemAction_proto_rawDescOnce.Do(func() { - file_backupitemaction_v2_BackupItemAction_proto_rawDescData = protoimpl.X.CompressGZIP(file_backupitemaction_v2_BackupItemAction_proto_rawDescData) + file_backupitemaction_v2_BackupItemAction_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_backupitemaction_v2_BackupItemAction_proto_rawDesc), len(file_backupitemaction_v2_BackupItemAction_proto_rawDesc))) }) return file_backupitemaction_v2_BackupItemAction_proto_rawDescData } var file_backupitemaction_v2_BackupItemAction_proto_msgTypes = make([]protoimpl.MessageInfo, 7) -var file_backupitemaction_v2_BackupItemAction_proto_goTypes = []interface{}{ +var file_backupitemaction_v2_BackupItemAction_proto_goTypes = []any{ (*ExecuteRequest)(nil), // 0: v2.ExecuteRequest (*ExecuteResponse)(nil), // 1: v2.ExecuteResponse (*BackupItemActionAppliesToRequest)(nil), // 2: v2.BackupItemActionAppliesToRequest @@ -561,97 +488,11 @@ func file_backupitemaction_v2_BackupItemAction_proto_init() { if File_backupitemaction_v2_BackupItemAction_proto != nil { return } - if !protoimpl.UnsafeEnabled { - file_backupitemaction_v2_BackupItemAction_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ExecuteRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_backupitemaction_v2_BackupItemAction_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ExecuteResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_backupitemaction_v2_BackupItemAction_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BackupItemActionAppliesToRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_backupitemaction_v2_BackupItemAction_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BackupItemActionAppliesToResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_backupitemaction_v2_BackupItemAction_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BackupItemActionProgressRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_backupitemaction_v2_BackupItemAction_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BackupItemActionProgressResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_backupitemaction_v2_BackupItemAction_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BackupItemActionCancelRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_backupitemaction_v2_BackupItemAction_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_backupitemaction_v2_BackupItemAction_proto_rawDesc), len(file_backupitemaction_v2_BackupItemAction_proto_rawDesc)), NumEnums: 0, NumMessages: 7, NumExtensions: 0, @@ -662,7 +503,6 @@ func file_backupitemaction_v2_BackupItemAction_proto_init() { MessageInfos: file_backupitemaction_v2_BackupItemAction_proto_msgTypes, }.Build() File_backupitemaction_v2_BackupItemAction_proto = out.File - file_backupitemaction_v2_BackupItemAction_proto_rawDesc = nil file_backupitemaction_v2_BackupItemAction_proto_goTypes = nil file_backupitemaction_v2_BackupItemAction_proto_depIdxs = nil } diff --git a/pkg/plugin/generated/itemblockaction/v1/ItemBlockAction.pb.go b/pkg/plugin/generated/itemblockaction/v1/ItemBlockAction.pb.go index cec604477..6d73eb826 100644 --- a/pkg/plugin/generated/itemblockaction/v1/ItemBlockAction.pb.go +++ b/pkg/plugin/generated/itemblockaction/v1/ItemBlockAction.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 +// protoc-gen-go v1.36.11-devel // protoc v4.25.2 // source: itemblockaction/v1/ItemBlockAction.proto @@ -12,6 +12,7 @@ import ( protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" + unsafe "unsafe" ) const ( @@ -22,20 +23,17 @@ const ( ) type ItemBlockActionAppliesToRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + sizeCache protoimpl.SizeCache } func (x *ItemBlockActionAppliesToRequest) Reset() { *x = ItemBlockActionAppliesToRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ItemBlockActionAppliesToRequest) String() string { @@ -46,7 +44,7 @@ func (*ItemBlockActionAppliesToRequest) ProtoMessage() {} func (x *ItemBlockActionAppliesToRequest) ProtoReflect() protoreflect.Message { mi := &file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -69,20 +67,17 @@ func (x *ItemBlockActionAppliesToRequest) GetPlugin() string { } type ItemBlockActionAppliesToResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` ResourceSelector *generated.ResourceSelector `protobuf:"bytes,1,opt,name=ResourceSelector,proto3" json:"ResourceSelector,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ItemBlockActionAppliesToResponse) Reset() { *x = ItemBlockActionAppliesToResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ItemBlockActionAppliesToResponse) String() string { @@ -93,7 +88,7 @@ func (*ItemBlockActionAppliesToResponse) ProtoMessage() {} func (x *ItemBlockActionAppliesToResponse) ProtoReflect() protoreflect.Message { mi := &file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -116,22 +111,19 @@ func (x *ItemBlockActionAppliesToResponse) GetResourceSelector() *generated.Reso } type ItemBlockActionGetRelatedItemsRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + Item []byte `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` + Backup []byte `protobuf:"bytes,3,opt,name=backup,proto3" json:"backup,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - Item []byte `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` - Backup []byte `protobuf:"bytes,3,opt,name=backup,proto3" json:"backup,omitempty"` + sizeCache protoimpl.SizeCache } func (x *ItemBlockActionGetRelatedItemsRequest) Reset() { *x = ItemBlockActionGetRelatedItemsRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ItemBlockActionGetRelatedItemsRequest) String() string { @@ -142,7 +134,7 @@ func (*ItemBlockActionGetRelatedItemsRequest) ProtoMessage() {} func (x *ItemBlockActionGetRelatedItemsRequest) ProtoReflect() protoreflect.Message { mi := &file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -179,20 +171,17 @@ func (x *ItemBlockActionGetRelatedItemsRequest) GetBackup() []byte { } type ItemBlockActionGetRelatedItemsResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + RelatedItems []*generated.ResourceIdentifier `protobuf:"bytes,1,rep,name=relatedItems,proto3" json:"relatedItems,omitempty"` unknownFields protoimpl.UnknownFields - - RelatedItems []*generated.ResourceIdentifier `protobuf:"bytes,1,rep,name=relatedItems,proto3" json:"relatedItems,omitempty"` + sizeCache protoimpl.SizeCache } func (x *ItemBlockActionGetRelatedItemsResponse) Reset() { *x = ItemBlockActionGetRelatedItemsResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ItemBlockActionGetRelatedItemsResponse) String() string { @@ -203,7 +192,7 @@ func (*ItemBlockActionGetRelatedItemsResponse) ProtoMessage() {} func (x *ItemBlockActionGetRelatedItemsResponse) ProtoReflect() protoreflect.Message { mi := &file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -227,70 +216,37 @@ func (x *ItemBlockActionGetRelatedItemsResponse) GetRelatedItems() []*generated. var File_itemblockaction_v1_ItemBlockAction_proto protoreflect.FileDescriptor -var file_itemblockaction_v1_ItemBlockAction_proto_rawDesc = []byte{ - 0x0a, 0x28, 0x69, 0x74, 0x65, 0x6d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x61, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x2f, 0x76, 0x31, 0x2f, 0x49, 0x74, 0x65, 0x6d, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x41, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x02, 0x76, 0x31, 0x1a, 0x0c, - 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x39, 0x0a, 0x1f, - 0x49, 0x74, 0x65, 0x6d, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, - 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x22, 0x6b, 0x0a, 0x20, 0x49, 0x74, 0x65, 0x6d, 0x42, - 0x6c, 0x6f, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, - 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x10, 0x52, - 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, - 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, - 0x6f, 0x72, 0x52, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x22, 0x6b, 0x0a, 0x25, 0x49, 0x74, 0x65, 0x6d, 0x42, 0x6c, 0x6f, 0x63, - 0x6b, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x6c, 0x61, 0x74, 0x65, - 0x64, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, - 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, - 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x61, 0x63, - 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, - 0x70, 0x22, 0x6b, 0x0a, 0x26, 0x49, 0x74, 0x65, 0x6d, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x41, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x49, 0x74, - 0x65, 0x6d, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x0c, 0x72, - 0x65, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, - 0x52, 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x32, 0xd3, - 0x01, 0x0a, 0x0f, 0x49, 0x74, 0x65, 0x6d, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x56, 0x0a, 0x09, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x12, - 0x23, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x74, 0x65, 0x6d, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x41, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x74, 0x65, 0x6d, 0x42, 0x6c, - 0x6f, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, - 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x68, 0x0a, 0x0f, 0x47, 0x65, - 0x74, 0x52, 0x65, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x12, 0x29, 0x2e, - 0x76, 0x31, 0x2e, 0x49, 0x74, 0x65, 0x6d, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x49, 0x74, 0x65, 0x6d, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x74, - 0x65, 0x6d, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x47, 0x65, 0x74, - 0x52, 0x65, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x48, 0x5a, 0x46, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, 0x6e, 0x7a, 0x75, 0x2f, - 0x76, 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, - 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2f, 0x69, 0x74, 0x65, 0x6d, - 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x31, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} +const file_itemblockaction_v1_ItemBlockAction_proto_rawDesc = "" + + "\n" + + "(itemblockaction/v1/ItemBlockAction.proto\x12\x02v1\x1a\fShared.proto\"9\n" + + "\x1fItemBlockActionAppliesToRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\"k\n" + + " ItemBlockActionAppliesToResponse\x12G\n" + + "\x10ResourceSelector\x18\x01 \x01(\v2\x1b.generated.ResourceSelectorR\x10ResourceSelector\"k\n" + + "%ItemBlockActionGetRelatedItemsRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12\x12\n" + + "\x04item\x18\x02 \x01(\fR\x04item\x12\x16\n" + + "\x06backup\x18\x03 \x01(\fR\x06backup\"k\n" + + "&ItemBlockActionGetRelatedItemsResponse\x12A\n" + + "\frelatedItems\x18\x01 \x03(\v2\x1d.generated.ResourceIdentifierR\frelatedItems2\xd3\x01\n" + + "\x0fItemBlockAction\x12V\n" + + "\tAppliesTo\x12#.v1.ItemBlockActionAppliesToRequest\x1a$.v1.ItemBlockActionAppliesToResponse\x12h\n" + + "\x0fGetRelatedItems\x12).v1.ItemBlockActionGetRelatedItemsRequest\x1a*.v1.ItemBlockActionGetRelatedItemsResponseBHZFgithub.com/vmware-tanzu/velero/pkg/plugin/generated/itemblockaction/v1b\x06proto3" var ( file_itemblockaction_v1_ItemBlockAction_proto_rawDescOnce sync.Once - file_itemblockaction_v1_ItemBlockAction_proto_rawDescData = file_itemblockaction_v1_ItemBlockAction_proto_rawDesc + file_itemblockaction_v1_ItemBlockAction_proto_rawDescData []byte ) func file_itemblockaction_v1_ItemBlockAction_proto_rawDescGZIP() []byte { file_itemblockaction_v1_ItemBlockAction_proto_rawDescOnce.Do(func() { - file_itemblockaction_v1_ItemBlockAction_proto_rawDescData = protoimpl.X.CompressGZIP(file_itemblockaction_v1_ItemBlockAction_proto_rawDescData) + file_itemblockaction_v1_ItemBlockAction_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_itemblockaction_v1_ItemBlockAction_proto_rawDesc), len(file_itemblockaction_v1_ItemBlockAction_proto_rawDesc))) }) return file_itemblockaction_v1_ItemBlockAction_proto_rawDescData } var file_itemblockaction_v1_ItemBlockAction_proto_msgTypes = make([]protoimpl.MessageInfo, 4) -var file_itemblockaction_v1_ItemBlockAction_proto_goTypes = []interface{}{ +var file_itemblockaction_v1_ItemBlockAction_proto_goTypes = []any{ (*ItemBlockActionAppliesToRequest)(nil), // 0: v1.ItemBlockActionAppliesToRequest (*ItemBlockActionAppliesToResponse)(nil), // 1: v1.ItemBlockActionAppliesToResponse (*ItemBlockActionGetRelatedItemsRequest)(nil), // 2: v1.ItemBlockActionGetRelatedItemsRequest @@ -317,61 +273,11 @@ func file_itemblockaction_v1_ItemBlockAction_proto_init() { if File_itemblockaction_v1_ItemBlockAction_proto != nil { return } - if !protoimpl.UnsafeEnabled { - file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ItemBlockActionAppliesToRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ItemBlockActionAppliesToResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ItemBlockActionGetRelatedItemsRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_itemblockaction_v1_ItemBlockAction_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ItemBlockActionGetRelatedItemsResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_itemblockaction_v1_ItemBlockAction_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_itemblockaction_v1_ItemBlockAction_proto_rawDesc), len(file_itemblockaction_v1_ItemBlockAction_proto_rawDesc)), NumEnums: 0, NumMessages: 4, NumExtensions: 0, @@ -382,7 +288,6 @@ func file_itemblockaction_v1_ItemBlockAction_proto_init() { MessageInfos: file_itemblockaction_v1_ItemBlockAction_proto_msgTypes, }.Build() File_itemblockaction_v1_ItemBlockAction_proto = out.File - file_itemblockaction_v1_ItemBlockAction_proto_rawDesc = nil file_itemblockaction_v1_ItemBlockAction_proto_goTypes = nil file_itemblockaction_v1_ItemBlockAction_proto_depIdxs = nil } diff --git a/pkg/plugin/generated/restoreitemaction/v2/RestoreItemAction.pb.go b/pkg/plugin/generated/restoreitemaction/v2/RestoreItemAction.pb.go index a7bbc421e..48444045d 100644 --- a/pkg/plugin/generated/restoreitemaction/v2/RestoreItemAction.pb.go +++ b/pkg/plugin/generated/restoreitemaction/v2/RestoreItemAction.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 +// protoc-gen-go v1.36.11-devel // protoc v4.25.2 // source: restoreitemaction/v2/RestoreItemAction.proto @@ -14,6 +14,7 @@ import ( emptypb "google.golang.org/protobuf/types/known/emptypb" reflect "reflect" sync "sync" + unsafe "unsafe" ) const ( @@ -24,23 +25,20 @@ const ( ) type RestoreItemActionExecuteRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - Item []byte `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` - Restore []byte `protobuf:"bytes,3,opt,name=restore,proto3" json:"restore,omitempty"` - ItemFromBackup []byte `protobuf:"bytes,4,opt,name=itemFromBackup,proto3" json:"itemFromBackup,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + Item []byte `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` + Restore []byte `protobuf:"bytes,3,opt,name=restore,proto3" json:"restore,omitempty"` + ItemFromBackup []byte `protobuf:"bytes,4,opt,name=itemFromBackup,proto3" json:"itemFromBackup,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RestoreItemActionExecuteRequest) Reset() { *x = RestoreItemActionExecuteRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *RestoreItemActionExecuteRequest) String() string { @@ -51,7 +49,7 @@ func (*RestoreItemActionExecuteRequest) ProtoMessage() {} func (x *RestoreItemActionExecuteRequest) ProtoReflect() protoreflect.Message { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -95,25 +93,22 @@ func (x *RestoreItemActionExecuteRequest) GetItemFromBackup() []byte { } type RestoreItemActionExecuteResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` Item []byte `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"` AdditionalItems []*generated.ResourceIdentifier `protobuf:"bytes,2,rep,name=additionalItems,proto3" json:"additionalItems,omitempty"` SkipRestore bool `protobuf:"varint,3,opt,name=skipRestore,proto3" json:"skipRestore,omitempty"` OperationID string `protobuf:"bytes,4,opt,name=operationID,proto3" json:"operationID,omitempty"` WaitForAdditionalItems bool `protobuf:"varint,5,opt,name=waitForAdditionalItems,proto3" json:"waitForAdditionalItems,omitempty"` AdditionalItemsReadyTimeout *durationpb.Duration `protobuf:"bytes,6,opt,name=additionalItemsReadyTimeout,proto3" json:"additionalItemsReadyTimeout,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RestoreItemActionExecuteResponse) Reset() { *x = RestoreItemActionExecuteResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *RestoreItemActionExecuteResponse) String() string { @@ -124,7 +119,7 @@ func (*RestoreItemActionExecuteResponse) ProtoMessage() {} func (x *RestoreItemActionExecuteResponse) ProtoReflect() protoreflect.Message { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -182,20 +177,17 @@ func (x *RestoreItemActionExecuteResponse) GetAdditionalItemsReadyTimeout() *dur } type RestoreItemActionAppliesToRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + sizeCache protoimpl.SizeCache } func (x *RestoreItemActionAppliesToRequest) Reset() { *x = RestoreItemActionAppliesToRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *RestoreItemActionAppliesToRequest) String() string { @@ -206,7 +198,7 @@ func (*RestoreItemActionAppliesToRequest) ProtoMessage() {} func (x *RestoreItemActionAppliesToRequest) ProtoReflect() protoreflect.Message { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -229,20 +221,17 @@ func (x *RestoreItemActionAppliesToRequest) GetPlugin() string { } type RestoreItemActionAppliesToResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` ResourceSelector *generated.ResourceSelector `protobuf:"bytes,1,opt,name=ResourceSelector,proto3" json:"ResourceSelector,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RestoreItemActionAppliesToResponse) Reset() { *x = RestoreItemActionAppliesToResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *RestoreItemActionAppliesToResponse) String() string { @@ -253,7 +242,7 @@ func (*RestoreItemActionAppliesToResponse) ProtoMessage() {} func (x *RestoreItemActionAppliesToResponse) ProtoReflect() protoreflect.Message { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -276,22 +265,19 @@ func (x *RestoreItemActionAppliesToResponse) GetResourceSelector() *generated.Re } type RestoreItemActionProgressRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + OperationID string `protobuf:"bytes,2,opt,name=operationID,proto3" json:"operationID,omitempty"` + Restore []byte `protobuf:"bytes,3,opt,name=restore,proto3" json:"restore,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - OperationID string `protobuf:"bytes,2,opt,name=operationID,proto3" json:"operationID,omitempty"` - Restore []byte `protobuf:"bytes,3,opt,name=restore,proto3" json:"restore,omitempty"` + sizeCache protoimpl.SizeCache } func (x *RestoreItemActionProgressRequest) Reset() { *x = RestoreItemActionProgressRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *RestoreItemActionProgressRequest) String() string { @@ -302,7 +288,7 @@ func (*RestoreItemActionProgressRequest) ProtoMessage() {} func (x *RestoreItemActionProgressRequest) ProtoReflect() protoreflect.Message { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -339,20 +325,17 @@ func (x *RestoreItemActionProgressRequest) GetRestore() []byte { } type RestoreItemActionProgressResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Progress *generated.OperationProgress `protobuf:"bytes,1,opt,name=progress,proto3" json:"progress,omitempty"` unknownFields protoimpl.UnknownFields - - Progress *generated.OperationProgress `protobuf:"bytes,1,opt,name=progress,proto3" json:"progress,omitempty"` + sizeCache protoimpl.SizeCache } func (x *RestoreItemActionProgressResponse) Reset() { *x = RestoreItemActionProgressResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *RestoreItemActionProgressResponse) String() string { @@ -363,7 +346,7 @@ func (*RestoreItemActionProgressResponse) ProtoMessage() {} func (x *RestoreItemActionProgressResponse) ProtoReflect() protoreflect.Message { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -386,22 +369,19 @@ func (x *RestoreItemActionProgressResponse) GetProgress() *generated.OperationPr } type RestoreItemActionCancelRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` + OperationID string `protobuf:"bytes,2,opt,name=operationID,proto3" json:"operationID,omitempty"` + Restore []byte `protobuf:"bytes,3,opt,name=restore,proto3" json:"restore,omitempty"` unknownFields protoimpl.UnknownFields - - Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` - OperationID string `protobuf:"bytes,2,opt,name=operationID,proto3" json:"operationID,omitempty"` - Restore []byte `protobuf:"bytes,3,opt,name=restore,proto3" json:"restore,omitempty"` + sizeCache protoimpl.SizeCache } func (x *RestoreItemActionCancelRequest) Reset() { *x = RestoreItemActionCancelRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *RestoreItemActionCancelRequest) String() string { @@ -412,7 +392,7 @@ func (*RestoreItemActionCancelRequest) ProtoMessage() {} func (x *RestoreItemActionCancelRequest) ProtoReflect() protoreflect.Message { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -449,22 +429,19 @@ func (x *RestoreItemActionCancelRequest) GetRestore() []byte { } type RestoreItemActionItemsReadyRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` Plugin string `protobuf:"bytes,1,opt,name=plugin,proto3" json:"plugin,omitempty"` Restore []byte `protobuf:"bytes,2,opt,name=restore,proto3" json:"restore,omitempty"` AdditionalItems []*generated.ResourceIdentifier `protobuf:"bytes,3,rep,name=additionalItems,proto3" json:"additionalItems,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RestoreItemActionItemsReadyRequest) Reset() { *x = RestoreItemActionItemsReadyRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *RestoreItemActionItemsReadyRequest) String() string { @@ -475,7 +452,7 @@ func (*RestoreItemActionItemsReadyRequest) ProtoMessage() {} func (x *RestoreItemActionItemsReadyRequest) ProtoReflect() protoreflect.Message { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[7] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -512,20 +489,17 @@ func (x *RestoreItemActionItemsReadyRequest) GetAdditionalItems() []*generated.R } type RestoreItemActionItemsReadyResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Ready bool `protobuf:"varint,1,opt,name=ready,proto3" json:"ready,omitempty"` unknownFields protoimpl.UnknownFields - - Ready bool `protobuf:"varint,1,opt,name=ready,proto3" json:"ready,omitempty"` + sizeCache protoimpl.SizeCache } func (x *RestoreItemActionItemsReadyResponse) Reset() { *x = RestoreItemActionItemsReadyResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *RestoreItemActionItemsReadyResponse) String() string { @@ -536,7 +510,7 @@ func (*RestoreItemActionItemsReadyResponse) ProtoMessage() {} func (x *RestoreItemActionItemsReadyResponse) ProtoReflect() protoreflect.Message { mi := &file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[8] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -560,142 +534,62 @@ func (x *RestoreItemActionItemsReadyResponse) GetReady() bool { var File_restoreitemaction_v2_RestoreItemAction_proto protoreflect.FileDescriptor -var file_restoreitemaction_v2_RestoreItemAction_proto_rawDesc = []byte{ - 0x0a, 0x2c, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x69, 0x74, 0x65, 0x6d, 0x61, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x32, 0x2f, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, - 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x02, - 0x76, 0x32, 0x1a, 0x0c, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, - 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x8f, 0x01, - 0x0a, 0x1f, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, - 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x18, 0x0a, - 0x07, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, - 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x26, 0x0a, 0x0e, 0x69, 0x74, 0x65, 0x6d, 0x46, - 0x72, 0x6f, 0x6d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x0e, 0x69, 0x74, 0x65, 0x6d, 0x46, 0x72, 0x6f, 0x6d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x22, - 0xd8, 0x02, 0x0a, 0x20, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x47, 0x0a, 0x0f, 0x61, 0x64, 0x64, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, - 0x52, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, - 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x73, 0x6b, 0x69, 0x70, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x73, 0x6b, 0x69, 0x70, 0x52, 0x65, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x49, 0x44, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x36, 0x0a, 0x16, 0x77, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, - 0x41, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x16, 0x77, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x41, 0x64, - 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x12, 0x5b, 0x0a, - 0x1b, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, - 0x52, 0x65, 0x61, 0x64, 0x79, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x1b, 0x61, - 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, - 0x61, 0x64, 0x79, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x22, 0x3b, 0x0a, 0x21, 0x52, 0x65, - 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, - 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x22, 0x6d, 0x0a, 0x22, 0x52, 0x65, 0x73, 0x74, 0x6f, - 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, - 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, - 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, - 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x52, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, - 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x76, 0x0a, 0x20, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, - 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, - 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, - 0x69, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, - 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x22, 0x5d, - 0x0a, 0x21, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, - 0x64, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, - 0x65, 0x73, 0x73, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x22, 0x74, 0x0a, - 0x1e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x73, - 0x74, 0x6f, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x72, 0x65, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x22, 0x9f, 0x01, 0x0a, 0x22, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, - 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, - 0x61, 0x64, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, - 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, - 0x69, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x07, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x47, 0x0a, 0x0f, - 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x18, - 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, - 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, - 0x66, 0x69, 0x65, 0x72, 0x52, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, - 0x49, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x3b, 0x0a, 0x23, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, - 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, - 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, - 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, - 0x64, 0x79, 0x32, 0xd0, 0x03, 0x0a, 0x11, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, - 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x5a, 0x0a, 0x09, 0x41, 0x70, 0x70, 0x6c, - 0x69, 0x65, 0x73, 0x54, 0x6f, 0x12, 0x25, 0x2e, 0x76, 0x32, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, - 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, - 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x76, - 0x32, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x12, - 0x23, 0x2e, 0x76, 0x32, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, - 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x76, 0x32, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x08, 0x50, 0x72, - 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x24, 0x2e, 0x76, 0x32, 0x2e, 0x52, 0x65, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, - 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x76, - 0x32, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x06, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x12, 0x22, 0x2e, - 0x76, 0x32, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x6a, 0x0a, 0x17, 0x41, 0x72, 0x65, - 0x41, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, - 0x65, 0x61, 0x64, 0x79, 0x12, 0x26, 0x2e, 0x76, 0x32, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, - 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x76, - 0x32, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x4a, 0x5a, 0x48, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, 0x6e, 0x7a, 0x75, - 0x2f, 0x76, 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, - 0x69, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2f, 0x72, 0x65, 0x73, - 0x74, 0x6f, 0x72, 0x65, 0x69, 0x74, 0x65, 0x6d, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, - 0x32, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} +const file_restoreitemaction_v2_RestoreItemAction_proto_rawDesc = "" + + "\n" + + ",restoreitemaction/v2/RestoreItemAction.proto\x12\x02v2\x1a\fShared.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1egoogle/protobuf/duration.proto\"\x8f\x01\n" + + "\x1fRestoreItemActionExecuteRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12\x12\n" + + "\x04item\x18\x02 \x01(\fR\x04item\x12\x18\n" + + "\arestore\x18\x03 \x01(\fR\arestore\x12&\n" + + "\x0eitemFromBackup\x18\x04 \x01(\fR\x0eitemFromBackup\"\xd8\x02\n" + + " RestoreItemActionExecuteResponse\x12\x12\n" + + "\x04item\x18\x01 \x01(\fR\x04item\x12G\n" + + "\x0fadditionalItems\x18\x02 \x03(\v2\x1d.generated.ResourceIdentifierR\x0fadditionalItems\x12 \n" + + "\vskipRestore\x18\x03 \x01(\bR\vskipRestore\x12 \n" + + "\voperationID\x18\x04 \x01(\tR\voperationID\x126\n" + + "\x16waitForAdditionalItems\x18\x05 \x01(\bR\x16waitForAdditionalItems\x12[\n" + + "\x1badditionalItemsReadyTimeout\x18\x06 \x01(\v2\x19.google.protobuf.DurationR\x1badditionalItemsReadyTimeout\";\n" + + "!RestoreItemActionAppliesToRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\"m\n" + + "\"RestoreItemActionAppliesToResponse\x12G\n" + + "\x10ResourceSelector\x18\x01 \x01(\v2\x1b.generated.ResourceSelectorR\x10ResourceSelector\"v\n" + + " RestoreItemActionProgressRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12 \n" + + "\voperationID\x18\x02 \x01(\tR\voperationID\x12\x18\n" + + "\arestore\x18\x03 \x01(\fR\arestore\"]\n" + + "!RestoreItemActionProgressResponse\x128\n" + + "\bprogress\x18\x01 \x01(\v2\x1c.generated.OperationProgressR\bprogress\"t\n" + + "\x1eRestoreItemActionCancelRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12 \n" + + "\voperationID\x18\x02 \x01(\tR\voperationID\x12\x18\n" + + "\arestore\x18\x03 \x01(\fR\arestore\"\x9f\x01\n" + + "\"RestoreItemActionItemsReadyRequest\x12\x16\n" + + "\x06plugin\x18\x01 \x01(\tR\x06plugin\x12\x18\n" + + "\arestore\x18\x02 \x01(\fR\arestore\x12G\n" + + "\x0fadditionalItems\x18\x03 \x03(\v2\x1d.generated.ResourceIdentifierR\x0fadditionalItems\";\n" + + "#RestoreItemActionItemsReadyResponse\x12\x14\n" + + "\x05ready\x18\x01 \x01(\bR\x05ready2\xd0\x03\n" + + "\x11RestoreItemAction\x12Z\n" + + "\tAppliesTo\x12%.v2.RestoreItemActionAppliesToRequest\x1a&.v2.RestoreItemActionAppliesToResponse\x12T\n" + + "\aExecute\x12#.v2.RestoreItemActionExecuteRequest\x1a$.v2.RestoreItemActionExecuteResponse\x12W\n" + + "\bProgress\x12$.v2.RestoreItemActionProgressRequest\x1a%.v2.RestoreItemActionProgressResponse\x12D\n" + + "\x06Cancel\x12\".v2.RestoreItemActionCancelRequest\x1a\x16.google.protobuf.Empty\x12j\n" + + "\x17AreAdditionalItemsReady\x12&.v2.RestoreItemActionItemsReadyRequest\x1a'.v2.RestoreItemActionItemsReadyResponseBJZHgithub.com/vmware-tanzu/velero/pkg/plugin/generated/restoreitemaction/v2b\x06proto3" var ( file_restoreitemaction_v2_RestoreItemAction_proto_rawDescOnce sync.Once - file_restoreitemaction_v2_RestoreItemAction_proto_rawDescData = file_restoreitemaction_v2_RestoreItemAction_proto_rawDesc + file_restoreitemaction_v2_RestoreItemAction_proto_rawDescData []byte ) func file_restoreitemaction_v2_RestoreItemAction_proto_rawDescGZIP() []byte { file_restoreitemaction_v2_RestoreItemAction_proto_rawDescOnce.Do(func() { - file_restoreitemaction_v2_RestoreItemAction_proto_rawDescData = protoimpl.X.CompressGZIP(file_restoreitemaction_v2_RestoreItemAction_proto_rawDescData) + file_restoreitemaction_v2_RestoreItemAction_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_restoreitemaction_v2_RestoreItemAction_proto_rawDesc), len(file_restoreitemaction_v2_RestoreItemAction_proto_rawDesc))) }) return file_restoreitemaction_v2_RestoreItemAction_proto_rawDescData } var file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes = make([]protoimpl.MessageInfo, 9) -var file_restoreitemaction_v2_RestoreItemAction_proto_goTypes = []interface{}{ +var file_restoreitemaction_v2_RestoreItemAction_proto_goTypes = []any{ (*RestoreItemActionExecuteRequest)(nil), // 0: v2.RestoreItemActionExecuteRequest (*RestoreItemActionExecuteResponse)(nil), // 1: v2.RestoreItemActionExecuteResponse (*RestoreItemActionAppliesToRequest)(nil), // 2: v2.RestoreItemActionAppliesToRequest @@ -739,121 +633,11 @@ func file_restoreitemaction_v2_RestoreItemAction_proto_init() { if File_restoreitemaction_v2_RestoreItemAction_proto != nil { return } - if !protoimpl.UnsafeEnabled { - file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RestoreItemActionExecuteRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RestoreItemActionExecuteResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RestoreItemActionAppliesToRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RestoreItemActionAppliesToResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RestoreItemActionProgressRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RestoreItemActionProgressResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RestoreItemActionCancelRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RestoreItemActionItemsReadyRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RestoreItemActionItemsReadyResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_restoreitemaction_v2_RestoreItemAction_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_restoreitemaction_v2_RestoreItemAction_proto_rawDesc), len(file_restoreitemaction_v2_RestoreItemAction_proto_rawDesc)), NumEnums: 0, NumMessages: 9, NumExtensions: 0, @@ -864,7 +648,6 @@ func file_restoreitemaction_v2_RestoreItemAction_proto_init() { MessageInfos: file_restoreitemaction_v2_RestoreItemAction_proto_msgTypes, }.Build() File_restoreitemaction_v2_RestoreItemAction_proto = out.File - file_restoreitemaction_v2_RestoreItemAction_proto_rawDesc = nil file_restoreitemaction_v2_RestoreItemAction_proto_goTypes = nil file_restoreitemaction_v2_RestoreItemAction_proto_depIdxs = nil } diff --git a/pkg/podexec/pod_command_executor.go b/pkg/podexec/pod_command_executor.go index 4ba4d4dc9..997795c35 100644 --- a/pkg/podexec/pod_command_executor.go +++ b/pkg/podexec/pod_command_executor.go @@ -36,6 +36,10 @@ import ( const defaultTimeout = 30 * time.Second +// maxHookTimeout bounds a user-supplied hook timeout, which can come from a pod +// annotation, so a single hook cannot hold up a backup for an unbounded time. +const maxHookTimeout = 4 * time.Hour + // PodCommandExecutor is capable of executing a command in a container in a pod. type PodCommandExecutor interface { // ExecutePodCommand executes a command in a container in a pod. If the command takes longer than @@ -112,9 +116,15 @@ func (e *defaultPodCommandExecutor) ExecutePodCommand(log logrus.FieldLogger, it localHook.OnError = api.HookErrorModeFail } - if localHook.Timeout.Duration == 0 { + // A non-positive timeout is not a valid bound. Timeouts sourced from pod annotations are + // parsed with time.ParseDuration, which accepts negative values, and a negative duration + // would otherwise leave the hook without any timeout at all. + if localHook.Timeout.Duration <= 0 { localHook.Timeout.Duration = defaultTimeout } + if localHook.Timeout.Duration > maxHookTimeout { + localHook.Timeout.Duration = maxHookTimeout + } hookLog := log.WithFields( logrus.Fields{ @@ -158,23 +168,26 @@ func (e *defaultPodCommandExecutor) ExecutePodCommand(log logrus.FieldLogger, it Stderr: &stderr, } - errCh := make(chan error) + // The timeout drives the context so the exec stream is actually canceled, rather than + // being left running on the API server after this function has returned. + ctx, cancel := context.WithTimeout(context.Background(), localHook.Timeout.Duration) + defer cancel() + + // Buffered so the goroutine below can always send its result and exit, even when this + // function has already returned on the timeout path. + errCh := make(chan error, 1) go func() { - err = executor.StreamWithContext(context.Background(), streamOptions) - errCh <- err + streamErr := executor.StreamWithContext(ctx, streamOptions) + // Inspect the local context as soon as the stream returns. Otherwise a stream error + // completed before the deadline could be misclassified if this goroutine sends its + // result before the caller is scheduled to receive it. + errCh <- normalizeExecHookError(streamErr, ctx.Err(), localHook.Timeout.Duration) }() - var timeoutCh <-chan time.Time - if localHook.Timeout.Duration > 0 { - timer := time.NewTimer(localHook.Timeout.Duration) - defer timer.Stop() - timeoutCh = timer.C - } - select { case err = <-errCh: - case <-timeoutCh: + case <-ctx.Done(): return errors.Errorf("timed out after %v", localHook.Timeout.Duration) } @@ -184,6 +197,14 @@ func (e *defaultPodCommandExecutor) ExecutePodCommand(log logrus.FieldLogger, it return err } +func normalizeExecHookError(streamErr, contextErr error, timeout time.Duration) error { + if errors.Is(contextErr, context.DeadlineExceeded) { + return errors.Errorf("timed out after %v", timeout) + } + + return streamErr +} + func ensureContainerExists(pod *corev1api.Pod, container string) error { existsAsMainContainer := slices.ContainsFunc(pod.Spec.Containers, func(c corev1api.Container) bool { return c.Name == container diff --git a/pkg/podexec/pod_command_executor_test.go b/pkg/podexec/pod_command_executor_test.go index 13b00877a..e30911a20 100644 --- a/pkg/podexec/pod_command_executor_test.go +++ b/pkg/podexec/pod_command_executor_test.go @@ -177,6 +177,33 @@ func TestExecutePodCommand(t *testing.T) { hookError: errors.New("hook error"), expectedError: "hook error", }, + { + name: "stream deadline exceeded before local timeout", + command: []string{"some", "command"}, + expectedContainerName: "foo", + expectedErrorMode: v1.HookErrorModeFail, + expectedTimeout: defaultTimeout, + hookError: context.DeadlineExceeded, + expectedError: context.DeadlineExceeded.Error(), + }, + { + // Timeouts from pod annotations go through time.ParseDuration, which accepts + // negative values. Without clamping, the hook would run with no timeout at all. + name: "negative timeout falls back to the default", + command: []string{"some", "command"}, + expectedContainerName: "foo", + expectedErrorMode: v1.HookErrorModeFail, + timeout: -1 * time.Second, + expectedTimeout: 30 * time.Second, + }, + { + name: "timeout above the maximum is capped", + command: []string{"some", "command"}, + expectedContainerName: "foo", + expectedErrorMode: v1.HookErrorModeFail, + timeout: 100000 * time.Hour, + expectedTimeout: maxHookTimeout, + }, } for _, test := range tests { @@ -246,6 +273,54 @@ func TestExecutePodCommand(t *testing.T) { } } +func TestNormalizeExecHookError(t *testing.T) { + hookErr := errors.New("hook error") + tests := []struct { + name string + streamErr error + contextErr error + expectedError string + preserveStreamErr bool + }{ + { + name: "local context deadline exceeded", + streamErr: context.DeadlineExceeded, + contextErr: context.DeadlineExceeded, + expectedError: "timed out after 30s", + }, + { + name: "stream deadline exceeded before local timeout", + streamErr: context.DeadlineExceeded, + expectedError: context.DeadlineExceeded.Error(), + preserveStreamErr: true, + }, + { + name: "ordinary hook error", + streamErr: hookErr, + expectedError: hookErr.Error(), + preserveStreamErr: true, + }, + { + name: "no errors", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := normalizeExecHookError(test.streamErr, test.contextErr, defaultTimeout) + if test.expectedError == "" { + require.NoError(t, err) + return + } + + require.EqualError(t, err, test.expectedError) + if test.preserveStreamErr && err != test.streamErr { + t.Fatalf("expected stream error to be returned unchanged") + } + }) + } +} + func TestEnsureContainerExists(t *testing.T) { pod := &corev1api.Pod{ Spec: corev1api.PodSpec{ diff --git a/pkg/podexec/pod_command_executor_timeout_test.go b/pkg/podexec/pod_command_executor_timeout_test.go new file mode 100644 index 000000000..a79389481 --- /dev/null +++ b/pkg/podexec/pod_command_executor_timeout_test.go @@ -0,0 +1,181 @@ +/* +Copyright 2026 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podexec + +import ( + "context" + "net/url" + "runtime" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/mock" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + + v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerotest "github.com/vmware-tanzu/velero/pkg/test" +) + +const timeoutTestPodJSON = `{ + "metadata": {"namespace": "ns", "name": "pod-1"}, + "spec": {"containers": [{"name": "container-1"}]} +}` + +// contextAwareExecutor returns once its context is canceled, like the SPDY executor does. +type contextAwareExecutor struct { + canceled chan struct{} + canceledOnce bool +} + +func (e *contextAwareExecutor) Stream(options remotecommand.StreamOptions) error { return nil } + +func (e *contextAwareExecutor) StreamWithContext(ctx context.Context, options remotecommand.StreamOptions) error { + <-ctx.Done() + if !e.canceledOnce { + e.canceledOnce = true + close(e.canceled) + } + return ctx.Err() +} + +// contextIgnoringExecutor lets the outer timeout path return before the stream does. +// Once released, the stream goroutine can only exit if its result channel is buffered. +type contextIgnoringExecutor struct { + release <-chan struct{} + returned *sync.WaitGroup +} + +func (e *contextIgnoringExecutor) Stream(options remotecommand.StreamOptions) error { return nil } + +func (e *contextIgnoringExecutor) StreamWithContext(ctx context.Context, options remotecommand.StreamOptions) error { + defer e.returned.Done() + <-e.release + return nil +} + +func newTimeoutTestExecutor(t *testing.T, exec remotecommand.Executor) (*defaultPodCommandExecutor, map[string]any) { + t.Helper() + + clientConfig := &rest.Config{} + poster := &mockPoster{} + podCommandExecutor := NewPodCommandExecutor(clientConfig, poster).(*defaultPodCommandExecutor) + + factory := &mockStreamExecutorFactory{} + podCommandExecutor.streamExecutorFactory = factory + + baseURL, _ := url.Parse("https://some.server") + contentConfig := rest.ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "", Version: "v1"}} + poster.On("Post").Return(rest.NewRequestWithClient(baseURL, "/api/v1", contentConfig, nil)) + factory.On("NewSPDYExecutor", clientConfig, "POST", mock.Anything).Return(exec, nil) + + pod, err := velerotest.GetAsMap(timeoutTestPodJSON) + if err != nil { + t.Fatal(err) + } + + return podCommandExecutor, pod +} + +func timeoutTestHook(timeout time.Duration) *v1.ExecHook { + return &v1.ExecHook{ + Container: "container-1", + Command: []string{"sh", "-c", "sleep 60"}, + Timeout: metav1.Duration{Duration: timeout}, + } +} + +// A hook that times out must have its exec stream canceled, otherwise the command keeps +// running on the API server after ExecutePodCommand has returned. +func TestExecutePodCommandCancelsStreamOnTimeout(t *testing.T) { + exec := &contextAwareExecutor{canceled: make(chan struct{})} + podCommandExecutor, pod := newTimeoutTestExecutor(t, exec) + + err := podCommandExecutor.ExecutePodCommand(velerotest.NewLogger(), pod, "ns", "pod-1", "hookName", timeoutTestHook(100*time.Millisecond)) + if err == nil { + t.Fatal("expected a timeout error") + } + + select { + case <-exec.canceled: + case <-time.After(2 * time.Second): + t.Fatal("stream was not canceled after the hook timed out") + } +} + +// When the stream returns because the context expired, both select cases are ready and one +// is picked at random, so the reported error must not depend on which one wins. +func TestExecutePodCommandTimeoutErrorIsDeterministic(t *testing.T) { + const ( + rounds = 50 + expectedError = "timed out after 1ms" + ) + + messages := map[string]int{} + for range rounds { + exec := &contextAwareExecutor{canceled: make(chan struct{})} + podCommandExecutor, pod := newTimeoutTestExecutor(t, exec) + + err := podCommandExecutor.ExecutePodCommand(velerotest.NewLogger(), pod, "ns", "pod-1", "hookName", timeoutTestHook(time.Millisecond)) + if err == nil { + t.Fatal("expected a timeout error") + } + if err.Error() != expectedError { + t.Fatalf("expected %q, got %q", expectedError, err) + } + messages[err.Error()]++ + } + + if len(messages) != 1 { + t.Fatalf("expected one error message, got %d: %v", len(messages), messages) + } +} + +func TestExecutePodCommandDoesNotLeakOnTimeout(t *testing.T) { + const rounds = 10 + + runtime.GC() + time.Sleep(200 * time.Millisecond) + before := runtime.NumGoroutine() + + release := make(chan struct{}) + returned := &sync.WaitGroup{} + for range rounds { + returned.Add(1) + exec := &contextIgnoringExecutor{release: release, returned: returned} + podCommandExecutor, pod := newTimeoutTestExecutor(t, exec) + + if err := podCommandExecutor.ExecutePodCommand(velerotest.NewLogger(), pod, "ns", "pod-1", "hookName", timeoutTestHook(50*time.Millisecond)); err == nil { + t.Fatal("expected a timeout error") + } + } + + // Every ExecutePodCommand call has already taken the timeout path. Releasing the + // streams now forces their goroutines to send into an errCh with no receiver. + close(release) + returned.Wait() + time.Sleep(200 * time.Millisecond) + runtime.GC() + time.Sleep(200 * time.Millisecond) + + if leaked := runtime.NumGoroutine() - before; leaked >= rounds { + t.Fatalf("%d goroutines leaked over %d timed out hooks", leaked, rounds) + } +} diff --git a/pkg/podvolume/backup_micro_service.go b/pkg/podvolume/backup_micro_service.go index 246221d25..1f71ba0b2 100644 --- a/pkg/podvolume/backup_micro_service.go +++ b/pkg/podvolume/backup_micro_service.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "github.com/vmware-tanzu/velero/internal/credentials" + veleroshared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/repository" @@ -192,10 +193,23 @@ func (r *BackupMicroService) RunCancelableDataPath(ctx context.Context) (string, tags := map[string]string{} + // "none" requests a full backup. "auto" requests that the data mover finds the most + // recent backup of the same volume as parent, which is what an empty ParentSnapshot + // already does, so both map to "". + parentSnapshot := pvb.Spec.ParentSnapshot + forceFull := false + switch pvb.Spec.ParentSnapshot { + case veleroshared.ParentSnapshotNone: + parentSnapshot = "" + forceFull = true + case veleroshared.ParentSnapshotAuto: + parentSnapshot = "" + } + if err := fsBackup.StartBackup(r.sourceTargetPath, pvb.Spec.UploaderSettings, &datapath.BackupStartParam{ RealSource: GetRealSource(pvb), - ParentSnapshot: "", - ForceFull: false, + ParentSnapshot: parentSnapshot, + ForceFull: forceFull, Tags: tags, }); err != nil { return "", errors.Wrap(err, "error starting data path backup") diff --git a/pkg/podvolume/backup_micro_service_test.go b/pkg/podvolume/backup_micro_service_test.go index eac17e4de..b83bafd8a 100644 --- a/pkg/podvolume/backup_micro_service_test.go +++ b/pkg/podvolume/backup_micro_service_test.go @@ -34,6 +34,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/uploader" + veleroshared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -155,7 +156,7 @@ func TestOnDataPathCompleted(t *testing.T) { { name: "marshal fail", marshalErr: errors.New("fake-marshal-error"), - expectedErr: "Failed to marshal backup result { false { } 0 0}: fake-marshal-error", + expectedErr: "Failed to marshal backup result { false { } 0 }: fake-marshal-error", }, { name: "succeed", @@ -446,3 +447,88 @@ func TestRunCancelableDataPath(t *testing.T) { cancel() } + +func TestRunCancelableDataPathParentSnapshot(t *testing.T) { + pvbName := "fake-pvb" + + tests := []struct { + name string + parentSnapshot string + expectedParentSnapshot string + expectedForceFull bool + }{ + { + name: "empty lets the data mover search for a parent", + parentSnapshot: "", + expectedParentSnapshot: "", + expectedForceFull: false, + }, + { + name: "auto lets the data mover search for a parent", + parentSnapshot: veleroshared.ParentSnapshotAuto, + expectedParentSnapshot: "", + expectedForceFull: false, + }, + { + name: "none forces a full backup", + parentSnapshot: veleroshared.ParentSnapshotNone, + expectedParentSnapshot: "", + expectedForceFull: true, + }, + { + name: "a specific snapshot ID is passed through unchanged", + parentSnapshot: "fake-parent-snapshot-id", + expectedParentSnapshot: "fake-parent-snapshot-id", + expectedForceFull: false, + }, + } + + scheme := runtime.NewScheme() + velerov1api.AddToScheme(scheme) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + pvbInProgress := builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, pvbName). + Phase(velerov1api.PodVolumeBackupPhaseInProgress). + Result() + pvbInProgress.Spec.ParentSnapshot = test.parentSnapshot + + fakeClient := clientFake.NewClientBuilder().WithScheme(scheme). + WithRuntimeObjects(pvbInProgress).Build() + + bs := &BackupMicroService{ + namespace: velerov1api.DefaultNamespace, + pvbName: pvbName, + ctx: t.Context(), + client: fakeClient, + dataPathMgr: datapath.NewManager(1), + eventRecorder: &backupMsTestHelper{}, + resultSignal: make(chan dataPathResult), + logger: velerotest.NewLogger(), + } + + var startParam *datapath.BackupStartParam + datapath.VGDPCreator = func(string, string, kbclient.Client, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR { + fsBR := datapathmockes.NewAsyncBR(t) + fsBR.On("Init", mock.Anything, mock.Anything).Return(nil) + fsBR.On("StartBackup", mock.Anything, mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + startParam = args.Get(2).(*datapath.BackupStartParam) + }).Return(nil) + return fsBR + } + + go func() { + time.Sleep(time.Millisecond * 500) + bs.resultSignal <- dataPathResult{result: "fake-succeed-result"} + }() + + _, err := bs.RunCancelableDataPath(t.Context()) + require.NoError(t, err) + + require.NotNil(t, startParam) + assert.Equal(t, test.expectedParentSnapshot, startParam.ParentSnapshot) + assert.Equal(t, test.expectedForceFull, startParam.ForceFull) + }) + } +} diff --git a/pkg/podvolume/backupper.go b/pkg/podvolume/backupper.go index 261f227f8..46ed4defd 100644 --- a/pkg/podvolume/backupper.go +++ b/pkg/podvolume/backupper.go @@ -34,6 +34,7 @@ import ( ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/resourcepolicies" + veleroshared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" veleroclient "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/label" @@ -598,5 +599,9 @@ func newPodVolumeBackup(backup *velerov1api.Backup, pod *corev1api.Pod, volume c pvb.Spec.UploaderSettings = uploaderutil.StoreBackupConfig(backup.Spec.UploaderConfig) } + if backup.Spec.BackupType == velerov1api.BackupTypeFull { + pvb.Spec.ParentSnapshot = veleroshared.ParentSnapshotNone + } + return pvb } diff --git a/pkg/podvolume/backupper_test.go b/pkg/podvolume/backupper_test.go index 59466e02a..92fab63ed 100644 --- a/pkg/podvolume/backupper_test.go +++ b/pkg/podvolume/backupper_test.go @@ -357,7 +357,7 @@ func createPVBObj(fail bool, withSnapshot bool, index int, uploaderType string) } func createNodeObj() *corev1api.Node { - return builder.ForNode("fake-node-name").Labels(map[string]string{"kubernetes.io/os": "linux"}).Result() + return builder.ForNode("fake-node-name").Labels(map[string]string{corev1api.LabelOSStable: "linux"}).Result() } func TestBackupPodVolumes(t *testing.T) { @@ -380,6 +380,8 @@ func TestBackupPodVolumes(t *testing.T) { pvbs int mockGetRepositoryType bool errs []string + expectedBackedup []string + expectedSkipped map[string]string }{ { name: "empty volume list", @@ -573,6 +575,10 @@ func TestBackupPodVolumes(t *testing.T) { uploaderType: "kopia", bsl: "fake-bsl", errs: []string{}, + expectedSkipped: map[string]string{ + "fake-volume-1": "volume fake-volume-1 is declared in pod fake-ns/fake-pod but not mounted by any container, skipping", + "fake-volume-2": "volume fake-volume-2 is declared in pod fake-ns/fake-pod but not mounted by any container, skipping", + }, }, { name: "return completed pvbs", @@ -589,14 +595,14 @@ func TestBackupPodVolumes(t *testing.T) { ctlClientObj: []runtime.Object{ createBackupRepoObj(), }, - runtimeScheme: scheme, - uploaderType: "kopia", - bsl: "fake-bsl", - pvbs: 1, - errs: []string{}, + runtimeScheme: scheme, + uploaderType: "kopia", + bsl: "fake-bsl", + pvbs: 1, + errs: []string{}, + expectedBackedup: []string{"fake-volume-1"}, }, } - // TODO add more verification around PVCBackupSummary returned by "BackupPodVolumes" for _, test := range tests { t.Run(test.name, func(t *testing.T) { ctx := t.Context() @@ -627,7 +633,7 @@ func TestBackupPodVolumes(t *testing.T) { funcGetRepositoryType = getRepositoryType } - pvbs, _, errs := bp.BackupPodVolumes(backupObj, test.sourcePod, test.volumes, nil, velerotest.NewLogger()) + pvbs, summary, errs := bp.BackupPodVolumes(backupObj, test.sourcePod, test.volumes, nil, velerotest.NewLogger()) if test.errs != nil { for i := 0; i < len(errs); i++ { @@ -636,6 +642,22 @@ func TestBackupPodVolumes(t *testing.T) { } assert.Len(t, pvbs, test.pvbs) + + if summary != nil { + assert.Len(t, summary.Backedup, len(test.expectedBackedup)) + for _, vol := range test.expectedBackedup { + assert.Contains(t, summary.Backedup, vol) + } + + assert.Len(t, summary.Skipped, len(test.expectedSkipped)) + for vol, reason := range test.expectedSkipped { + require.Contains(t, summary.Skipped, vol) + assert.Equal(t, reason, summary.Skipped[vol].Reason) + } + } else { + assert.Empty(t, test.expectedBackedup) + assert.Empty(t, test.expectedSkipped) + } }) } } diff --git a/pkg/podvolume/restore_micro_service.go b/pkg/podvolume/restore_micro_service.go index 24f001147..2f778a3f9 100644 --- a/pkg/podvolume/restore_micro_service.go +++ b/pkg/podvolume/restore_micro_service.go @@ -184,7 +184,9 @@ func (r *RestoreMicroService) RunCancelableDataPath(ctx context.Context) (string log.Info("Async fs br init") - if err := fsRestore.StartRestore(pvr.Spec.SnapshotID, r.sourceTargetPath, pvr.Spec.UploaderSettings); err != nil { + if err := fsRestore.StartRestore(pvr.Spec.SnapshotID, r.sourceTargetPath, pvr.Spec.UploaderSettings, &datapath.RestoreStartParam{ + Incremental: pvr.Spec.RestoreType == string(velerov1api.VolumeDataPolicyTypeIncremental), + }); err != nil { return "", errors.Wrap(err, "error starting data path restore") } diff --git a/pkg/podvolume/restore_micro_service_test.go b/pkg/podvolume/restore_micro_service_test.go index 007060160..1964d5035 100644 --- a/pkg/podvolume/restore_micro_service_test.go +++ b/pkg/podvolume/restore_micro_service_test.go @@ -436,12 +436,12 @@ func TestRunCancelableDataPathRestore(t *testing.T) { if test.startErr != nil { fsBR.On("Init", mock.Anything, mock.Anything).Return(nil) - fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(test.startErr) + fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.startErr) } if test.dataPathStarted { fsBR.On("Init", mock.Anything, mock.Anything).Return(nil) - fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(nil) + fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) } return fsBR diff --git a/pkg/podvolume/restorer.go b/pkg/podvolume/restorer.go index bac22298e..53d35215c 100644 --- a/pkg/podvolume/restorer.go +++ b/pkg/podvolume/restorer.go @@ -63,10 +63,9 @@ type restorer struct { kubeClient kubernetes.Interface crClient ctrlclient.Client - resultsLock sync.Mutex - results map[string]chan *velerov1api.PodVolumeRestore - nodeAgentCheck chan error - log logrus.FieldLogger + resultsLock sync.Mutex + results map[string]chan *velerov1api.PodVolumeRestore + log logrus.FieldLogger } func newRestorer( @@ -106,9 +105,9 @@ func newRestorer( if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseCompleted || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseFailed || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseCanceled { r.resultsLock.Lock() - defer r.resultsLock.Unlock() - resChan, ok := r.results[resultsKey(pvr.Spec.Pod.Namespace, pvr.Spec.Pod.Name)] + r.resultsLock.Unlock() + if !ok { log.Errorf("No results channel found for pod %s/%s to send pod volume restore %s/%s on", pvr.Spec.Pod.Namespace, pvr.Spec.Pod.Name, pvr.Namespace, pvr.Name) return @@ -147,13 +146,13 @@ func (r *restorer) RestorePodVolumes(data RestoreData, tracker *volume.RestoreVo r.repoLocker.Lock(repo.Name) defer r.repoLocker.Unlock(repo.Name) - resultsChan := make(chan *velerov1api.PodVolumeRestore) + resultsChan := make(chan *velerov1api.PodVolumeRestore, len(volumesToRestore)) r.resultsLock.Lock() r.results[resultsKey(data.Pod.Namespace, data.Pod.Name)] = resultsChan r.resultsLock.Unlock() - r.nodeAgentCheck = make(chan error) + nodeAgentCheck := make(chan error) var ( errs []error @@ -217,7 +216,7 @@ func (r *restorer) RestorePodVolumes(data RestoreData, tracker *volume.RestoreVo err = nodeagent.IsRunningInNode(checkCtx, data.Restore.Namespace, nodeName, r.crClient) if err != nil { r.log.WithField("node", nodeName).WithError(err).Error("node-agent pod is not running in node, abort the restore") - r.nodeAgentCheck <- errors.Wrapf(err, "node-agent pod is not running in node %s", nodeName) + nodeAgentCheck <- errors.Wrapf(err, "node-agent pod is not running in node %s", nodeName) } } }() @@ -235,7 +234,7 @@ ForEachVolume: errs = append(errs, errors.Errorf("pod volume restore canceled: %s", res.Status.Message)) } tracker.TrackPodVolume(res) - case err := <-r.nodeAgentCheck: + case err := <-nodeAgentCheck: errs = append(errs, err) break ForEachVolume } @@ -298,6 +297,10 @@ func newPodVolumeRestore(restore *velerov1api.Restore, pod *corev1api.Pod, backu pvr.Spec.UploaderSettings = uploaderutil.StoreRestoreConfig(restore.Spec.UploaderConfig) } + if restore.IsVolumeDataInplaceRestore() { + pvr.Spec.RestoreType = string(restore.Spec.ExistingVolumeDataPolicy) + } + return pvr } diff --git a/pkg/repository/maintenance/maintenance.go b/pkg/repository/maintenance/maintenance.go index 33c3fb1f8..86525d54f 100644 --- a/pkg/repository/maintenance/maintenance.go +++ b/pkg/repository/maintenance/maintenance.go @@ -351,7 +351,7 @@ func WaitJobComplete(cli client.Client, ctx context.Context, jobName, ns string, if maintenanceJob.Status.Failed > 0 { if r, err := getResultFromJob(cli, maintenanceJob); err != nil { log.WithError(err).Warn("Failed to get maintenance job result") - result = "Repo maintenance failed but result is not retrieveable" + result = "Repo maintenance failed but result is not retrievable" } else { result = r } @@ -414,7 +414,7 @@ func WaitAllJobsComplete(ctx context.Context, cli client.Client, repo *velerov1a if job.Status.Failed > 0 { if msg, err := getResultFromJob(cli, job); err != nil { log.WithError(err).Warnf("Failed to get result of maintenance job %s", job.Name) - message = fmt.Sprintf("Repo maintenance failed but result is not retrieveable, err: %v", err) + message = fmt.Sprintf("Repo maintenance failed but result is not retrievable, err: %v", err) } else { message = msg } diff --git a/pkg/repository/maintenance/maintenance_test.go b/pkg/repository/maintenance/maintenance_test.go index 05fce89e9..ee34241ce 100644 --- a/pkg/repository/maintenance/maintenance_test.go +++ b/pkg/repository/maintenance/maintenance_test.go @@ -789,7 +789,7 @@ func TestWaitAllJobsComplete(t *testing.T) { { Result: velerov1api.BackupRepositoryMaintenanceFailed, StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, - Message: "Repo maintenance failed but result is not retrieveable, err: no pod found for job job2", + Message: "Repo maintenance failed but result is not retrievable, err: no pod found for job job2", }, }, }, diff --git a/pkg/repository/manager/manager.go b/pkg/repository/manager/manager.go index d34c97624..f8b10db5e 100644 --- a/pkg/repository/manager/manager.go +++ b/pkg/repository/manager/manager.go @@ -18,6 +18,7 @@ package repository import ( "context" + "crypto/fips140" "fmt" "time" @@ -173,7 +174,13 @@ func (m *manager) PrepareRepo(repo *velerov1api.BackupRepository) error { if err != nil { return errors.WithStack(err) } - return prd.PrepareRepo(context.Background(), param) + + // Disable FIPS-140 compliance check, because Kopia doesn't support FIPS-140 yet. + var prepareErr error + fips140.WithoutEnforcement(func() { + prepareErr = prd.PrepareRepo(context.Background(), param) + }) + return prepareErr } func (m *manager) PruneRepo(repo *velerov1api.BackupRepository) error { @@ -244,11 +251,20 @@ func (m *manager) BatchForget(ctx context.Context, repo *velerov1api.BackupRepos return []error{errors.WithStack(err)} } - if err := prd.BoostRepoConnect(context.Background(), param); err != nil { + // Disable FIPS-140 compliance check, because Kopia doesn't support FIPS-140 yet. + var connectErr error + fips140.WithoutEnforcement(func() { + connectErr = prd.BoostRepoConnect(context.Background(), param) + }) + if connectErr != nil { return []error{errors.WithStack(err)} } - return prd.BatchForget(context.Background(), snapshots, param) + forgetErr := make([]error, 0) + fips140.WithoutEnforcement(func() { + forgetErr = prd.BatchForget(context.Background(), snapshots, param) + }) + return forgetErr } func (m *manager) DefaultMaintenanceFrequency(repo *velerov1api.BackupRepository) (time.Duration, error) { diff --git a/pkg/repository/provider/unified_repo.go b/pkg/repository/provider/unified_repo.go index bfe1a2bd9..24c744fe4 100644 --- a/pkg/repository/provider/unified_repo.go +++ b/pkg/repository/provider/unified_repo.go @@ -384,7 +384,7 @@ func (urp *unifiedRepoProvider) BatchForget(ctx context.Context, snapshotIDs []s err = bkRepo.Flush(ctx) if err != nil { - return []error{errors.Wrap(err, "error to flush repo")} + return append(errs, errors.Wrap(err, "error to flush repo")) } log.Debug("Forget snapshot complete") @@ -501,11 +501,16 @@ func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, cr return map[string]string{}, errors.New("invalid storage provider") } - config := backupLocation.Spec.Config - if config == nil { - config = map[string]string{} + config := make(map[string]string) + if backupLocation.Spec.Config != nil { + for k, v := range backupLocation.Spec.Config { + config[k] = v + } } + // Delete any user-provided credentialsFile to prevent path traversal vulnerabilities + delete(config, repoconfig.CredentialsFileKey) + if backupLocation.Spec.Credential != nil { config[repoconfig.CredentialsFileKey], err = credentialsFileStore.Path(backupLocation.Spec.Credential) if err != nil { @@ -549,11 +554,16 @@ func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repo return map[string]string{}, errors.New("invalid storage provider") } - config := backupLocation.Spec.Config - if config == nil { - config = map[string]string{} + config := make(map[string]string) + if backupLocation.Spec.Config != nil { + for k, v := range backupLocation.Spec.Config { + config[k] = v + } } + // Delete any user-provided credentialsFile to prevent path traversal vulnerabilities + delete(config, repoconfig.CredentialsFileKey) + bucket := strings.Trim(config["bucket"], "/") prefix := strings.Trim(config["prefix"], "/") if backupLocation.Spec.ObjectStorage != nil { diff --git a/pkg/repository/provider/unified_repo_test.go b/pkg/repository/provider/unified_repo_test.go index e0e0a8b8f..d2131d6bc 100644 --- a/pkg/repository/provider/unified_repo_test.go +++ b/pkg/repository/provider/unified_repo_test.go @@ -85,7 +85,7 @@ func TestGetStorageCredentials(t *testing.T) { Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/aws", Config: map[string]string{ - "credentialsFile": "credentials-from-config-map", + "credentialsFile": "credentials-from-config-map", // This should be ignored }, }, }, @@ -96,7 +96,7 @@ func TestGetStorageCredentials(t *testing.T) { }, credFileStore: new(credmock.FileStore), expected: map[string]string{ - "accessKeyID": "from: credentials-from-config-map", + "accessKeyID": "from: ", "providerName": "", "secretAccessKey": "", "sessionToken": "", @@ -108,7 +108,7 @@ func TestGetStorageCredentials(t *testing.T) { Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/aws", Config: map[string]string{ - "credentialsFile": "credentials-from-config-map", + "credentialsFile": "credentials-from-config-map", // This should be ignored }, Credential: &corev1api.SecretKeySelector{}, }, @@ -134,7 +134,7 @@ func TestGetStorageCredentials(t *testing.T) { Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/aws", Config: map[string]string{ - "credentialsFile": "credentials-from-config-map", + "credentialsFile": "credentials-from-config-map", // This should be ignored }, }, }, @@ -176,16 +176,16 @@ func TestGetStorageCredentials(t *testing.T) { Spec: velerov1api.BackupStorageLocationSpec{ Provider: "velero.io/gcp", Config: map[string]string{ - "credentialsFile": "credentials-from-config-map", + "credentialsFile": "credentials-from-config-map", // This should be ignored }, }, }, getGCPCredentials: func(config map[string]string) string { - return "credentials-from-config-map" + return config["credentialsFile"] }, credFileStore: new(credmock.FileStore), expected: map[string]string{ - "credFile": "credentials-from-config-map", + "credFile": "", }, }, } @@ -1062,6 +1062,41 @@ func TestBatchForget(t *testing.T) { }, expectedErr: []string{"error to flush repo: fake-error-4"}, }, + { + name: "delete and flush fail", + getter: new(credmock.SecretStore), + credStoreReturn: "fake-password", + funcTable: localFuncTable{ + getStorageVariables: func(*velerov1api.BackupStorageLocation, string, string, map[string]string, velerocredentials.CredentialGetter) (map[string]string, error) { + return map[string]string{}, nil + }, + getStorageCredentials: func(*velerov1api.BackupStorageLocation, velerocredentials.FileStore) (map[string]string, error) { + return map[string]string{}, nil + }, + }, + repoService: new(reposervicenmocks.BackupRepoService), + backupRepo: new(reposervicenmocks.BackupRepo), + retFuncOpen: []any{ + func(context.Context, udmrepo.RepoOptions) udmrepo.BackupRepo { + return backupRepo + }, + + func(context.Context, udmrepo.RepoOptions) error { + return nil + }, + }, + retFuncDelete: func(context.Context, udmrepo.ID) error { + return errors.New("fake-delete-error") + }, + retFuncFlush: func(context.Context) error { + return errors.New("fake-flush-error") + }, + snapshots: []string{"snapshot-1"}, + expectedErr: []string{ + "error to delete manifest snapshot-1: fake-delete-error", + "error to flush repo: fake-flush-error", + }, + }, } for _, tc := range testCases { diff --git a/pkg/repository/udmrepo/kopialib/lib_repo.go b/pkg/repository/udmrepo/kopialib/lib_repo.go index 151bf1cb2..e60128358 100644 --- a/pkg/repository/udmrepo/kopialib/lib_repo.go +++ b/pkg/repository/udmrepo/kopialib/lib_repo.go @@ -19,7 +19,6 @@ package kopialib import ( "context" "encoding/json" - "fmt" "io" "os" "strings" @@ -74,8 +73,22 @@ type logThrottle struct { interval time.Duration } +type objectPrefetch struct { + ctx context.Context + cancel context.CancelFunc + entries []object.IndirectObjectEntry + curOffset int64 + cond *sync.Cond + mu sync.Mutex + nextEntry int + budget int64 +} + type kopiaObjectReader struct { rawReader object.Reader + rawRepo repo.Repository + prefetch *objectPrefetch + logger logrus.FieldLogger } type kopiaObjectWriter struct { @@ -339,7 +352,7 @@ func (km *kopiaMaintenance) maintainProgress(uploaded int64) { } } -func (kr *kopiaRepository) OpenObject(ctx context.Context, id udmrepo.ID) (udmrepo.ObjectReader, error) { +func (kr *kopiaRepository) OpenObject(ctx context.Context, id udmrepo.ID, opt udmrepo.ObjectReadOptions) (udmrepo.ObjectReader, error) { if kr.rawRepo == nil { return nil, errors.New("repo is closed or not open") } @@ -354,9 +367,42 @@ func (kr *kopiaRepository) OpenObject(ctx context.Context, id udmrepo.ID) (udmre return nil, errors.Wrap(err, "error to open object") } - return &kopiaObjectReader{ + var prefetch *objectPrefetch + if opt.Prefetch { + if e, err := kr.getFlattenedEntries(ctx, objID); err != nil { + kr.logger.WithError(err).Warnf("Failed to load entries for object %v, skip prefetch", id) + } else { + pCtx, pCancel := context.WithCancel(ctx) + prefetch = &objectPrefetch{ + ctx: pCtx, + cancel: pCancel, + budget: int64(opt.PrefetchBudgetMB) << 20, + entries: e, + } + + prefetch.cond = sync.NewCond(&prefetch.mu) + } + } + + rd := &kopiaObjectReader{ rawReader: reader, - }, nil + rawRepo: kr.rawRepo, + prefetch: prefetch, + logger: kr.logger, + } + + if rd.prefetch != nil { + go rd.prefetchProc() + + go func() { + <-rd.prefetch.ctx.Done() + prefetch.mu.Lock() + prefetch.cond.Broadcast() + prefetch.mu.Unlock() + }() + } + + return rd, nil } func (kr *kopiaRepository) GetManifest(ctx context.Context, id udmrepo.ID, mani *udmrepo.RepoManifest) error { @@ -551,7 +597,7 @@ func (kr *kopiaRepository) WriteMetadata(ctx context.Context, meta *udmrepo.Meta } func (kr *kopiaRepository) ReadMetadata(ctx context.Context, id udmrepo.ID) (*udmrepo.Metadata, error) { - reader, err := kr.OpenObject(ctx, id) + reader, err := kr.OpenObject(ctx, id, udmrepo.ObjectReadOptions{}) if err != nil { return nil, errors.Wrapf(err, "error to open metadata object %v", id) } @@ -793,7 +839,16 @@ func (kor *kopiaObjectReader) Read(p []byte) (int, error) { return 0, errors.New("object reader is closed or not open") } - return kor.rawReader.Read(p) + n, err := kor.rawReader.Read(p) + if n > 0 { + if kor.prefetch != nil { + kor.prefetch.mu.Lock() + kor.prefetch.curOffset += int64(n) + kor.prefetch.cond.Signal() + kor.prefetch.mu.Unlock() + } + } + return n, err } func (kor *kopiaObjectReader) Seek(offset int64, whence int) (int64, error) { @@ -801,10 +856,83 @@ func (kor *kopiaObjectReader) Seek(offset int64, whence int) (int64, error) { return -1, errors.New("object reader is closed or not open") } - return kor.rawReader.Seek(offset, whence) + off, err := kor.rawReader.Seek(offset, whence) + if err == nil { + if kor.prefetch != nil { + kor.prefetch.mu.Lock() + kor.prefetch.curOffset = off + kor.prefetch.cond.Signal() + kor.prefetch.mu.Unlock() + } + } + + return off, err +} + +func (kor *kopiaObjectReader) prefetchProc() { + prefetch := kor.prefetch + if prefetch == nil { + return + } + + for { + prefetch.mu.Lock() + + select { + case <-prefetch.ctx.Done(): + prefetch.mu.Unlock() + return + default: + } + + curOffset := prefetch.curOffset + + for prefetch.nextEntry < len(prefetch.entries) { + entry := prefetch.entries[prefetch.nextEntry] + if entry.Start+entry.Length <= curOffset { + prefetch.nextEntry++ + } else { + break + } + } + + if prefetch.nextEntry >= len(prefetch.entries) { + prefetch.mu.Unlock() + return + } + + var toFetch []object.ID + for prefetch.nextEntry < len(prefetch.entries) { + entry := prefetch.entries[prefetch.nextEntry] + + if entry.Start > curOffset+prefetch.budget { + break + } + + toFetch = append(toFetch, entry.Object) + prefetch.nextEntry++ + } + + if len(toFetch) == 0 { + prefetch.cond.Wait() + prefetch.mu.Unlock() + continue + } + + prefetch.mu.Unlock() + + _, err := kor.rawRepo.PrefetchObjects(prefetch.ctx, toFetch, "") + if err != nil && err != context.Canceled { + kor.logger.WithError(err).Warnf("Failed to prefetch contents for offset %v", curOffset) + } + } } func (kor *kopiaObjectReader) Close() error { + if kor.prefetch != nil && kor.prefetch.cancel != nil { + kor.prefetch.cancel() + } + if kor.rawReader == nil { return nil } @@ -913,8 +1041,7 @@ func (kow *kopiaObjectWriterEx) Write(p []byte) (int, error) { kow.entryLock.Unlock() buffOffset := curPos - offset - objName := fmt.Sprintf("%s-b%v", kow.description, entryID) - kow.writeObjectAsync(objName, entryID, p[buffOffset:buffOffset+kow.blockSize]) + kow.writeObjectAsync(entryID, p[buffOffset:buffOffset+kow.blockSize]) curPos += kow.blockSize } @@ -922,38 +1049,38 @@ func (kow *kopiaObjectWriterEx) Write(p []byte) (int, error) { return length, nil } -func (kow *kopiaObjectWriterEx) writeObject(objName string, p []byte) (object.ID, error) { +func (kow *kopiaObjectWriterEx) writeObject(p []byte) (object.ID, error) { writer := kow.rawRepoWriter.NewObjectWriter(kopia.SetupKopiaLog(kow.ctx, kow.logger), object.WriterOptions{ - Description: objName, + Description: kow.description, Compressor: kow.compressor, Splitter: kow.splitter, }) if writer == nil { - return object.EmptyID, errors.Errorf("error opening writer for %s", objName) + return object.EmptyID, errors.New("error opening writer") } defer writer.Close() written, err := writer.Write(p) if err != nil { - return object.EmptyID, errors.Wrapf(err, "error writing for %s", objName) + return object.EmptyID, errors.Wrap(err, "error writing data") } if written != len(p) { - return object.EmptyID, errors.Errorf("short write for %s", objName) + return object.EmptyID, errors.New("short write") } objID, err := writer.Result() if err != nil { - return object.EmptyID, errors.Wrapf(err, "error flushing data for %s", objName) + return object.EmptyID, errors.Wrap(err, "error flushing data") } return objID, nil } -func (kow *kopiaObjectWriterEx) writeObjectSync(objName string, entry int, p []byte) error { - objID, err := kow.writeObject(objName, p) +func (kow *kopiaObjectWriterEx) writeObjectSync(entry int, p []byte) error { + objID, err := kow.writeObject(p) if err != nil { return err } @@ -965,10 +1092,10 @@ func (kow *kopiaObjectWriterEx) writeObjectSync(objName string, entry int, p []b return nil } -func (kow *kopiaObjectWriterEx) writeObjectAsync(objName string, entryID int, p []byte) { +func (kow *kopiaObjectWriterEx) writeObjectAsync(entryID int, p []byte) { if kow.asyncWritesSem == nil { - if err := kow.writeObjectSync(objName, entryID, p); err != nil { - kow.saveWriteError(errors.Wrapf(err, "error writing object for %s", objName)) + if err := kow.writeObjectSync(entryID, p); err != nil { + kow.saveWriteError(errors.Wrapf(err, "error writing object for %s, entry %d", kow.description, entryID)) } } else { kow.asyncWritesSem <- struct{}{} @@ -977,8 +1104,8 @@ func (kow *kopiaObjectWriterEx) writeObjectAsync(objName string, entryID int, p copy(buffer, p) kow.asyncWritesGroup.Go(func() { - if err := kow.writeObjectSync(objName, entryID, buffer); err != nil { - kow.saveWriteError(errors.Wrapf(err, "error writing object for %s", objName)) + if err := kow.writeObjectSync(entryID, buffer); err != nil { + kow.saveWriteError(errors.Wrapf(err, "error writing object for %s, entry %d", kow.description, entryID)) } kow.asyncBuffer.Return(buffer) @@ -987,10 +1114,10 @@ func (kow *kopiaObjectWriterEx) writeObjectAsync(objName string, entryID int, p } } -func (kow *kopiaObjectWriterEx) writeZeroObject(objName string, entryID int) error { +func (kow *kopiaObjectWriterEx) writeZeroObject(entryID int) error { if kow.zeroObject == object.EmptyID { zeroBuffer := make([]byte, kow.blockSize) - objectID, err := kow.writeObject(objName, zeroBuffer) + objectID, err := kow.writeObject(zeroBuffer) if err != nil { return err } @@ -1071,9 +1198,8 @@ func (kow *kopiaObjectWriterEx) WriteAt(p []byte, offset int64) (int, error) { }) kow.entryLock.Unlock() - objName := fmt.Sprintf("%s-b%v", kow.description, entryID) - if err := kow.writeZeroObject(objName, entryID); err != nil { - return 0, errors.Wrapf(err, "error writing zero object for %s", objName) + if err := kow.writeZeroObject(entryID); err != nil { + return 0, errors.Wrapf(err, "error writing zero object for %s, entry %v", kow.description, entryID) } curPos += kow.blockSize @@ -1093,8 +1219,7 @@ func (kow *kopiaObjectWriterEx) WriteAt(p []byte, offset int64) (int, error) { kow.entryLock.Unlock() buffOffset := curPos - offset - objName := fmt.Sprintf("%s-b%v", kow.description, entryID) - kow.writeObjectAsync(objName, entryID, p[buffOffset:buffOffset+kow.blockSize]) + kow.writeObjectAsync(entryID, p[buffOffset:buffOffset+kow.blockSize]) curPos += kow.blockSize } diff --git a/pkg/repository/udmrepo/kopialib/lib_repo_ex_test.go b/pkg/repository/udmrepo/kopialib/lib_repo_ex_test.go index 3294063a6..c9b383ea2 100644 --- a/pkg/repository/udmrepo/kopialib/lib_repo_ex_test.go +++ b/pkg/repository/udmrepo/kopialib/lib_repo_ex_test.go @@ -291,7 +291,7 @@ func TestKopiaObjectWriterEx_Write(t *testing.T) { t.Helper() err := kow.getWriteError() require.Error(t, err) - assert.Contains(t, err.Error(), "error opening writer for -b0") + assert.Contains(t, err.Error(), "error writing object for , entry 0: error opening writer") }, }, { @@ -936,7 +936,7 @@ func TestKopiaObjectWriterEx_WriteAt(t *testing.T) { }, inputData: make([]byte, 1024), offset: 1024, - expectedErr: "error writing zero object for -b0: error writing for -b0: simulated zero object write error", + expectedErr: "error writing zero object for , entry 0: error writing data: simulated zero object write error", }, { name: "writeObject short write", @@ -964,7 +964,7 @@ func TestKopiaObjectWriterEx_WriteAt(t *testing.T) { t.Helper() err := kow.getWriteError() require.Error(t, err) - assert.Contains(t, err.Error(), "short write for -b0") + assert.Contains(t, err.Error(), "error writing object for , entry 0: short write") }, }, } @@ -1208,6 +1208,10 @@ func TestKopiaObjectWriterEx_MixedWriteAndWriteAt(t *testing.T) { assert.Equal(t, int64(3072), kow.entries[3].Start) } +// TestKopiaObjectWriterEx_ConcurrentAsyncErrors verifies the async error contract +// under real scheduling: once an async block write fails, the error either fails a +// subsequent Write call fast or surfaces at Result — it is never lost. Which of the +// two happens first depends on goroutine scheduling, and both are correct. func TestKopiaObjectWriterEx_ConcurrentAsyncErrors(t *testing.T) { mockRepoWriter := repomocks.NewMockRepositoryWriter(t) mockWriter := repomocks.NewWriter(t) @@ -1231,14 +1235,65 @@ func TestKopiaObjectWriterEx_ConcurrentAsyncErrors(t *testing.T) { data := make([]byte, 1024) - // Issue multiple writes so they all spawn async goroutines - // First few writes shouldn't fail immediately until getWriteError catches the asynchronous fault + // Issue multiple writes so they all spawn async goroutines. A later Write may + // observe the stored async error and fail fast — that is correct behavior. + for i := 0; i < 10; i++ { + l, err := kow.Write(data) + if err != nil { + assert.Contains(t, err.Error(), "simulated async error") + break + } + assert.Equal(t, 1024, l) + } + + // Regardless of whether a Write observed the error first, Result must report it. + id, err := kow.Result() + + require.Error(t, err) + assert.Contains(t, err.Error(), "simulated async error") + assert.Equal(t, udmrepo.ID(""), id) +} + +// TestKopiaObjectWriterEx_AsyncErrorSurfacesAtResult pins the late-error schedule: +// async writes are held until all writes have been queued, so no Write call observes +// the failure and Result alone must report it. +func TestKopiaObjectWriterEx_AsyncErrorSurfacesAtResult(t *testing.T) { + mockRepoWriter := repomocks.NewMockRepositoryWriter(t) + mockWriter := repomocks.NewWriter(t) + + releaseWrites := make(chan struct{}) + mockWriter.On("Write", mock.Anything).Run(func(mock.Arguments) { + <-releaseWrites + }).Return(0, errors.New("simulated async error")) + mockWriter.On("Close").Return(nil) + + mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter) + + sem := make(chan struct{}, 10) + buf := freelist.New(10*1024, 1024) + + kow := &kopiaObjectWriterEx{ + ctx: context.Background(), + rawRepoWriter: mockRepoWriter, + blockSize: 1024, + asyncWritesSem: sem, + asyncBuffer: buf, + logger: velerotest.NewLogger(), + } + + data := make([]byte, 1024) + + // All async writes block on releaseWrites, so no error can be stored yet and + // every Write must succeed. for i := 0; i < 10; i++ { l, err := kow.Write(data) require.NoError(t, err) assert.Equal(t, 1024, l) } + close(releaseWrites) + + // Result waits for the async writers to finish and must report their error. id, err := kow.Result() require.Error(t, err) diff --git a/pkg/repository/udmrepo/kopialib/lib_repo_test.go b/pkg/repository/udmrepo/kopialib/lib_repo_test.go index 370b82b9e..b4d487c43 100644 --- a/pkg/repository/udmrepo/kopialib/lib_repo_test.go +++ b/pkg/repository/udmrepo/kopialib/lib_repo_test.go @@ -22,12 +22,14 @@ import ( "encoding/json" "math" "os" + "sync" "testing" "time" "github.com/cockroachdb/errors" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/repo/object" "github.com/kopia/kopia/snapshot" @@ -285,6 +287,7 @@ func TestOpenObject(t *testing.T) { name string rawRepo *repomocks.MockRepository objectID string + opt udmrepo.ObjectReadOptions retErr error expectedErr string }{ @@ -304,21 +307,38 @@ func TestOpenObject(t *testing.T) { retErr: errors.New("fake-open-error"), expectedErr: "error to open object: fake-open-error", }, + { + name: "raw open success, without prefetch", + rawRepo: repomocks.NewMockRepository(t), + objectID: "D0123456789abcdef0123456789abcdef", + }, + { + name: "raw open success, with prefetch", + rawRepo: repomocks.NewMockRepository(t), + objectID: "D0123456789abcdef0123456789abcdef", + opt: udmrepo.ObjectReadOptions{Prefetch: true, PrefetchBudgetMB: 10}, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - kr := &kopiaRepository{} + kr := &kopiaRepository{ + logger: velerotest.NewLogger(), + } if tc.rawRepo != nil { - if tc.retErr != nil { - tc.rawRepo.On("OpenObject", mock.Anything, mock.Anything).Return(nil, tc.retErr) + if tc.name != "objectID is invalid" { + if tc.retErr != nil { + tc.rawRepo.On("OpenObject", mock.Anything, mock.Anything).Return(nil, tc.retErr) + } else { + tc.rawRepo.On("OpenObject", mock.Anything, mock.Anything).Return(nil, nil) + } } kr.rawRepo = tc.rawRepo } - _, err := kr.OpenObject(t.Context(), udmrepo.ID(tc.objectID)) + _, err := kr.OpenObject(t.Context(), udmrepo.ID(tc.objectID), tc.opt) if tc.expectedErr == "" { assert.NoError(t, err) @@ -845,6 +865,7 @@ func TestReaderClose(t *testing.T) { name string rawObjReader *repomocks.Reader rawReaderRetErr error + withPrefetch bool expectedErr string }{ { @@ -860,6 +881,11 @@ func TestReaderClose(t *testing.T) { name: "succeed", rawObjReader: repomocks.NewReader(t), }, + { + name: "succeed with prefetch", + rawObjReader: repomocks.NewReader(t), + withPrefetch: true, + }, } for _, tc := range testCases { @@ -871,8 +897,20 @@ func TestReaderClose(t *testing.T) { kr.rawReader = tc.rawObjReader } + if tc.withPrefetch { + ctx, cancel := context.WithCancel(t.Context()) + kr.prefetch = &objectPrefetch{ + ctx: ctx, + cancel: cancel, + } + } + err := kr.Close() + if tc.withPrefetch { + require.ErrorIs(t, kr.prefetch.ctx.Err(), context.Canceled) + } + if tc.expectedErr == "" { assert.NoError(t, err) } else { @@ -1832,3 +1870,173 @@ func TestListSnapshot(t *testing.T) { }) } } + +func mustParseID(s string) object.ID { + id, _ := object.ParseID(s) + return id +} + +func TestPrefetchProc(t *testing.T) { + testCases := []struct { + name string + setupPrefetch func(ctx context.Context, cancel context.CancelFunc) *objectPrefetch + mockRepo func(mockRepo *repomocks.MockRepository) + runConcurrently bool + trigger func(prefetch *objectPrefetch) + }{ + { + name: "nil prefetch", + setupPrefetch: func(ctx context.Context, cancel context.CancelFunc) *objectPrefetch { + return nil + }, + }, + { + name: "context canceled", + setupPrefetch: func(ctx context.Context, cancel context.CancelFunc) *objectPrefetch { + cancel() + p := &objectPrefetch{ + ctx: ctx, + } + p.cond = sync.NewCond(&p.mu) + return p + }, + }, + { + name: "fetch all entries and exit", + setupPrefetch: func(ctx context.Context, cancel context.CancelFunc) *objectPrefetch { + p := &objectPrefetch{ + ctx: ctx, + entries: []object.IndirectObjectEntry{ + {Start: 0, Length: 100, Object: mustParseID("D0123456789abcdef0123456789abcdef")}, + {Start: 100, Length: 100, Object: mustParseID("D0123456789abcdef0123456789abcdeg")}, + }, + budget: 200, + curOffset: 0, + } + p.cond = sync.NewCond(&p.mu) + return p + }, + mockRepo: func(mockRepo *repomocks.MockRepository) { + mockRepo.On("PrefetchObjects", mock.Anything, []object.ID{mustParseID("D0123456789abcdef0123456789abcdef"), mustParseID("D0123456789abcdef0123456789abcdeg")}, "").Return(([]content.ID)(nil), nil).Once() + }, + }, + { + name: "fetch partial, wait, and fetch rest", + runConcurrently: true, + setupPrefetch: func(ctx context.Context, cancel context.CancelFunc) *objectPrefetch { + p := &objectPrefetch{ + ctx: ctx, + entries: []object.IndirectObjectEntry{ + {Start: 0, Length: 100, Object: mustParseID("D0123456789abcdef0123456789abcdef")}, + {Start: 100, Length: 100, Object: mustParseID("D0123456789abcdef0123456789abcdeg")}, + }, + budget: 50, + curOffset: 0, + } + p.cond = sync.NewCond(&p.mu) + return p + }, + mockRepo: func(mockRepo *repomocks.MockRepository) { + mockRepo.On("PrefetchObjects", mock.Anything, []object.ID{mustParseID("D0123456789abcdef0123456789abcdef")}, "").Return(([]content.ID)(nil), nil).Once() + mockRepo.On("PrefetchObjects", mock.Anything, []object.ID{mustParseID("D0123456789abcdef0123456789abcdeg")}, "").Return(([]content.ID)(nil), nil).Once() + }, + trigger: func(prefetch *objectPrefetch) { + // Wait a bit for the first fetch and wait to happen + time.Sleep(50 * time.Millisecond) + prefetch.mu.Lock() + prefetch.curOffset = 100 + prefetch.cond.Signal() + prefetch.mu.Unlock() + }, + }, + { + name: "cancel while waiting on cond", + runConcurrently: true, + setupPrefetch: func(ctx context.Context, cancel context.CancelFunc) *objectPrefetch { + p := &objectPrefetch{ + ctx: ctx, + cancel: cancel, + entries: []object.IndirectObjectEntry{ + {Start: 0, Length: 100, Object: mustParseID("D0123456789abcdef0123456789abcdef")}, + {Start: 100, Length: 100, Object: mustParseID("D0123456789abcdef0123456789abcdeg")}, + }, + budget: 50, + curOffset: 0, + } + p.cond = sync.NewCond(&p.mu) + // Simulate the watcher goroutine spawned in OpenObject + go func() { + <-ctx.Done() + p.mu.Lock() + p.cond.Broadcast() + p.mu.Unlock() + }() + return p + }, + mockRepo: func(mockRepo *repomocks.MockRepository) { + mockRepo.On("PrefetchObjects", mock.Anything, []object.ID{mustParseID("D0123456789abcdef0123456789abcdef")}, "").Return(([]content.ID)(nil), nil).Once() + }, + trigger: func(prefetch *objectPrefetch) { + // Wait a bit for the first fetch and wait to happen + time.Sleep(50 * time.Millisecond) + prefetch.cancel() // This triggers the watcher, broadcasts, and exits prefetchProc + }, + }, + { + name: "prefetch error should not panic and continue", + setupPrefetch: func(ctx context.Context, cancel context.CancelFunc) *objectPrefetch { + p := &objectPrefetch{ + ctx: ctx, + entries: []object.IndirectObjectEntry{ + {Start: 0, Length: 100, Object: mustParseID("D0123456789abcdef0123456789abcdef")}, + }, + budget: 200, + curOffset: 0, + } + p.cond = sync.NewCond(&p.mu) + return p + }, + mockRepo: func(mockRepo *repomocks.MockRepository) { + mockRepo.On("PrefetchObjects", mock.Anything, []object.ID{mustParseID("D0123456789abcdef0123456789abcdef")}, "").Return(([]content.ID)(nil), errors.New("fake-error")).Once() + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + mockRepo := repomocks.NewMockRepository(t) + if tc.mockRepo != nil { + tc.mockRepo(mockRepo) + } + + kor := &kopiaObjectReader{ + rawRepo: mockRepo, + logger: velerotest.NewLogger(), + prefetch: tc.setupPrefetch(ctx, cancel), + } + + if tc.runConcurrently { + done := make(chan struct{}) + go func() { + kor.prefetchProc() + close(done) + }() + if tc.trigger != nil { + tc.trigger(kor.prefetch) + } + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("prefetchProc did not finish in time") + } + } else { + kor.prefetchProc() + } + + mockRepo.AssertExpectations(t) + }) + } +} diff --git a/pkg/repository/udmrepo/kopialib/repo_init.go b/pkg/repository/udmrepo/kopialib/repo_init.go index 4e62c9087..5c272298c 100644 --- a/pkg/repository/udmrepo/kopialib/repo_init.go +++ b/pkg/repository/udmrepo/kopialib/repo_init.go @@ -43,12 +43,18 @@ type kopiaBackendStore struct { store backend.Store } +type kopiaBackendStoreFactory struct { + name string + description string + newStore func() backend.Store +} + // backendStores lists the supported backend storages at present -var backendStores = []kopiaBackendStore{ - {udmrepo.StorageTypeAzure, "an Azure blob storage", &backend.AzureBackend{}}, - {udmrepo.StorageTypeFs, "a filesystem", &backend.FsBackend{}}, - {udmrepo.StorageTypeGcs, "a Google Cloud Storage bucket", &backend.GCSBackend{}}, - {udmrepo.StorageTypeS3, "an S3 bucket", &backend.S3Backend{}}, +var backendStores = []kopiaBackendStoreFactory{ + {udmrepo.StorageTypeAzure, "an Azure blob storage", func() backend.Store { return &backend.AzureBackend{} }}, + {udmrepo.StorageTypeFs, "a filesystem", func() backend.Store { return &backend.FsBackend{} }}, + {udmrepo.StorageTypeGcs, "a Google Cloud Storage bucket", func() backend.Store { return &backend.GCSBackend{} }}, + {udmrepo.StorageTypeS3, "an S3 bucket", func() backend.Store { return &backend.S3Backend{} }}, } const udmRepoBlobID = "udmrepo.Repository" @@ -226,7 +232,11 @@ func connectStore(ctx context.Context, repoOption udmrepo.RepoOptions, logger lo func findBackendStore(storage string) *kopiaBackendStore { for _, options := range backendStores { if strings.EqualFold(options.name, storage) { - return &options + return &kopiaBackendStore{ + name: options.name, + description: options.description, + store: options.newStore(), + } } } diff --git a/pkg/repository/udmrepo/kopialib/repo_init_test.go b/pkg/repository/udmrepo/kopialib/repo_init_test.go index c8b8e6aa1..130bb7b4d 100644 --- a/pkg/repository/udmrepo/kopialib/repo_init_test.go +++ b/pkg/repository/udmrepo/kopialib/repo_init_test.go @@ -41,6 +41,29 @@ import ( "github.com/cockroachdb/errors" ) +func TestFindBackendStore(t *testing.T) { + // findBackendStore should return a unique instance on each call + // so that concurrently executing controllers do not overwrite each other's credentials/options. + t.Run("returns distinct instances", func(t *testing.T) { + store1 := findBackendStore(udmrepo.StorageTypeS3) + require.NotNil(t, store1) + + store2 := findBackendStore(udmrepo.StorageTypeS3) + require.NotNil(t, store2) + + // The pointers to the wrapper struct must be different + assert.NotSame(t, store1, store2, "findBackendStore should return different kopiaBackendStore instances") + + // The pointers to the actual underlying store must be different + assert.NotSame(t, store1.store, store2.store, "findBackendStore should return different backend.Store instances") + }) + + t.Run("returns nil for unknown storage type", func(t *testing.T) { + store := findBackendStore("unknown-type") + assert.Nil(t, store) + }) +} + type comparableError struct { message string } @@ -133,11 +156,11 @@ func TestCreateBackupRepo(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { logger := velerotest.NewLogger() - backendStores = []kopiaBackendStore{ - {udmrepo.StorageTypeAzure, "fake store", tc.backendStore}, - {udmrepo.StorageTypeFs, "fake store", tc.backendStore}, - {udmrepo.StorageTypeGcs, "fake store", tc.backendStore}, - {udmrepo.StorageTypeS3, "fake store", tc.backendStore}, + backendStores = []kopiaBackendStoreFactory{ + {udmrepo.StorageTypeAzure, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeFs, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeGcs, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeS3, "fake store", func() backend.Store { return tc.backendStore }}, } if tc.backendStore != nil { @@ -219,11 +242,11 @@ func TestConnectBackupRepo(t *testing.T) { logger := velerotest.NewLogger() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - backendStores = []kopiaBackendStore{ - {udmrepo.StorageTypeAzure, "fake store", tc.backendStore}, - {udmrepo.StorageTypeFs, "fake store", tc.backendStore}, - {udmrepo.StorageTypeGcs, "fake store", tc.backendStore}, - {udmrepo.StorageTypeS3, "fake store", tc.backendStore}, + backendStores = []kopiaBackendStoreFactory{ + {udmrepo.StorageTypeAzure, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeFs, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeGcs, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeS3, "fake store", func() backend.Store { return tc.backendStore }}, } if tc.backendStore != nil { @@ -441,11 +464,11 @@ func TestGetRepositoryStatus(t *testing.T) { logger := velerotest.NewLogger() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - backendStores = []kopiaBackendStore{ - {udmrepo.StorageTypeAzure, "fake store", tc.backendStore}, - {udmrepo.StorageTypeFs, "fake store", tc.backendStore}, - {udmrepo.StorageTypeGcs, "fake store", tc.backendStore}, - {udmrepo.StorageTypeS3, "fake store", tc.backendStore}, + backendStores = []kopiaBackendStoreFactory{ + {udmrepo.StorageTypeAzure, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeFs, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeGcs, "fake store", func() backend.Store { return tc.backendStore }}, + {udmrepo.StorageTypeS3, "fake store", func() backend.Store { return tc.backendStore }}, } if tc.backendStore != nil { diff --git a/pkg/repository/udmrepo/mocks/BackupRepo.go b/pkg/repository/udmrepo/mocks/BackupRepo.go index 623c4d70d..3206422b4 100644 --- a/pkg/repository/udmrepo/mocks/BackupRepo.go +++ b/pkg/repository/udmrepo/mocks/BackupRepo.go @@ -699,8 +699,8 @@ func (_c *BackupRepo_NewObjectWriter_Call) RunAndReturn(run func(ctx context.Con } // OpenObject provides a mock function for the type BackupRepo -func (_mock *BackupRepo) OpenObject(ctx context.Context, id udmrepo.ID) (udmrepo.ObjectReader, error) { - ret := _mock.Called(ctx, id) +func (_mock *BackupRepo) OpenObject(ctx context.Context, id udmrepo.ID, opt udmrepo.ObjectReadOptions) (udmrepo.ObjectReader, error) { + ret := _mock.Called(ctx, id, opt) if len(ret) == 0 { panic("no return value specified for OpenObject") @@ -708,18 +708,18 @@ func (_mock *BackupRepo) OpenObject(ctx context.Context, id udmrepo.ID) (udmrepo var r0 udmrepo.ObjectReader var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context, udmrepo.ID) (udmrepo.ObjectReader, error)); ok { - return returnFunc(ctx, id) + if returnFunc, ok := ret.Get(0).(func(context.Context, udmrepo.ID, udmrepo.ObjectReadOptions) (udmrepo.ObjectReader, error)); ok { + return returnFunc(ctx, id, opt) } - if returnFunc, ok := ret.Get(0).(func(context.Context, udmrepo.ID) udmrepo.ObjectReader); ok { - r0 = returnFunc(ctx, id) + if returnFunc, ok := ret.Get(0).(func(context.Context, udmrepo.ID, udmrepo.ObjectReadOptions) udmrepo.ObjectReader); ok { + r0 = returnFunc(ctx, id, opt) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(udmrepo.ObjectReader) } } - if returnFunc, ok := ret.Get(1).(func(context.Context, udmrepo.ID) error); ok { - r1 = returnFunc(ctx, id) + if returnFunc, ok := ret.Get(1).(func(context.Context, udmrepo.ID, udmrepo.ObjectReadOptions) error); ok { + r1 = returnFunc(ctx, id, opt) } else { r1 = ret.Error(1) } @@ -734,11 +734,12 @@ type BackupRepo_OpenObject_Call struct { // OpenObject is a helper method to define mock.On call // - ctx context.Context // - id udmrepo.ID -func (_e *BackupRepo_Expecter) OpenObject(ctx interface{}, id interface{}) *BackupRepo_OpenObject_Call { - return &BackupRepo_OpenObject_Call{Call: _e.mock.On("OpenObject", ctx, id)} +// - opt udmrepo.ObjectReadOptions +func (_e *BackupRepo_Expecter) OpenObject(ctx interface{}, id interface{}, opt interface{}) *BackupRepo_OpenObject_Call { + return &BackupRepo_OpenObject_Call{Call: _e.mock.On("OpenObject", ctx, id, opt)} } -func (_c *BackupRepo_OpenObject_Call) Run(run func(ctx context.Context, id udmrepo.ID)) *BackupRepo_OpenObject_Call { +func (_c *BackupRepo_OpenObject_Call) Run(run func(ctx context.Context, id udmrepo.ID, opt udmrepo.ObjectReadOptions)) *BackupRepo_OpenObject_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -748,9 +749,14 @@ func (_c *BackupRepo_OpenObject_Call) Run(run func(ctx context.Context, id udmre if args[1] != nil { arg1 = args[1].(udmrepo.ID) } + var arg2 udmrepo.ObjectReadOptions + if args[2] != nil { + arg2 = args[2].(udmrepo.ObjectReadOptions) + } run( arg0, arg1, + arg2, ) }) return _c @@ -761,7 +767,7 @@ func (_c *BackupRepo_OpenObject_Call) Return(objectReader udmrepo.ObjectReader, return _c } -func (_c *BackupRepo_OpenObject_Call) RunAndReturn(run func(ctx context.Context, id udmrepo.ID) (udmrepo.ObjectReader, error)) *BackupRepo_OpenObject_Call { +func (_c *BackupRepo_OpenObject_Call) RunAndReturn(run func(ctx context.Context, id udmrepo.ID, opt udmrepo.ObjectReadOptions) (udmrepo.ObjectReader, error)) *BackupRepo_OpenObject_Call { _c.Call.Return(run) return _c } diff --git a/pkg/repository/udmrepo/repo.go b/pkg/repository/udmrepo/repo.go index 76cdc5f1d..5873db743 100644 --- a/pkg/repository/udmrepo/repo.go +++ b/pkg/repository/udmrepo/repo.go @@ -72,6 +72,11 @@ type ObjectWriteOptions struct { ParentObject ID // The object in the previous snapshot, for incremental backup } +type ObjectReadOptions struct { + Prefetch bool + PrefetchBudgetMB int +} + type AdvancedFeatureInfo struct { MultiPartBackup bool // if set to true, it means the repo supports multiple-part backup } @@ -136,7 +141,7 @@ type BackupRepoService interface { type BackupRepo interface { // OpenObject opens an existing object for read. // id: the object's unified identifier. - OpenObject(ctx context.Context, id ID) (ObjectReader, error) + OpenObject(ctx context.Context, id ID, opt ObjectReadOptions) (ObjectReader, error) // GetManifest gets a manifest data from the backup repository. GetManifest(ctx context.Context, id ID, mani *RepoManifest) error diff --git a/pkg/restore/actions/csi/pvc_action.go b/pkg/restore/actions/csi/pvc_action.go index 2203682be..a14b985a7 100644 --- a/pkg/restore/actions/csi/pvc_action.go +++ b/pkg/restore/actions/csi/pvc_action.go @@ -20,17 +20,20 @@ import ( "context" "encoding/json" "fmt" - - snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + "time" "github.com/cockroachdb/errors" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumesnapshot/v1" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" utilrand "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/client-go/kubernetes" crclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -44,6 +47,9 @@ import ( uploaderUtil "github.com/vmware-tanzu/velero/pkg/uploader/util" "github.com/vmware-tanzu/velero/pkg/util" "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/csi" + "github.com/vmware-tanzu/velero/pkg/util/datamover" + "github.com/vmware-tanzu/velero/pkg/util/kube" ) const ( @@ -53,12 +59,14 @@ const ( // pvcRestoreItemAction is a restore item action plugin for Velero type pvcRestoreItemAction struct { - log logrus.FieldLogger - crClient crclient.Client + log logrus.FieldLogger + crClient crclient.Client + kubeClient kubernetes.Interface + csiSnapshotClient snapshotter.SnapshotV1Interface } // AppliesTo returns information indicating that the -// PVCRestoreItemAction should be run while restoring PVCs. +// PVCCSIRestoreItemAction should be run while restoring PVCs. func (p *pvcRestoreItemAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ IncludedResources: []string{"persistentvolumeclaims"}, @@ -83,28 +91,178 @@ func (p *pvcRestoreItemAction) Execute( } logger := p.log.WithFields(logrus.Fields{ - "Action": "PVCRestoreItemAction", + "Action": "PVCCSIRestoreItemAction", "PVC": pvc.Namespace + "/" + pvc.Name, "Restore": input.Restore.Namespace + "/" + input.Restore.Name, }) - logger.Info("Starting PVCRestoreItemAction for PVC") + logger.Info("Starting PVCCSIRestoreItemAction for PVC") + // make sure this RIA only runs for CSI snapshot vsName, nameOK := pvcFromBackup.Annotations[velerov1api.VolumeSnapshotLabel] if !nameOK { - logger.Info("Skipping PVCRestoreItemAction for PVC, PVC does not have a CSI VolumeSnapshot.") + logger.Info("Skipping PVCCSIRestoreItemAction for PVC, PVC does not have a CSI VolumeSnapshot.") return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: input.Item, }, nil } - // If PVC already exists, returns early. - if p.isResourceExist(pvc, *input.Restore) { + pvcExists, existingPVC, err := p.isResourceExist(&pvc, *input.Restore) + if err != nil { + logger.Error(err) + return nil, errors.WithStack(err) + } + + var output *velero.RestoreItemActionExecuteOutput + if boolptr.IsSetToFalse(input.Restore.Spec.RestorePVs) { + output, err = p.executeWithoutPVRestore(logger, input, pvcExists, &pvc) + } else { + backup := new(velerov1api.Backup) + if err := p.crClient.Get(context.TODO(), crclient.ObjectKey{Namespace: input.Restore.Namespace, Name: input.Restore.Spec.BackupName}, backup); err != nil { + return nil, fmt.Errorf("fail to get backup for restore: %s", err.Error()) + } + if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) { + output, err = p.executeWithDataMove(logger, input, backup, pvcExists, existingPVC, &pvc, &pvcFromBackup) + } else { + output, err = p.executeWithoutDataMove(logger, input, pvcExists, &pvc, vsName) + } + } + if err != nil { + logger.Error(err) + return nil, errors.WithStack(err) + } + + logger.Info("Returning from PVCCSIRestoreItemAction for PVC") + + return output, nil +} + +func (p *pvcRestoreItemAction) executeWithoutPVRestore(logger *logrus.Entry, input *velero.RestoreItemActionExecuteInput, pvcExists bool, pvc *corev1api.PersistentVolumeClaim) (*velero.RestoreItemActionExecuteOutput, error) { + if pvcExists { logger.Warnf("PVC already exists. Skip restore this PVC.") return &velero.RestoreItemActionExecuteOutput{ UpdatedItem: input.Item, }, nil } + logger.Info("Restore did not request for PVs to be restored from snapshot") + pvc.Spec.VolumeName = "" + pvc.Spec.DataSource = nil + pvc.Spec.DataSourceRef = nil + + unstructuredPVC, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pvc) + if err != nil { + return nil, errors.WithStack(err) + } + + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: &unstructured.Unstructured{Object: unstructuredPVC}, + }, nil +} + +func (p *pvcRestoreItemAction) executeWithoutDataMove(logger *logrus.Entry, input *velero.RestoreItemActionExecuteInput, pvcExists bool, pvc *corev1api.PersistentVolumeClaim, vsName string) (*velero.RestoreItemActionExecuteOutput, error) { + if pvcExists { + logger.Warnf("PVC already exists. Skip restore this PVC.") + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: input.Item, + }, nil + } + + //To avoid confilcs, vs and vsc get a new uniq name based in restore UID + // and vs name old name + newVSName := util.GenerateSha256FromRestoreUIDAndVsName(string(input.Restore.UID), vsName) + + logger.Debugf("Setting PVC source to VolumeSnapshot new name: %s", newVSName) + resetPVCSourceToVolumeSnapshot(pvc, newVSName) + + // Force-restore the VolumeSnapshot even when restore resource filters + // would otherwise exclude it (mirrors backup-side must-include). + annotations := pvc.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + pvc.SetAnnotations(annotations) + + unstructuredPVC, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pvc) + if err != nil { + return nil, errors.WithStack(err) + } + + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: &unstructured.Unstructured{Object: unstructuredPVC}, + AdditionalItems: []velero.ResourceIdentifier{ + { + GroupResource: kuberesource.VolumeSnapshots, + Name: vsName, + Namespace: pvc.Namespace, + }, + }, + }, nil +} + +func (p *pvcRestoreItemAction) executeWithDataMove(logger *logrus.Entry, input *velero.RestoreItemActionExecuteInput, backup *velerov1api.Backup, pvcExists bool, existingPVC, pvc, pvcFromBackup *corev1api.PersistentVolumeClaim) (out *velero.RestoreItemActionExecuteOutput, err error) { + ctx := context.Background() + var existingPV *corev1api.PersistentVolume + + // If PVC already exists and is not in-place restore, returns early. + if pvcExists && !input.Restore.IsVolumeDataInplaceRestore() { + logger.Warnf("PVC already exists and ExistingVolumeDataPolicy is not in-place restore. Skip restore this PVC.") + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: input.Item, + }, nil + } + + logger.Info("Start DataMover restore.") + + // If PVC doesn't have a DataUploadNameLabel, which should be created + // during backup, then CSI cannot handle the volume during to restore, + // so return early to let Velero tries to fall back to Velero native snapshot. + if _, ok := pvcFromBackup.Annotations[velerov1api.DataUploadNameAnnotation]; !ok { + logger.Warnf("PVC doesn't have a DataUpload for data mover. Return.") + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: input.Item, + }, nil + } + + var dataUploadResult *velerov2alpha1.DataUploadResult + dataUploadResult, err = getDataUploadResult(ctx, input.Restore, pvc, p.crClient) + if err != nil { + return nil, errors.Wrapf(err, "fail get DataUploadResult for restore: %s", input.Restore.Name) + } + + var volumeSnapshot *snapshotv1api.VolumeSnapshot + restoreType := input.Restore.Spec.ExistingVolumeDataPolicy + if pvcExists { + if existingPVC.Status.Phase != corev1api.ClaimBound { + return nil, errors.New("ExistingVolumeDataPolicy is in-place restore, but the existing PVC is not bound.") + } + // take a CSI snapshot of the existing PVC as the baseline of CBT + if input.Restore.IsVolumeDataInplaceIncrementalRestore() && datamover.IsVeleroBlockDataMover(dataUploadResult.DataMover) { + logger.Info("ExistingVolumeDataPolicy is in-place incremental restore and data mover is velero-block. Taking a CSI snapshot of the existing PVC as the baseline of CBT...") + volumeSnapshot, err = p.createVolumeSnapshot(ctx, logger, input.Restore, *existingPVC, dataUploadResult.SnapshotClass, backup.Spec.CSISnapshotTimeout.Duration) + if err != nil { + logger.Warnf("fail to create VolumeSnapshot for existing PVC %s/%s: %s, fallback to in-place full restore", existingPVC.Namespace, existingPVC.Name, err.Error()) + restoreType = velerov1api.VolumeDataPolicyTypeFull + } else { + defer func() { + if err != nil { + csi.CleanupVolumeSnapshot(ctx, volumeSnapshot, p.crClient, logger) + } + }() + } + } + + // delete the existing PVC, otherwise the target PVC cannot be restored + existingPV, err = p.deleteExistingPVC(ctx, logger, pvc, existingPVC, backup.Spec.CSISnapshotTimeout.Duration) + if err != nil { + return nil, errors.WithStack(err) + } + } + + operationID := label.GetValidName( + string(velerov1api.AsyncOperationIDPrefixDataDownload) + + string(input.Restore.UID) + "." + string(pvcFromBackup.UID)) + // If cross-namespace restore is configured, change the namespace // for PVC object to be restored newNamespace, ok := input.Restore.Spec.NamespaceMapping[pvc.GetNamespace()] @@ -113,81 +271,26 @@ func (p *pvcRestoreItemAction) Execute( newNamespace = pvc.Namespace } - operationID := "" - - additionalItems := []velero.ResourceIdentifier{} - if boolptr.IsSetToFalse(input.Restore.Spec.RestorePVs) { - logger.Info("Restore did not request for PVs to be restored from snapshot") - pvc.Spec.VolumeName = "" - pvc.Spec.DataSource = nil - pvc.Spec.DataSourceRef = nil - } else { - backup := new(velerov1api.Backup) - err := p.crClient.Get( - context.TODO(), - crclient.ObjectKey{ - Namespace: input.Restore.Namespace, - Name: input.Restore.Spec.BackupName, - }, - backup, - ) - - if err != nil { - logger.Error("Fail to get backup for restore.") - return nil, fmt.Errorf("fail to get backup for restore: %s", err.Error()) - } - - if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) { - logger.Info("Start DataMover restore.") - - // If PVC doesn't have a DataUploadNameLabel, which should be created - // during backup, then CSI cannot handle the volume during to restore, - // so return early to let Velero tries to fall back to Velero native snapshot. - if _, ok := pvcFromBackup.Annotations[velerov1api.DataUploadNameAnnotation]; !ok { - logger.Warnf("PVC doesn't have a DataUpload for data mover. Return.") - return &velero.RestoreItemActionExecuteOutput{ - UpdatedItem: input.Item, - }, nil - } - - operationID = label.GetValidName( - string(velerov1api.AsyncOperationIDPrefixDataDownload) + - string(input.Restore.UID) + "." + string(pvcFromBackup.UID)) - dataDownload, err := restoreFromDataUploadResult( - context.Background(), input.Restore, backup, &pvc, newNamespace, - operationID, p.crClient) - if err != nil { - logger.Errorf("Fail to restore from DataUploadResult: %s", err.Error()) - return nil, errors.WithStack(err) - } - logger.Infof("DataDownload %s/%s is created successfully.", - dataDownload.Namespace, dataDownload.Name) - } else { - //To avoid confilcs, vs and vsc get a new uniq name based in restore UID - // and vs name old name - newVSName := util.GenerateSha256FromRestoreUIDAndVsName(string(input.Restore.UID), vsName) - - p.log.Debugf("Setting PVC source to VolumeSnapshot new name: %s", newVSName) - resetPVCSourceToVolumeSnapshot(&pvc, newVSName) - - additionalItems = append(additionalItems, velero.ResourceIdentifier{ - GroupResource: kuberesource.VolumeSnapshots, - Name: vsName, - Namespace: pvc.Namespace, - }) - } + var dataDownload *velerov2alpha1.DataDownload + dataDownload, err = restoreFromDataUploadResult( + context.Background(), dataUploadResult, input.Restore, backup, pvc, existingPV, newNamespace, + operationID, string(restoreType), volumeSnapshot, p.crClient) + if err != nil { + logger.Errorf("Fail to restore from DataUploadResult: %s", err.Error()) + return nil, errors.WithStack(err) } + logger.Infof("DataDownload %s/%s is created successfully.", + dataDownload.Namespace, dataDownload.Name) - pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pvc) + var unstructuredPVC map[string]any + unstructuredPVC, err = runtime.DefaultUnstructuredConverter.ToUnstructured(pvc) if err != nil { return nil, errors.WithStack(err) } - logger.Info("Returning from PVCRestoreItemAction for PVC") return &velero.RestoreItemActionExecuteOutput{ - UpdatedItem: &unstructured.Unstructured{Object: pvcMap}, - OperationID: operationID, - AdditionalItems: additionalItems, + UpdatedItem: &unstructured.Unstructured{Object: unstructuredPVC}, + OperationID: operationID, }, nil } @@ -397,8 +500,14 @@ func newDataDownload( backup *velerov1api.Backup, dataUploadResult *velerov2alpha1.DataUploadResult, pvc *corev1api.PersistentVolumeClaim, - newNamespace, operationID string, + pv *corev1api.PersistentVolume, + newNamespace, operationID, restoreType string, + volumeSnapshot *snapshotv1api.VolumeSnapshot, ) *velerov2alpha1.DataDownload { + pvName := "" + if pv != nil { + pvName = pv.Name + } dataDownload := &velerov2alpha1.DataDownload{ TypeMeta: metav1.TypeMeta{ APIVersion: velerov2alpha1.SchemeGroupVersion.String(), @@ -425,6 +534,7 @@ func newDataDownload( Spec: velerov2alpha1.DataDownloadSpec{ TargetVolume: velerov2alpha1.TargetVolumeSpec{ PVC: pvc.Name, + PV: pvName, Namespace: newNamespace, FSType: dataUploadResult.FSType, }, @@ -435,8 +545,15 @@ func newDataDownload( SourceNamespace: dataUploadResult.SourceNamespace, OperationTimeout: backup.Spec.CSISnapshotTimeout, NodeOS: dataUploadResult.NodeOS, + RestoreType: restoreType, }, } + if volumeSnapshot != nil { + dataDownload.Spec.CSISnapshot = &velerov2alpha1.CSISnapshotSpec{ + VolumeSnapshot: volumeSnapshot.Name, + VolumeSnapshotNamespace: volumeSnapshot.Namespace, + } + } if restore.Spec.UploaderConfig != nil { dataDownload.Spec.DataMoverConfig = uploaderUtil.StoreRestoreConfig(restore.Spec.UploaderConfig) } @@ -445,17 +562,15 @@ func newDataDownload( func restoreFromDataUploadResult( ctx context.Context, + dataUploadResult *velerov2alpha1.DataUploadResult, restore *velerov1api.Restore, backup *velerov1api.Backup, pvc *corev1api.PersistentVolumeClaim, - newNamespace, operationID string, + pv *corev1api.PersistentVolume, + newNamespace, operationID, restoreType string, + volumeSnapshot *snapshotv1api.VolumeSnapshot, crClient crclient.Client, ) (*velerov2alpha1.DataDownload, error) { - dataUploadResult, err := getDataUploadResult(ctx, restore, pvc, crClient) - if err != nil { - return nil, errors.Wrapf(err, "fail get DataUploadResult for restore: %s", - restore.Name) - } pvc.Spec.VolumeName = "" if pvc.Spec.Selector == nil { pvc.Spec.Selector = &metav1.LabelSelector{} @@ -472,10 +587,13 @@ func restoreFromDataUploadResult( backup, dataUploadResult, pvc, + pv, newNamespace, operationID, + restoreType, + volumeSnapshot, ) - err = crClient.Create(ctx, dataDownload) + err := crClient.Create(ctx, dataDownload) if err != nil { return nil, errors.Wrapf(err, "fail to create DataDownload") } @@ -484,9 +602,9 @@ func restoreFromDataUploadResult( } func (p *pvcRestoreItemAction) isResourceExist( - pvc corev1api.PersistentVolumeClaim, + pvc *corev1api.PersistentVolumeClaim, restore velerov1api.Restore, -) bool { +) (bool, *corev1api.PersistentVolumeClaim, error) { // get target namespace to restore into, if different from source namespace targetNamespace := pvc.Namespace if target, ok := restore.Spec.NamespaceMapping[pvc.Namespace]; ok { @@ -494,17 +612,115 @@ func (p *pvcRestoreItemAction) isResourceExist( } tmpPVC := new(corev1api.PersistentVolumeClaim) - if err := p.crClient.Get( + err := p.crClient.Get( context.Background(), crclient.ObjectKey{ Name: pvc.Name, Namespace: targetNamespace, }, tmpPVC, - ); err == nil { - return true + ) + if err == nil { + return true, tmpPVC, nil } - return false + if apierrors.IsNotFound(err) { + return false, nil, nil + } + return false, nil, errors.Wrapf(err, "fail to get PVC %s in namespace %s", pvc.Name, targetNamespace) +} + +func (p *pvcRestoreItemAction) deleteExistingPVC(ctx context.Context, logger *logrus.Entry, targetPVC *corev1api.PersistentVolumeClaim, existingPVC *corev1api.PersistentVolumeClaim, operationTimeout time.Duration) (*corev1api.PersistentVolume, error) { + // Capture the "selected-node" annotation from the existing PVC before it is deleted below, + // and carry it on the target PVC via a Velero-internal carrier annotation. The restore + // engine translates the carrier back to the Kubernetes "selected-node" annotation after + // all RestoreItemActions have run, so the recreated target PVC keeps the same scheduling + // constraint regardless of the order in which RestoreItemActions execute (the generic PVC + // RIA unconditionally strips the Kubernetes annotation). + selectedNode, exists := existingPVC.Annotations[kube.KubeAnnSelectedNode] + if exists { + logger.Infof("Carrying %q annotation with value %q for target PVC to keep the same selected node as the existing PVC", kube.KubeAnnSelectedNode, selectedNode) + if targetPVC.Annotations == nil { + targetPVC.Annotations = map[string]string{} + } + targetPVC.Annotations[velerov1api.InplaceRestoreSelectedNodeAnnotation] = selectedNode + } + + var err error + logger.Info("ExistingVolumeDataPolicy is in-place restore. Deleting the existing PVC but keep the PV...") + pv := &corev1api.PersistentVolume{} + if err = p.crClient.Get(context.Background(), crclient.ObjectKey{Name: existingPVC.Spec.VolumeName}, pv); err != nil { + return nil, errors.Errorf("Fail to get PV %s: %s", existingPVC.Spec.VolumeName, err.Error()) + } + + // set reclaim policy to retain + updatedPV, err := kube.SetPVReclaimPolicy(ctx, p.kubeClient.CoreV1(), pv, corev1api.PersistentVolumeReclaimRetain) + if err != nil { + return nil, errors.Wrapf(err, "fail to set PV reclaim policy to retain for PV %s", pv.Name) + } + if updatedPV != nil { + pv = updatedPV + } + + if err = kube.EnsureDeletePVC(ctx, p.kubeClient.CoreV1(), existingPVC.Name, existingPVC.Namespace, operationTimeout); err != nil { + return nil, errors.Wrapf(err, "fail to delete the existing PVC %s in namespace %s", existingPVC.Name, existingPVC.Namespace) + } + + logger.Info("Existing PVC deleted") + + return pv, nil +} + +func (p *pvcRestoreItemAction) createVolumeSnapshot(ctx context.Context, logger *logrus.Entry, restore *velerov1api.Restore, pvc corev1api.PersistentVolumeClaim, vsClass string, operationTimeout time.Duration) (vs *snapshotv1api.VolumeSnapshot, err error) { + logger.Infof("creating VolumeSnapshot for PVC %s/%s with VolumeSnapshotClass %s", pvc.Namespace, pvc.Name, vsClass) + + labels := map[string]string{ + velerov1api.RestoreNameLabel: label.GetValidName(restore.Name), + } + for k, v := range pvc.ObjectMeta.Labels { + labels[k] = v + } + + vs = &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "velero-" + pvc.Name + "-", + Namespace: pvc.Namespace, + Labels: labels, + }, + Spec: snapshotv1api.VolumeSnapshotSpec{ + Source: snapshotv1api.VolumeSnapshotSource{ + PersistentVolumeClaimName: &pvc.Name, + }, + VolumeSnapshotClassName: &vsClass, + }, + } + + if err := p.crClient.Create(ctx, vs); err != nil { + return nil, errors.Wrapf(err, "failed to create the VolumeSnapshot for PVC %s/%s", pvc.Namespace, pvc.Name) + } + + logger.Infof("VolumeSnapshot %s for PVC %s/%s created", vs.Name, pvc.Namespace, pvc.Name) + vsName := vs.Name + vsNamespace := vs.Namespace + + _, err = csi.WaitUntilVSCHandleIsReady(vs, p.crClient, logger, operationTimeout) + if err != nil { + csi.CleanupVolumeSnapshot(ctx, vs, p.crClient, logger) + return nil, errors.Wrapf(err, "failed to wait for VolumeSnapshotContent of VolumeSnapshot %s/%s to be ready within timeout %v", + vsNamespace, vsName, operationTimeout) + } + + var updatedVS *snapshotv1api.VolumeSnapshot + updatedVS, err = csi.WaitVolumeSnapshotReady(ctx, p.csiSnapshotClient, vs.Name, vs.Namespace, operationTimeout, logger) + if err != nil { + csi.CleanupVolumeSnapshot(ctx, vs, p.crClient, logger) + return nil, errors.Wrapf(err, "failed to wait for VolumeSnapshot %s/%s to become Ready within timeout %v", + vsNamespace, vsName, operationTimeout) + } + vs = updatedVS + + logger.Infof("VolumeSnapshot %s for PVC %s/%s is ready to use", vs.Name, pvc.Namespace, pvc.Name) + + return vs, nil } func NewPvcRestoreItemAction(f client.Factory) plugincommon.HandlerInitializer { @@ -514,9 +730,25 @@ func NewPvcRestoreItemAction(f client.Factory) plugincommon.HandlerInitializer { return nil, err } + kubeClient, err := f.KubeClient() + if err != nil { + return nil, err + } + + clientConfig, err := f.ClientConfig() + if err != nil { + return nil, err + } + csiSnapshotClient, err := snapshotter.NewForConfig(clientConfig) + if err != nil { + return nil, err + } + return &pvcRestoreItemAction{ - log: logger, - crClient: crClient, + log: logger, + crClient: crClient, + kubeClient: kubeClient, + csiSnapshotClient: csiSnapshotClient, }, nil } } diff --git a/pkg/restore/actions/csi/pvc_action_test.go b/pkg/restore/actions/csi/pvc_action_test.go index ea712c027..47e8937a1 100644 --- a/pkg/restore/actions/csi/pvc_action_test.go +++ b/pkg/restore/actions/csi/pvc_action_test.go @@ -28,12 +28,15 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" crclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" @@ -371,15 +374,19 @@ func TestExecute(t *testing.T) { backup *velerov1api.Backup restore *velerov1api.Restore pvc *corev1api.PersistentVolumeClaim + pv *corev1api.PersistentVolume + pvcFromBackup *corev1api.PersistentVolumeClaim vs *snapshotv1api.VolumeSnapshot dataUploadResult *corev1api.ConfigMap expectedErr string expectedDataDownload *velerov2alpha1.DataDownload expectedPVC *corev1api.PersistentVolumeClaim preCreatePVC bool + kubeClientObj []runtime.Object }{ { name: "Don't restore PV", + backup: builder.ForBackup("velero", "testBackup").Result(), restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").RestorePVs(false).Result(), pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName")).Result(), expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName")).VolumeName("").Result(), @@ -402,15 +409,40 @@ func TestExecute(t *testing.T) { vs: builder.ForVolumeSnapshot("velero", vsName).ObjectMeta( builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi"), ).Result(), - expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName")).Result(), + expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations( + velerov1api.VolumeSnapshotLabel, "vsName", + velerov1api.MustIncludeAdditionalItemRestoreAnnotation, "true", + )).Result(), }, { - name: "Restore from VolumeSnapshot without volume-snapshot-name annotation", - backup: builder.ForBackup("velero", "testBackup").Result(), - restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").Result(), - pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName", AnnSelectedNode, "node1")).Result(), - vs: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi")).Result(), - expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName", AnnSelectedNode, "node1")).Result(), + name: "Restore from VolumeSnapshot with nil PVC annotations", + backup: builder.ForBackup("velero", "testBackup").Result(), + restore: builder.ForRestore("velero", "testRestore").ObjectMeta(builder.WithUID("restoreUID")).Backup("testBackup").Result(), + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPVC", + Namespace: "velero", + }, + }, + pvcFromBackup: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName")).Result(), + vs: builder.ForVolumeSnapshot("velero", vsName).ObjectMeta( + builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi"), + ).Result(), + expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations( + velerov1api.MustIncludeAdditionalItemRestoreAnnotation, "true", + )).Result(), + }, + { + name: "Restore from VolumeSnapshot without volume-snapshot-name annotation", + backup: builder.ForBackup("velero", "testBackup").Result(), + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName", AnnSelectedNode, "node1")).Result(), + vs: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi")).Result(), + expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations( + velerov1api.VolumeSnapshotLabel, "vsName", + AnnSelectedNode, "node1", + velerov1api.MustIncludeAdditionalItemRestoreAnnotation, "true", + )).Result(), }, { name: "DataUploadResult cannot be found", @@ -460,6 +492,47 @@ func TestExecute(t *testing.T) { pvc: builder.ForPersistentVolumeClaim("restore", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName", velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), preCreatePVC: true, }, + { + name: "PVC exists and in-place restore set", + backup: builder.ForBackup("velero", "testBackup").SnapshotMoveData(true).Result(), + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").ExistingVolumeDataPolicy(string(velerov1api.VolumeDataPolicyTypeFull)).ItemOperationTimeout(time.Minute * 10).ObjectMeta(builder.WithUID("uid")).Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").Phase(corev1api.ClaimBound).ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName", velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), + pv: builder.ForPersistentVolume("testPV").ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).Result(), + dataUploadResult: builder.ForConfigMap("velero", "testCM").Data("uid", "{}").ObjectMeta(builder.WithLabels(velerov1api.RestoreUIDLabel, "uid", velerov1api.PVCNamespaceNameLabel, "velero.testPVC", velerov1api.ResourceUsageLabel, label.GetValidName(string(velerov1api.VeleroResourceUsageDataUploadResult)))).Result(), + preCreatePVC: true, + kubeClientObj: []runtime.Object{ + builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").Phase(corev1api.ClaimBound).ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName", velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), + }, + expectedDataDownload: func() *velerov2alpha1.DataDownload { + d := builder.ForDataDownload("velero", "name").TargetVolume(velerov2alpha1.TargetVolumeSpec{PVC: "testPVC", Namespace: "velero", PV: "testPV"}). + ObjectMeta(builder.WithOwnerReference([]metav1.OwnerReference{{APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "Restore", Name: "testRestore", UID: "uid", Controller: boolptr.True()}}), + builder.WithLabelsMap(map[string]string{velerov1api.AsyncOperationIDLabel: "dd-uid.", velerov1api.RestoreNameLabel: "testRestore", velerov1api.RestoreUIDLabel: "uid"}), + builder.WithGenerateName("testRestore-")).Result() + d.Spec.RestoreType = "full" + return d + }(), + }, + { + name: "PVC exists and in-place incremental restore set, createVolumeSnapshot fails", + backup: builder.ForBackup("velero", "testBackup").SnapshotMoveData(true).Result(), + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").ExistingVolumeDataPolicy(string(velerov1api.VolumeDataPolicyTypeIncremental)).ItemOperationTimeout(time.Minute * 10).ObjectMeta(builder.WithUID("uid")).Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").Phase(corev1api.ClaimBound).ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName", velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), + pv: builder.ForPersistentVolume("testPV").ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain).Result(), + dataUploadResult: builder.ForConfigMap("velero", "testCM").Data("uid", "{\"DataMover\":\"velero-block\", \"SnapshotClass\":\"test-snapclass\"}").ObjectMeta(builder.WithLabels(velerov1api.RestoreUIDLabel, "uid", velerov1api.PVCNamespaceNameLabel, "velero.testPVC", velerov1api.ResourceUsageLabel, label.GetValidName(string(velerov1api.VeleroResourceUsageDataUploadResult)))).Result(), + preCreatePVC: true, + kubeClientObj: []runtime.Object{ + builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").Phase(corev1api.ClaimBound).ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "vsName", velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), + }, + expectedDataDownload: func() *velerov2alpha1.DataDownload { + d := builder.ForDataDownload("velero", "name").TargetVolume(velerov2alpha1.TargetVolumeSpec{PVC: "testPVC", Namespace: "velero", PV: "testPV"}). + ObjectMeta(builder.WithOwnerReference([]metav1.OwnerReference{{APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "Restore", Name: "testRestore", UID: "uid", Controller: boolptr.True()}}), + builder.WithLabelsMap(map[string]string{velerov1api.AsyncOperationIDLabel: "dd-uid.", velerov1api.RestoreNameLabel: "testRestore", velerov1api.RestoreUIDLabel: "uid"}), + builder.WithGenerateName("testRestore-")).Result() + d.Spec.RestoreType = "full" + d.Spec.DataMover = "velero-block" + return d + }(), + }, } for _, tc := range tests { @@ -473,6 +546,10 @@ func TestExecute(t *testing.T) { object = append(object, tc.vs) } + if tc.pv != nil { + object = append(object, tc.pv) + } + input := new(velero.RestoreItemActionExecuteInput) if tc.pvc != nil { @@ -480,7 +557,13 @@ func TestExecute(t *testing.T) { require.NoError(t, err) input.Item = &unstructured.Unstructured{Object: pvcMap} - input.ItemFromBackup = &unstructured.Unstructured{Object: pvcMap} + if tc.pvcFromBackup != nil { + pvcFromBackupMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pvcFromBackup) + require.NoError(t, err) + input.ItemFromBackup = &unstructured.Unstructured{Object: pvcFromBackupMap} + } else { + input.ItemFromBackup = &unstructured.Unstructured{Object: pvcMap} + } input.Restore = tc.restore } if tc.preCreatePVC { @@ -492,8 +575,9 @@ func TestExecute(t *testing.T) { } pvcRIA := pvcRestoreItemAction{ - log: logrus.New(), - crClient: velerotest.NewFakeControllerRuntimeClient(t, object...), + log: logrus.New(), + crClient: velerotest.NewFakeControllerRuntimeClient(t, object...), + kubeClient: fake.NewSimpleClientset(tc.kubeClientObj...), } output, err := pvcRIA.Execute(input) @@ -508,6 +592,12 @@ func TestExecute(t *testing.T) { err := runtime.DefaultUnstructuredConverter.FromUnstructured(output.UpdatedItem.UnstructuredContent(), pvc) require.NoError(t, err) require.Equal(t, tc.expectedPVC.GetObjectMeta(), pvc.GetObjectMeta()) + if tc.name == "Restore from VolumeSnapshot" { + require.Equal(t, "true", pvc.GetAnnotations()[velerov1api.MustIncludeAdditionalItemRestoreAnnotation]) + require.Len(t, output.AdditionalItems, 1) + require.Equal(t, "volumesnapshots.snapshot.storage.k8s.io", output.AdditionalItems[0].GroupResource.String()) + require.Equal(t, "vsName", output.AdditionalItems[0].Name) + } if pvc.Spec.Selector != nil && pvc.Spec.Selector.MatchLabels != nil { // This is used for long name and namespace case. if len(tc.pvc.Namespace+"."+tc.pvc.Name) >= validation.DNS1035LabelMaxLength { @@ -529,6 +619,128 @@ func TestExecute(t *testing.T) { } } +// TestPrepareForInplaceRestoreSelectedNode verifies that prepareForInplaceRestore captures +// the selected-node annotation from the existing PVC into the Velero-internal carrier +// annotation (not the Kubernetes annotation) on the target PVC, before deleting the PVC. +func TestPrepareForInplaceRestoreSelectedNode(t *testing.T) { + tests := []struct { + name string + existingPVC *corev1api.PersistentVolumeClaim + expectedCarrier string + expectCarrierSet bool + expectKubeAnnoSet bool + }{ + { + name: "existing PVC with selected-node sets carrier annotation only", + existingPVC: builder.ForPersistentVolumeClaim("ns-1", "pvc-1"). + ObjectMeta(builder.WithAnnotations(AnnSelectedNode, "node-1")). + VolumeName("pv-1"). + Phase(corev1api.ClaimBound).Result(), + expectedCarrier: "node-1", + expectCarrierSet: true, + expectKubeAnnoSet: false, + }, + { + name: "existing PVC without selected-node sets neither annotation", + existingPVC: builder.ForPersistentVolumeClaim("ns-1", "pvc-1"). + VolumeName("pv-1"). + Phase(corev1api.ClaimBound).Result(), + expectCarrierSet: false, + expectKubeAnnoSet: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + pv := builder.ForPersistentVolume("pv-1").Result() + kubeClient := fake.NewSimpleClientset(tc.existingPVC, pv) + pvcRIA := pvcRestoreItemAction{ + log: logrus.New(), + crClient: velerotest.NewFakeControllerRuntimeClient(t, pv), + kubeClient: kubeClient, + } + + targetPVC := builder.ForPersistentVolumeClaim("ns-1", "pvc-1").Result() + returnedPV, err := pvcRIA.deleteExistingPVC( + t.Context(), logrus.New().WithField("test", tc.name), + targetPVC, tc.existingPVC, time.Minute) + require.NoError(t, err) + require.Equal(t, "pv-1", returnedPV.Name) + + carrier, carrierOK := targetPVC.Annotations[velerov1api.InplaceRestoreSelectedNodeAnnotation] + require.Equal(t, tc.expectCarrierSet, carrierOK) + if tc.expectCarrierSet { + require.Equal(t, tc.expectedCarrier, carrier) + } + _, kubeAnnoOK := targetPVC.Annotations[AnnSelectedNode] + require.Equal(t, tc.expectKubeAnnoSet, kubeAnnoOK) + }) + } +} + +// TestExecuteInplaceRestore exercises the public Execute() entry for an in-place restore +// with an existing PVC: the carrier annotation must be emitted on the returned item, the +// Kubernetes selected-node annotation must not be set by this RIA, the existing PVC must be +// deleted, and a DataDownload with the in-place restoreType must be created. +func TestExecuteInplaceRestore(t *testing.T) { + existingPVC := builder.ForPersistentVolumeClaim("velero", "testPVC"). + ObjectMeta(builder.WithAnnotations(AnnSelectedNode, "node-1")). + VolumeName("testPV"). + Phase(corev1api.ClaimBound).Result() + existingPV := builder.ForPersistentVolume("testPV").Result() + backup := builder.ForBackup("velero", "testBackup").SnapshotMoveData(true).Result() + restore := builder.ForRestore("velero", "testRestore").Backup("testBackup"). + ObjectMeta(builder.WithUID("uid")).ExistingVolumeDataPolicy("full").Result() + pvcFromBackup := builder.ForPersistentVolumeClaim("velero", "testPVC"). + ObjectMeta(builder.WithAnnotations( + velerov1api.VolumeSnapshotLabel, "vsName", + velerov1api.DataUploadNameAnnotation, "velero/testDU", + )).Result() + dataUploadResult := builder.ForConfigMap("velero", "testCM").Data("uid", "{}"). + ObjectMeta(builder.WithLabels( + velerov1api.RestoreUIDLabel, "uid", + velerov1api.PVCNamespaceNameLabel, "velero.testPVC", + velerov1api.ResourceUsageLabel, label.GetValidName(string(velerov1api.VeleroResourceUsageDataUploadResult)), + )).Result() + + pvcRIA := pvcRestoreItemAction{ + log: logrus.New(), + crClient: velerotest.NewFakeControllerRuntimeClient(t, existingPVC, existingPV, backup, dataUploadResult), + kubeClient: fake.NewSimpleClientset(existingPVC, existingPV), + } + + pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pvcFromBackup.DeepCopy()) + require.NoError(t, err) + pvcFromBackupMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pvcFromBackup) + require.NoError(t, err) + + output, err := pvcRIA.Execute(&velero.RestoreItemActionExecuteInput{ + Item: &unstructured.Unstructured{Object: pvcMap}, + ItemFromBackup: &unstructured.Unstructured{Object: pvcFromBackupMap}, + Restore: restore, + }) + require.NoError(t, err) + + updatedPVC := new(corev1api.PersistentVolumeClaim) + require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured( + output.UpdatedItem.UnstructuredContent(), updatedPVC)) + + // Carrier annotation carries the captured value; the Kubernetes annotation is not set by this RIA. + require.Equal(t, "node-1", updatedPVC.Annotations[velerov1api.InplaceRestoreSelectedNodeAnnotation]) + require.NotContains(t, updatedPVC.Annotations, AnnSelectedNode) + + // The existing PVC is deleted so the exposer can bind a temporary PVC to the PV. + _, err = pvcRIA.kubeClient.CoreV1().PersistentVolumeClaims("velero").Get(t.Context(), "testPVC", metav1.GetOptions{}) + require.True(t, apierrors.IsNotFound(err)) + + // A DataDownload with the in-place restoreType referencing the existing PV is created. + dataDownloadList := new(velerov2alpha1.DataDownloadList) + require.NoError(t, pvcRIA.crClient.List(t.Context(), dataDownloadList, &crclient.ListOptions{})) + require.Len(t, dataDownloadList.Items, 1) + require.Equal(t, "full", dataDownloadList.Items[0].Spec.RestoreType) + require.Equal(t, "testPV", dataDownloadList.Items[0].Spec.TargetVolume.PV) +} + func TestPVCAppliesTo(t *testing.T) { p := pvcRestoreItemAction{ log: logrus.StandardLogger(), @@ -558,6 +770,8 @@ func TestNewPvcRestoreItemAction(t *testing.T) { f1 := &factorymocks.Factory{} f1.On("KubebuilderClient").Return(crClient, nil) + f1.On("KubeClient").Return(nil, nil) + f1.On("ClientConfig").Return(&rest.Config{}, nil) plugin1 := NewPvcRestoreItemAction(f1) _, err1 := plugin1(logger) require.NoError(t, err1) diff --git a/pkg/restore/actions/csi/volumesnapshot_action.go b/pkg/restore/actions/csi/volumesnapshot_action.go index da5d4d281..13b7cb246 100644 --- a/pkg/restore/actions/csi/volumesnapshot_action.go +++ b/pkg/restore/actions/csi/volumesnapshot_action.go @@ -66,6 +66,9 @@ func resetVolumeSnapshotSpecForRestore(vs *snapshotv1api.VolumeSnapshot, vscName } func resetVolumeSnapshotAnnotation(vs *snapshotv1api.VolumeSnapshot) { + if vs.ObjectMeta.Annotations == nil { + vs.ObjectMeta.Annotations = make(map[string]string) + } vs.ObjectMeta.Annotations[velerov1api.VSCDeletionPolicyAnnotation] = string(snapshotv1api.VolumeSnapshotContentRetain) } @@ -282,12 +285,6 @@ func (p *volumeSnapshotRestoreItemAction) Execute( vs.Namespace, vs.Name) } - vsMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&vs) - if err != nil { - p.log.Errorf("Fail to convert VS %s to unstructured", vs.Namespace+"/"+vs.Name) - return nil, errors.WithStack(err) - } - if vsFromBackup.Status == nil || vsFromBackup.Status.BoundVolumeSnapshotContentName == nil { p.log.Errorf("VS %s doesn't have bound VSC", vsFromBackup.Name) @@ -299,6 +296,21 @@ func (p *volumeSnapshotRestoreItemAction) Execute( Name: *vsFromBackup.Status.BoundVolumeSnapshotContentName, } + // Force-restore the bound VSC even when restore resource filters would + // otherwise exclude it (mirrors backup-side must-include for CSI deps). + annotations := vs.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + vs.SetAnnotations(annotations) + + vsMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&vs) + if err != nil { + p.log.Errorf("Fail to convert VS %s to unstructured", vs.Namespace+"/"+vs.Name) + return nil, errors.WithStack(err) + } + p.log.Infof(`Returning from VolumeSnapshotRestoreItemAction with VolumeSnapshotContent in additionalItems`) diff --git a/pkg/restore/actions/csi/volumesnapshot_action_test.go b/pkg/restore/actions/csi/volumesnapshot_action_test.go index de3e592c0..9d72971d0 100644 --- a/pkg/restore/actions/csi/volumesnapshot_action_test.go +++ b/pkg/restore/actions/csi/volumesnapshot_action_test.go @@ -103,6 +103,26 @@ func TestResetVolumeSnapshotSpecForRestore(t *testing.T) { } } +func TestResetVolumeSnapshotAnnotation(t *testing.T) { + t.Run("should set deletion policy annotation when annotations is nil", func(t *testing.T) { + vs := snapshotv1api.VolumeSnapshot{} + resetVolumeSnapshotAnnotation(&vs) + assert.NotNil(t, vs.ObjectMeta.Annotations) + assert.Equal(t, string(snapshotv1api.VolumeSnapshotContentRetain), vs.ObjectMeta.Annotations[velerov1api.VSCDeletionPolicyAnnotation]) + }) + + t.Run("should preserve existing annotations and set deletion policy annotation", func(t *testing.T) { + vs := snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"foo": "bar"}, + }, + } + resetVolumeSnapshotAnnotation(&vs) + assert.Equal(t, "bar", vs.ObjectMeta.Annotations["foo"]) + assert.Equal(t, string(snapshotv1api.VolumeSnapshotContentRetain), vs.ObjectMeta.Annotations[velerov1api.VSCDeletionPolicyAnnotation]) + }) +} + func TestVSExecute(t *testing.T) { newVscName := util.GenerateSha256FromRestoreUIDAndVsName("restoreUID", "vsName") tests := []struct { @@ -145,6 +165,18 @@ func TestVSExecute(t *testing.T) { expectErr: false, expectedVS: builder.ForVolumeSnapshot("ns", "test").SourceVolumeSnapshotContentName(newVscName).Result(), }, + { + name: "Normal case with nil VS annotations, VSC should be created", + vs: builder.ForVolumeSnapshot("ns", "vsName"). + SourceVolumeSnapshotContentName(newVscName). + VolumeSnapshotClass("vscClass"). + Status(). + BoundVolumeSnapshotContentName("vscName"). + Result(), + restore: builder.ForRestore("velero", "restore").ObjectMeta(builder.WithUID("restoreUID")).Result(), + expectErr: false, + expectedVS: builder.ForVolumeSnapshot("ns", "test").SourceVolumeSnapshotContentName(newVscName).Result(), + }, } for _, test := range tests { @@ -184,6 +216,10 @@ func TestVSExecute(t *testing.T) { require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured( result.UpdatedItem.UnstructuredContent(), &vs)) require.Equal(t, test.expectedVS.Spec, vs.Spec) + require.Equal(t, "true", vs.GetAnnotations()[velerov1api.MustIncludeAdditionalItemRestoreAnnotation]) + require.Len(t, result.AdditionalItems, 1) + require.Equal(t, "volumesnapshotcontents.snapshot.storage.k8s.io", result.AdditionalItems[0].GroupResource.String()) + require.Equal(t, "vscName", result.AdditionalItems[0].Name) } }) } diff --git a/pkg/restore/actions/dataupload_retrieve_action.go b/pkg/restore/actions/dataupload_retrieve_action.go index 77e4766f5..27db07471 100644 --- a/pkg/restore/actions/dataupload_retrieve_action.go +++ b/pkg/restore/actions/dataupload_retrieve_action.go @@ -82,6 +82,9 @@ func (d *DataUploadRetrieveAction) Execute(input *velero.RestoreItemActionExecut NodeOS: dataUpload.Status.NodeOS, FSType: dataUpload.Spec.SourceFSType, } + if dataUpload.Spec.CSISnapshot != nil { + dataUploadResult.SnapshotClass = dataUpload.Spec.CSISnapshot.SnapshotClass + } jsonBytes, err := json.Marshal(dataUploadResult) if err != nil { diff --git a/pkg/restore/actions/dataupload_retrieve_action_test.go b/pkg/restore/actions/dataupload_retrieve_action_test.go index 64be241bf..33a46a0a3 100644 --- a/pkg/restore/actions/dataupload_retrieve_action_test.go +++ b/pkg/restore/actions/dataupload_retrieve_action_test.go @@ -66,6 +66,29 @@ func TestDataUploadRetrieveActionExectue(t *testing.T) { }, expectedDataUploadResult: builder.ForConfigMap("velero", "").ObjectMeta(builder.WithGenerateName("testDU-"), builder.WithLabels(velerov1.PVCNamespaceNameLabel, "testNamespace.testPVC", velerov1.RestoreUIDLabel, "testingUID", velerov1.ResourceUsageLabel, string(velerov1.VeleroResourceUsageDataUploadResult))).Data("testingUID", `{"backupStorageLocation":"testLocation","snapshotID":"fake-id","sourceNamespace":"testNamespace","snapshotSize":1000}`).Result(), }, + { + name: "DataUploadRetrieve Action test with optional fields", + dataUpload: func() *velerov2alpha1.DataUpload { + du := builder.ForDataUpload("velero", "testDU"). + SourceNamespace("testNamespace"). + SourcePVC("testPVC"). + SnapshotID("fake-id"). + TotalBytes(1000). + DataMover("velero"). + NodeOS("linux"). + CSISnapshot(&velerov2alpha1.CSISnapshotSpec{SnapshotClass: "testClass"}). + Result() + du.Status.DataMoverResult = &map[string]string{"key": "value"} + du.Spec.SourceFSType = "ext4" + return du + }(), + restore: builder.ForRestore("velero", "testRestore").ObjectMeta(builder.WithUID("testingUID")).Backup("testBackup").Result(), + runtimeScheme: scheme, + veleroObjs: []runtime.Object{ + builder.ForBackup("velero", "testBackup").StorageLocation("testLocation").Result(), + }, + expectedDataUploadResult: builder.ForConfigMap("velero", "").ObjectMeta(builder.WithGenerateName("testDU-"), builder.WithLabels(velerov1.PVCNamespaceNameLabel, "testNamespace.testPVC", velerov1.RestoreUIDLabel, "testingUID", velerov1.ResourceUsageLabel, string(velerov1.VeleroResourceUsageDataUploadResult))).Data("testingUID", `{"backupStorageLocation":"testLocation","datamover":"velero","snapshotID":"fake-id","sourceNamespace":"testNamespace","dataMoverResult":{"key":"value"},"nodeOS":"linux","snapshotSize":1000,"fsType":"ext4","snapshotClass":"testClass"}`).Result(), + }, { name: "Long source namespace and PVC name should also work", dataUpload: builder.ForDataUpload("velero", "testDU").SourceNamespace("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1").SourcePVC("kibishii-data-kibishii-deployment-0").Result(), diff --git a/pkg/restore/actions/pod_volume_restore_action.go b/pkg/restore/actions/pod_volume_restore_action.go index 5f2b3db3e..cbfcbfb35 100644 --- a/pkg/restore/actions/pod_volume_restore_action.go +++ b/pkg/restore/actions/pod_volume_restore_action.go @@ -198,6 +198,25 @@ func (a *PodVolumeRestoreAction) Execute(input *velero.RestoreItemActionExecuteI securityContext = *pod.Spec.Containers[0].SecurityContext.DeepCopy() securityContextSet = true } + // if no configmap or container-level securityContext is set, fall back to the pod-level + // spec.securityContext runAsUser/runAsGroup: the workload's own identity is the one that + // wrote the restored files, so it's the one that can read them back + if !securityContextSet && pod.Spec.SecurityContext != nil && + (pod.Spec.SecurityContext.RunAsUser != nil || pod.Spec.SecurityContext.RunAsGroup != nil) { + securityContext = defaultSecurityCtx() + if pod.Spec.SecurityContext.RunAsUser != nil { + securityContext.RunAsUser = pod.Spec.SecurityContext.RunAsUser + // defaultSecurityCtx() hardcodes RunAsNonRoot: true, which contradicts a pod-level + // RunAsUser of 0 (root); defer to the pod's own RunAsNonRoot setting in that case + if *pod.Spec.SecurityContext.RunAsUser == 0 { + securityContext.RunAsNonRoot = pod.Spec.SecurityContext.RunAsNonRoot + } + } + if pod.Spec.SecurityContext.RunAsGroup != nil { + securityContext.RunAsGroup = pod.Spec.SecurityContext.RunAsGroup + } + securityContextSet = true + } if !securityContextSet { securityContext = defaultSecurityCtx() } diff --git a/pkg/restore/actions/pod_volume_restore_action_test.go b/pkg/restore/actions/pod_volume_restore_action_test.go index bc9662ab7..614a5d1be 100644 --- a/pkg/restore/actions/pod_volume_restore_action_test.go +++ b/pkg/restore/actions/pod_volume_restore_action_test.go @@ -156,6 +156,155 @@ func TestPodVolumeRestoreActionExecute(t *testing.T) { defaultRestoreHelperImage := "velero/velero:v1.0" + podLevelUID := int64(999) + podLevelGID := int64(999) + podLevelSecurityContext := corev1api.SecurityContext{ + AllowPrivilegeEscalation: boolptr.False(), + Capabilities: &corev1api.Capabilities{ + Drop: []corev1api.Capability{"ALL"}, + }, + SeccompProfile: &corev1api.SeccompProfile{ + Type: corev1api.SeccompProfileTypeRuntimeDefault, + }, + RunAsUser: &podLevelUID, + RunAsGroup: &podLevelGID, + RunAsNonRoot: boolptr.True(), + } + + podWithPodLevelSecurityContext := builder.ForPod("ns-1", "my-pod"). + ObjectMeta(builder.WithAnnotations("snapshot.velero.io/myvol", "")). + Volumes( + builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), + ). + Result() + podWithPodLevelSecurityContext.Spec.SecurityContext = &corev1api.PodSecurityContext{RunAsUser: &podLevelUID, RunAsGroup: &podLevelGID} + + wantPodWithPodLevelSecurityContext := builder.ForPod("ns-1", "my-pod"). + ObjectMeta(builder.WithAnnotations("snapshot.velero.io/myvol", "")). + Volumes( + builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), + ). + InitContainers( + newRestoreInitContainerBuilder(defaultRestoreHelperImage, ""). + Resources(&resourceReqs). + SecurityContext(&podLevelSecurityContext). + VolumeMounts(builder.ForVolumeMount("myvol", "/restores/myvol").Result()). + Command([]string{"/velero-restore-helper"}).Result()). + Result() + wantPodWithPodLevelSecurityContext.Spec.SecurityContext = &corev1api.PodSecurityContext{RunAsUser: &podLevelUID, RunAsGroup: &podLevelGID} + + podLevelRootUID := int64(0) + podLevelRootSecurityContext := corev1api.SecurityContext{ + AllowPrivilegeEscalation: boolptr.False(), + Capabilities: &corev1api.Capabilities{ + Drop: []corev1api.Capability{"ALL"}, + }, + SeccompProfile: &corev1api.SeccompProfile{ + Type: corev1api.SeccompProfileTypeRuntimeDefault, + }, + RunAsUser: &podLevelRootUID, + } + + podWithPodLevelRootSecurityContext := builder.ForPod("ns-1", "my-pod"). + ObjectMeta(builder.WithAnnotations("snapshot.velero.io/myvol", "")). + Volumes( + builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), + ). + Result() + podWithPodLevelRootSecurityContext.Spec.SecurityContext = &corev1api.PodSecurityContext{RunAsUser: &podLevelRootUID} + + wantPodWithPodLevelRootSecurityContext := builder.ForPod("ns-1", "my-pod"). + ObjectMeta(builder.WithAnnotations("snapshot.velero.io/myvol", "")). + Volumes( + builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), + ). + InitContainers( + newRestoreInitContainerBuilder(defaultRestoreHelperImage, ""). + Resources(&resourceReqs). + SecurityContext(&podLevelRootSecurityContext). + VolumeMounts(builder.ForVolumeMount("myvol", "/restores/myvol").Result()). + Command([]string{"/velero-restore-helper"}).Result()). + Result() + wantPodWithPodLevelRootSecurityContext.Spec.SecurityContext = &corev1api.PodSecurityContext{RunAsUser: &podLevelRootUID} + + podLevelGroupOnlyGID := int64(777) + podLevelGroupOnlySecurityContext := corev1api.SecurityContext{ + AllowPrivilegeEscalation: boolptr.False(), + Capabilities: &corev1api.Capabilities{ + Drop: []corev1api.Capability{"ALL"}, + }, + SeccompProfile: &corev1api.SeccompProfile{ + Type: corev1api.SeccompProfileTypeRuntimeDefault, + }, + RunAsUser: &id, + RunAsGroup: &podLevelGroupOnlyGID, + RunAsNonRoot: boolptr.True(), + } + + podWithPodLevelGroupOnlySecurityContext := builder.ForPod("ns-1", "my-pod"). + ObjectMeta(builder.WithAnnotations("snapshot.velero.io/myvol", "")). + Volumes( + builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), + ). + Result() + podWithPodLevelGroupOnlySecurityContext.Spec.SecurityContext = &corev1api.PodSecurityContext{RunAsGroup: &podLevelGroupOnlyGID} + + wantPodWithPodLevelGroupOnlySecurityContext := builder.ForPod("ns-1", "my-pod"). + ObjectMeta(builder.WithAnnotations("snapshot.velero.io/myvol", "")). + Volumes( + builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), + ). + InitContainers( + newRestoreInitContainerBuilder(defaultRestoreHelperImage, ""). + Resources(&resourceReqs). + SecurityContext(&podLevelGroupOnlySecurityContext). + VolumeMounts(builder.ForVolumeMount("myvol", "/restores/myvol").Result()). + Command([]string{"/velero-restore-helper"}).Result()). + Result() + wantPodWithPodLevelGroupOnlySecurityContext.Spec.SecurityContext = &corev1api.PodSecurityContext{RunAsGroup: &podLevelGroupOnlyGID} + + bothLevelsPodUID := int64(500) + bothLevelsContainerUID := int64(999) + bothLevelsContainerSecurityContext := corev1api.SecurityContext{ + AllowPrivilegeEscalation: boolptr.False(), + Capabilities: &corev1api.Capabilities{ + Drop: []corev1api.Capability{"ALL"}, + }, + SeccompProfile: &corev1api.SeccompProfile{ + Type: corev1api.SeccompProfileTypeRuntimeDefault, + }, + RunAsUser: &bothLevelsContainerUID, + RunAsNonRoot: boolptr.True(), + } + + podWithBothLevelsSecurityContext := builder.ForPod("ns-1", "my-pod"). + ObjectMeta(builder.WithAnnotations("snapshot.velero.io/myvol", "")). + Volumes( + builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), + ). + Containers( + builder.ForContainer("app-container", "app-image"). + SecurityContext(&bothLevelsContainerSecurityContext).Result()). + Result() + podWithBothLevelsSecurityContext.Spec.SecurityContext = &corev1api.PodSecurityContext{RunAsUser: &bothLevelsPodUID} + + wantPodWithBothLevelsSecurityContext := builder.ForPod("ns-1", "my-pod"). + ObjectMeta(builder.WithAnnotations("snapshot.velero.io/myvol", "")). + Volumes( + builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), + ). + Containers( + builder.ForContainer("app-container", "app-image"). + SecurityContext(&bothLevelsContainerSecurityContext).Result()). + InitContainers( + newRestoreInitContainerBuilder(defaultRestoreHelperImage, ""). + Resources(&resourceReqs). + SecurityContext(&bothLevelsContainerSecurityContext). + VolumeMounts(builder.ForVolumeMount("myvol", "/restores/myvol").Result()). + Command([]string{"/velero-restore-helper"}).Result()). + Result() + wantPodWithBothLevelsSecurityContext.Spec.SecurityContext = &corev1api.PodSecurityContext{RunAsUser: &bothLevelsPodUID} + tests := []struct { name string pod *corev1api.Pod @@ -350,6 +499,62 @@ func TestPodVolumeRestoreActionExecute(t *testing.T) { VolumeMounts(builder.ForVolumeMount("myvol", "/restores/myvol").Result()). Command([]string{"/velero-restore-helper"}).Result()).Result(), }, + { + name: "Restoring pod with pod-level securityContext (no container-level SecurityContext) uses pod-level runAsUser/runAsGroup for the restore initContainer", + pod: podWithPodLevelSecurityContext, + podVolumeBackups: []runtime.Object{ + builder.ForPodVolumeBackup(veleroNs, "pvb-1"). + PodName("my-pod"). + PodNamespace("ns-1"). + Volume("myvol"). + ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, backupName)). + SnapshotID("foo"). + Result(), + }, + want: wantPodWithPodLevelSecurityContext, + }, + { + name: "Restoring pod with pod-level securityContext.runAsUser=0 does not force RunAsNonRoot on the restore initContainer", + pod: podWithPodLevelRootSecurityContext, + podVolumeBackups: []runtime.Object{ + builder.ForPodVolumeBackup(veleroNs, "pvb-1"). + PodName("my-pod"). + PodNamespace("ns-1"). + Volume("myvol"). + ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, backupName)). + SnapshotID("foo"). + Result(), + }, + want: wantPodWithPodLevelRootSecurityContext, + }, + { + name: "Restoring pod with pod-level securityContext.runAsGroup only (no runAsUser) still applies the group to the restore initContainer", + pod: podWithPodLevelGroupOnlySecurityContext, + podVolumeBackups: []runtime.Object{ + builder.ForPodVolumeBackup(veleroNs, "pvb-1"). + PodName("my-pod"). + PodNamespace("ns-1"). + Volume("myvol"). + ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, backupName)). + SnapshotID("foo"). + Result(), + }, + want: wantPodWithPodLevelGroupOnlySecurityContext, + }, + { + name: "Restoring pod with both container-level and pod-level SecurityContext set uses the container-level SecurityContext for the restore initContainer (container-level takes priority)", + pod: podWithBothLevelsSecurityContext, + podVolumeBackups: []runtime.Object{ + builder.ForPodVolumeBackup(veleroNs, "pvb-1"). + PodName("my-pod"). + PodNamespace("ns-1"). + Volume("myvol"). + ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, backupName)). + SnapshotID("foo"). + Result(), + }, + want: wantPodWithBothLevelsSecurityContext, + }, { name: "pod volume backups in a different namespace are ignored when looking for matches due to namespace scoping", pod: builder.ForPod("ns-1", "my-pod"). diff --git a/pkg/restore/restore.go b/pkg/restore/restore.go index 8205accb9..aec181a97 100644 --- a/pkg/restore/restore.go +++ b/pkg/restore/restore.go @@ -87,7 +87,6 @@ const ObjectStatusRestoreAnnotationKey = "velero.io/restore-status" var resourceMustHave = []string{ "datauploads.velero.io", - "volumesnapshotcontents.snapshot.storage.k8s.io", } type VolumeSnapshotterGetter interface { @@ -638,21 +637,19 @@ func resolveRestoreNamespacedFilterPolicies( func resolveResourceFilter( rf resourcepolicies.ResourceFilter, ) (*resolvedResourceFilter, error) { - var selector labels.Selector - if len(rf.LabelSelector) > 0 { - var err error - selector, err = labels.ValidatedSelectorFromSet(labels.Set(rf.LabelSelector)) - if err != nil { - return nil, fmt.Errorf("invalid label selector in resource filter: %w", err) - } + selector, err := resourcepolicies.SelectorFromPolicyLabelSelector(rf.LabelSelector) + if err != nil { + return nil, fmt.Errorf("invalid label selector in resource filter: %w", err) } var orSelectors []labels.Selector for _, ols := range rf.OrLabelSelectors { - s, err := labels.ValidatedSelectorFromSet(labels.Set(ols)) + s, err := resourcepolicies.SelectorFromPolicyLabelSelector(ols) if err != nil { return nil, fmt.Errorf("invalid OR label selector in resource filter: %w", err) } - orSelectors = append(orSelectors, s) + if s != nil { + orSelectors = append(orSelectors, s) + } } var nameIE *collections.IncludesExcludes if len(rf.Names) > 0 || len(rf.ExcludedNames) > 0 { @@ -1007,6 +1004,19 @@ func (ctx *restoreContext) processSelectedResource( targetNS = namespace } } + + // Make sure the resource in the "resourceMustHave" set will always be created in the namespace where velero is installed. + if ctx.resourceMustHave.Has(groupResource.String()) && targetNS != "" && targetNS != ctx.restore.Namespace { + err := fmt.Errorf("resource %s/%s is must-have per velero internal setting, and is namespace-scoped, but its target namespace %q is not Velero's namespace %q", groupResource.String(), selectedItem.name, targetNS, ctx.restore.Namespace) + ctx.log.WithFields(logrus.Fields{ + "resource": groupResource.String(), + "name": selectedItem.name, + "targetNamespace": targetNS, + "veleroNamespace": ctx.restore.Namespace, + }).Error(err.Error()) + errs.Add(targetNS, err) + continue + } // If we don't know whether this namespace exists yet, attempt to create // it in order to ensure it exists. Try to get it from the backup tarball // (in order to get any backed-up metadata), but if we don't find it there, @@ -1014,11 +1024,13 @@ func (ctx *restoreContext) processSelectedResource( if namespace != "" && !existingNamespaces.Has(targetNS) { logger := ctx.log.WithField("namespace", namespace) - ns := getNamespace( - logger, - archive.GetItemFilePath(ctx.restoreDir, "namespaces", "", namespace), - targetNS, - ) + nsPath, err := archive.GetItemFilePath(ctx.restoreDir, "namespaces", "", namespace) + if err != nil { + errs.AddVeleroError(err) + continue + } + + ns := getNamespace(logger, nsPath, targetNS) _, nsCreated, err := kube.EnsureNamespaceExistsAndIsReady( ns, ctx.namespaceClient, @@ -1062,7 +1074,7 @@ func (ctx *restoreContext) processSelectedResource( continue } - w, e, _ := ctx.restoreItem(obj, groupResource, targetNS) + w, e, _ := ctx.restoreItem(obj, groupResource, targetNS, false) warnings.Merge(&w) errs.Merge(&e) processedItems++ @@ -1388,7 +1400,7 @@ func (ctx *restoreContext) getResource(groupResource schema.GroupResource, obj * return u, nil } -func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupResource schema.GroupResource, namespace string) (results.Result, results.Result, bool) { +func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupResource schema.GroupResource, namespace string, mustInclude bool) (results.Result, results.Result, bool) { warnings, errs := results.Result{}, results.Result{} // itemExists bool is used to determine whether to include this item in the "wait for additional items" list itemExists := false @@ -1405,31 +1417,51 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso // Check if group/resource should be restored. We need to do this here since // this method may be getting called for an additional item which is a group/resource // that's excluded. - if !ctx.resourceIncludesExcludes.ShouldInclude(groupResource.String()) && !ctx.resourceMustHave.Has(groupResource.String()) { - restoreLogger.Info("Not restoring item because resource is excluded") - return warnings, errs, itemExists - } - - // Check if namespace/cluster-scoped resource should be restored. We need - // to do this here since this method may be getting called for an additional - // item which is in a namespace that's excluded, or which is cluster-scoped - // and should be excluded. Note that we're checking the object's namespace ( - // via obj.GetNamespace()) instead of the namespace parameter, because we want - // to check the *original* namespace, not the remapped one if it's been remapped. // // Note: Additional items intentionally bypass fine-grained resource filter policies // (like per-namespace label/name selectors) to avoid breaking semantic dependencies, - // but they must still pass the global exclusions enforced below. - if namespace != "" { - if !ctx.namespaceIncludesExcludes.ShouldInclude(obj.GetNamespace()) && !ctx.resourceMustHave.Has(groupResource.String()) { - restoreLogger.Info("Not restoring item because namespace is excluded") + // but they must still pass the global exclusions enforced below unless mustInclude is set. + if mustInclude { + restoreLogger.Info("Skipping the resource/namespace exclusion checks because the item is marked as must-include") + } else { + if !ctx.resourceIncludesExcludes.ShouldInclude(groupResource.String()) && !ctx.resourceMustHave.Has(groupResource.String()) { + restoreLogger.Info("Not restoring item because resource is excluded") return warnings, errs, itemExists } + // Check if namespace/cluster-scoped resource should be restored. We need + // to do this here since this method may be getting called for an additional + // item which is in a namespace that's excluded, or which is cluster-scoped + // and should be excluded. Note that we're checking the object's namespace ( + // via obj.GetNamespace()) instead of the namespace parameter, because we want + // to check the *original* namespace, not the remapped one if it's been remapped. + if namespace != "" { + if !ctx.namespaceIncludesExcludes.ShouldInclude(obj.GetNamespace()) && !ctx.resourceMustHave.Has(groupResource.String()) { + restoreLogger.Info("Not restoring item because namespace is excluded") + return warnings, errs, itemExists + } + } else { + if boolptr.IsSetToFalse(ctx.restore.Spec.IncludeClusterResources) { + restoreLogger.Info("Not restoring item because it's cluster-scoped") + return warnings, errs, itemExists + } + } + } + + // Namespace creation runs unconditionally when namespace != "", regardless of + // mustInclude. This ensures target namespaces exist for additional items that + // bypass the namespace-exclusion check above. + if namespace != "" { // If the namespace scoped resource should be restored, ensure that the // namespace into which the resource is being restored into exists. // This is the *remapped* namespace that we are ensuring exists. - nsToEnsure := getNamespace(restoreLogger, archive.GetItemFilePath(ctx.restoreDir, "namespaces", "", obj.GetNamespace()), namespace) + nsPath, err := archive.GetItemFilePath(ctx.restoreDir, "namespaces", "", obj.GetNamespace()) + if err != nil { + errs.AddVeleroError(err) + return warnings, errs, itemExists + } + + nsToEnsure := getNamespace(restoreLogger, nsPath, namespace) _, nsCreated, err := kube.EnsureNamespaceExistsAndIsReady(nsToEnsure, ctx.namespaceClient, ctx.resourceTerminatingTimeout, ctx.resourceDeletionStatusTracker) if err != nil { errs.AddVeleroError(err) @@ -1444,11 +1476,6 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso } ctx.restoredItems[itemKey] = restoredItemStatus{action: ItemRestoreResultCreated, itemExists: true, createdName: nsToEnsure.Name} } - } else { - if boolptr.IsSetToFalse(ctx.restore.Spec.IncludeClusterResources) { - restoreLogger.Info("Not restoring item because it's cluster-scoped") - return warnings, errs, itemExists - } } // Make a copy of object retrieved from backup to make it available unchanged @@ -1609,6 +1636,19 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso return warnings, errs, itemExists } + // Strip any pre-existing Velero-internal in-place restore carrier annotation coming from + // the backup metadata before RestoreItemActions run. The carrier is only trusted when it + // is set by a RestoreItemAction (the PVC CSI RIA) during this restore; a stale carrier + // baked into the backup must not be translated into the Kubernetes "selected-node" + // annotation, which could pin a newly provisioned PVC to a stale node. + if annotations := obj.GetAnnotations(); annotations != nil { + if _, present := annotations[velerov1api.InplaceRestoreSelectedNodeAnnotation]; present { + restoreLogger.Infof("Removing pre-existing %q annotation from backup metadata", velerov1api.InplaceRestoreSelectedNodeAnnotation) + delete(annotations, velerov1api.InplaceRestoreSelectedNodeAnnotation) + obj.SetAnnotations(annotations) + } + } + restoreLogger.Infof("restore status includes excludes: %+v", ctx.resourceStatusIncludesExcludes) for _, action := range ctx.getApplicableActions(groupResource, namespace) { @@ -1670,9 +1710,34 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso obj = unstructuredObj + mustIncludeAdditionalItems := false + if annotations := obj.GetAnnotations(); annotations != nil { + if _, present := annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation]; present { + // Only the string value "true" enables the bypass. + if annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] == "true" { + mustIncludeAdditionalItems = true + restoreLogger.Info("RestoreItemAction marked additional items as must-include; bypassing resource/namespace exclusion checks for them") + } + // Always strip the annotation so it never lands on the cluster, + // regardless of whether the value enabled the bypass. + delete(annotations, velerov1api.MustIncludeAdditionalItemRestoreAnnotation) + obj.SetAnnotations(annotations) + } + } + var filteredAdditionalItems []velero.ResourceIdentifier for _, additionalItem := range executeOutput.AdditionalItems { - itemPath := archive.GetItemFilePath(ctx.restoreDir, additionalItem.GroupResource.String(), additionalItem.Namespace, additionalItem.Name) + itemPath, err := archive.GetItemFilePath(ctx.restoreDir, additionalItem.GroupResource.String(), additionalItem.Namespace, additionalItem.Name) + if err != nil { + restoreLogger.WithError(err).WithFields(logrus.Fields{ + "additionalResource": additionalItem.GroupResource.String(), + "additionalResourceNamespace": additionalItem.Namespace, + "additionalResourceName": additionalItem.Name, + }).Warn("unable to restore additional item") + warnings.Add(additionalItem.Namespace, err) + + continue + } if _, err := ctx.fileSystem.Stat(itemPath); err != nil { restoreLogger.WithError(err).WithFields(logrus.Fields{ @@ -1689,6 +1754,7 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso additionalObj, err := archive.Unmarshal(ctx.fileSystem, itemPath) if err != nil { errs.Add(namespace, errors.Wrapf(err, "error restoring additional item %s", additionalResourceID)) + continue } additionalItemNamespace := additionalItem.Namespace @@ -1698,7 +1764,7 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso } } - w, e, additionalItemExists := ctx.restoreItem(additionalObj, additionalItem.GroupResource, additionalItemNamespace) + w, e, additionalItemExists := ctx.restoreItem(additionalObj, additionalItem.GroupResource, additionalItemNamespace, mustIncludeAdditionalItems) if additionalItemExists { filteredAdditionalItems = append(filteredAdditionalItems, additionalItem) } @@ -1715,6 +1781,23 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso } } + // Translate the Velero-internal carrier annotation (set by the PVC CSI RestoreItemAction + // during an in-place volume data restore) back to the Kubernetes "selected-node" annotation. + // This runs after all RestoreItemActions so the result does not depend on the order in which + // the actions executed: the generic PVC RIA unconditionally strips the Kubernetes annotation, + // while the carrier annotation passes through untouched. The carrier itself is always + // stripped so it never lands on the cluster. + if annotations := obj.GetAnnotations(); annotations != nil { + if selectedNode, present := annotations[velerov1api.InplaceRestoreSelectedNodeAnnotation]; present { + if selectedNode != "" { + restoreLogger.Infof("Restoring %q annotation with value %q from in-place restore carrier annotation", kube.KubeAnnSelectedNode, selectedNode) + annotations[kube.KubeAnnSelectedNode] = selectedNode + } + delete(annotations, velerov1api.InplaceRestoreSelectedNodeAnnotation) + obj.SetAnnotations(annotations) + } + } + // This comes after running item actions because we have built-in actions that restore // a PVC's associated PV (if applicable). As part of the PV being restored, the 'pvsToProvision' // set may be inserted into, and this needs to happen *before* running the following block of logic. @@ -1890,7 +1973,7 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso if err != nil { warnings.Add(namespace, err) // check if there is existingResourcePolicy and if it is set to update policy - if len(ctx.restore.Spec.ExistingResourcePolicy) > 0 && ctx.restore.Spec.ExistingResourcePolicy == velerov1api.PolicyTypeUpdate { + if len(ctx.restore.Spec.ExistingResourcePolicy) > 0 && ctx.restore.Spec.ExistingResourcePolicy == velerov1api.ResourcePolicyTypeUpdate { // remove restore labels so that we apply the latest backup/restore names on the object via patch removeRestoreLabels(fromCluster) //try patching just the backup/restore labels @@ -1910,12 +1993,14 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso restoreLogger.Infof("restore API has resource policy defined %s, executing restore workflow accordingly for changed resource %s %s", resourcePolicy, fromCluster.GroupVersionKind().Kind, kube.NamespaceAndName(fromCluster)) // existingResourcePolicy is set as none, add warning - if resourcePolicy == velerov1api.PolicyTypeNone { + if resourcePolicy == velerov1api.ResourcePolicyTypeNone { e := errors.Errorf("could not restore, %s %q already exists. Warning: the in-cluster version is different than the backed-up version", obj.GetKind(), obj.GetName()) warnings.Add(namespace, e) + itemStatus.action = ItemRestoreResultSkipped + ctx.restoredItems[itemKey] = itemStatus // existingResourcePolicy is set as update, attempt patch on the resource and add warning if it fails - } else if resourcePolicy == velerov1api.PolicyTypeUpdate { + } else if resourcePolicy == velerov1api.ResourcePolicyTypeUpdate { // processing update as existingResourcePolicy warningsFromUpdateRP, errsFromUpdateRP := ctx.processUpdateResourcePolicy(fromCluster, fromClusterWithLabels, obj, namespace, resourceClient) if warningsFromUpdateRP.IsEmpty() && errsFromUpdateRP.IsEmpty() { @@ -1929,6 +2014,8 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso // Preserved Velero behavior when existingResourcePolicy is not specified by the user e := errors.Errorf("could not restore, %s:%s already exists. Warning: the in-cluster version is different than the backed-up version", obj.GetKind(), obj.GetName()) + itemStatus.action = ItemRestoreResultSkipped + ctx.restoredItems[itemKey] = itemStatus warnings.Add(namespace, e) } } @@ -1936,7 +2023,7 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso } //update backup/restore labels on the unchanged resources if existingResourcePolicy is set as update - if ctx.restore.Spec.ExistingResourcePolicy == velerov1api.PolicyTypeUpdate { + if ctx.restore.Spec.ExistingResourcePolicy == velerov1api.ResourcePolicyTypeUpdate { resourcePolicy := ctx.restore.Spec.ExistingResourcePolicy restoreLogger.Infof("restore API has resource policy defined %s, executing restore workflow accordingly for unchanged resource %s %s ", resourcePolicy, obj.GroupVersionKind().Kind, kube.NamespaceAndName(fromCluster)) // remove restore labels so that we apply the latest backup/restore names on the object via patch @@ -2028,10 +2115,7 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso return warnings, errs, itemExists } - // Do not create podvolumerestore when current restore excludes pv/pvc - if ctx.resourceIncludesExcludes.ShouldInclude(kuberesource.PersistentVolumeClaims.String()) && - ctx.resourceIncludesExcludes.ShouldInclude(kuberesource.PersistentVolumes.String()) && - len(podvolume.GetVolumeBackupsForPod(ctx.podVolumeBackups, pod, originalNamespace)) > 0 { + if len(podvolume.GetVolumeBackupsForPod(ctx.podVolumeBackups, pod, originalNamespace)) > 0 { restorePodVolumeBackups(ctx, createdObj, originalNamespace) } } @@ -2649,9 +2733,9 @@ func (ctx *restoreContext) getSelectedRestoreableItems(resource string, original // Peek-and-map logic for unresolvable kinds if rf == nil && len(items) > 0 { - peekPath := archive.GetItemFilePath(ctx.restoreDir, resourceForPath, originalNamespace, items[0]) - // Ignore unmarshal errors during peek; the main restore loop will catch and report them - if obj, err := archive.Unmarshal(ctx.fileSystem, peekPath); err == nil { + peekPath, pathErr := archive.GetItemFilePath(ctx.restoreDir, resourceForPath, originalNamespace, items[0]) + // Ignore path and unmarshal errors during peek; the main restore loop will catch and report them + if obj, err := archive.Unmarshal(ctx.fileSystem, peekPath); pathErr == nil && err == nil { actualKind := obj.GroupVersionKind().Kind for _, filter := range nsFilter.resourceFilterMap { for _, k := range filter.originalKinds { @@ -2692,9 +2776,9 @@ func (ctx *restoreContext) getSelectedRestoreableItems(resource string, original // Note: Unlike the namespaced path, this fallback is always reachable // because the main restore loop does not have a fast-path skip for // unlisted cluster-scoped resources. - peekPath := archive.GetItemFilePath(ctx.restoreDir, resourceForPath, originalNamespace, items[0]) - // Ignore unmarshal errors during peek; the main restore loop will catch and report them - if obj, err := archive.Unmarshal(ctx.fileSystem, peekPath); err == nil { + peekPath, pathErr := archive.GetItemFilePath(ctx.restoreDir, resourceForPath, originalNamespace, items[0]) + // Ignore path and unmarshal errors during peek; the main restore loop will catch and report them + if obj, err := archive.Unmarshal(ctx.fileSystem, peekPath); pathErr == nil && err == nil { actualKind := obj.GroupVersionKind().Kind for _, filter := range ctx.clusterScopedFilterMap { for _, k := range filter.originalKinds { @@ -2720,7 +2804,11 @@ func (ctx *restoreContext) getSelectedRestoreableItems(resource string, original } for _, item := range items { - itemPath := archive.GetItemFilePath(ctx.restoreDir, resourceForPath, originalNamespace, item) + itemPath, err := archive.GetItemFilePath(ctx.restoreDir, resourceForPath, originalNamespace, item) + if err != nil { + errs.Add(targetNamespace, err) + continue + } obj, err := archive.Unmarshal(ctx.fileSystem, itemPath) if err != nil { @@ -2785,7 +2873,7 @@ func (ctx *restoreContext) getSelectedRestoreableItems(resource string, original } if skipItem { - ctx.log.Infof("restore orSelector labels did not match, skipping restore of item: %s", skipItem, item) + ctx.log.Infof("restore orSelector labels did not match, skipping restore of item: %s", item) continue } } diff --git a/pkg/restore/restore_policies_test.go b/pkg/restore/restore_policies_test.go index a027f66aa..569d8923d 100644 --- a/pkg/restore/restore_policies_test.go +++ b/pkg/restore/restore_policies_test.go @@ -170,7 +170,8 @@ namespacedFilterPolicies: - kinds: - '*' labelSelector: - app: test + matchLabels: + app: test `, tarball: test.NewTarWriter(t). AddItems("pods", diff --git a/pkg/restore/restore_test.go b/pkg/restore/restore_test.go index fc4051387..5c75fff42 100644 --- a/pkg/restore/restore_test.go +++ b/pkg/restore/restore_test.go @@ -44,6 +44,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/dynamic" + k8sfake "k8s.io/client-go/kubernetes/fake" kubetesting "k8s.io/client-go/testing" "github.com/vmware-tanzu/velero/internal/volume" @@ -60,6 +61,7 @@ import ( vsv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/volumesnapshotter/v1" "github.com/vmware-tanzu/velero/pkg/podvolume" uploadermocks "github.com/vmware-tanzu/velero/pkg/podvolume/mocks" + riav1 "github.com/vmware-tanzu/velero/pkg/restore/actions" "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util/kube" @@ -754,6 +756,29 @@ func TestRestoreResourceFiltering(t *testing.T) { apiResources: []*test.APIResource{test.ServiceAccounts()}, want: map[*test.APIResource][]string{test.ServiceAccounts(): {"ns-1/sa-1"}}, }, + { + // Regression for #9957: VSC must not be force-included via resourceMustHave + // when the restore only selects unrelated resource types. + name: "volumesnapshotcontents are not force-included for selective resource restores", + restore: defaultRestore().IncludedResources("storageclasses").IncludeClusterResources(true).Result(), + backup: defaultBackup().Result(), + tarball: test.NewTarWriter(t). + AddItems("storageclasses.storage.k8s.io", + builder.ForStorageClass("sc-1").Result(), + ). + AddItems("volumesnapshotcontents.snapshot.storage.k8s.io", + builder.ForVolumeSnapshotContent("vsc-1").Result(), + ). + Done(), + apiResources: []*test.APIResource{ + test.StorageClasses(), + test.VolumeSnapshotContents(), + }, + want: map[*test.APIResource][]string{ + test.StorageClasses(): {"/sc-1"}, + test.VolumeSnapshotContents(): nil, + }, + }, } for _, tc := range tests { @@ -790,6 +815,87 @@ func TestRestoreResourceFiltering(t *testing.T) { } } +func TestRestoreMustHaveResourceNamespaceEnforcement(t *testing.T) { + tests := []struct { + name string + restore *velerov1api.Restore + backup *velerov1api.Backup + apiResources []*test.APIResource + tarball io.Reader + want map[*test.APIResource][]string + expectError bool + }{ + { + name: "resourceMustHave item in velero namespace is restored", + restore: defaultRestore().IncludedNamespaces("velero").Result(), + backup: defaultBackup().Result(), + tarball: test.NewTarWriter(t). + AddItems("datauploads.velero.io", + builder.ForDataUpload("velero", "du-1").Result(), + ). + Done(), + apiResources: []*test.APIResource{ + test.DataUploads(), + }, + want: map[*test.APIResource][]string{ + test.DataUploads(): {"velero/du-1"}, + }, + expectError: false, + }, + { + name: "resourceMustHave item outside velero namespace is rejected and produces error", + restore: defaultRestore().IncludedNamespaces("app-foo").Result(), + backup: defaultBackup().Result(), + tarball: test.NewTarWriter(t). + AddItems("datauploads.velero.io", + builder.ForDataUpload("attacker-ns", "du-2").Result(), + ). + Done(), + apiResources: []*test.APIResource{ + test.DataUploads(), + }, + want: map[*test.APIResource][]string{ + test.DataUploads(): {}, + }, + expectError: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t) + + for _, r := range tc.apiResources { + h.DiscoveryClient.WithAPIResource(r) + } + require.NoError(t, h.restorer.discoveryHelper.Refresh()) + + resPolicies, err := resourcepolicies.GetResourcePoliciesFromRestore(t.Context(), tc.restore, h.restorer.kbClient, h.log) + require.NoError(t, err) + + data := &Request{ + Log: h.log, + Restore: tc.restore, + Backup: tc.backup, + BackupReader: tc.tarball, + ResPolicies: resPolicies, + } + _, errs := h.restorer.Restore( + data, + nil, + nil, + ) + + if tc.expectError { + assert.False(t, errs.IsEmpty(), "expected errors but got empty") + } else { + assert.True(t, errs.IsEmpty(), "expected no errors but got %v", errs) + } + assertAPIContents(t, h, tc.want) + }) + } +} + // TestRestoreNamespaceMapping runs restores with namespace mappings specified, // and verifies that the set of items created in the API are in the correct // namespaces. Validation is done by looking at the namespaces/names of the items @@ -1063,6 +1169,7 @@ func TestRestoreItems(t *testing.T) { apiResources []*test.APIResource tarball io.Reader want []*test.APIResource + wantWarnings Result expectedRestoreItems map[itemKey]restoredItemStatus disableInformer bool }{ @@ -1305,6 +1412,52 @@ func TestRestoreItems(t *testing.T) { test.Pods(builder.ForPod("ns-1", "sa-1").ObjectMeta(builder.WithLabels("velero.io/backup-name", "foo", "velero.io/restore-name", "bar")).Result()), }, }, + { + name: "mark item as skipped when pod exists in cluster and is different from backed up one, existing resource policy is none", + restore: defaultRestore().ExistingResourcePolicy("none").Result(), + backup: defaultBackup().Result(), + tarball: test.NewTarWriter(t). + AddItems("pods", builder.ForPod("ns-1", "pod-1").ObjectMeta(builder.WithLabels("app", "backed-up")).Result()). + Done(), + apiResources: []*test.APIResource{ + test.Pods(builder.ForPod("ns-1", "pod-1").ObjectMeta(builder.WithLabels("app", "in-cluster")).Result()), + }, + want: []*test.APIResource{ + test.Pods(builder.ForPod("ns-1", "pod-1").ObjectMeta(builder.WithLabels("app", "in-cluster")).Result()), + }, + wantWarnings: Result{ + Namespaces: map[string][]string{ + "ns-1": {"could not restore, Pod \"pod-1\" already exists. Warning: the in-cluster version is different than the backed-up version"}, + }, + }, + expectedRestoreItems: map[itemKey]restoredItemStatus{ + {resource: "v1/Namespace", namespace: "", name: "ns-1"}: {action: "created", itemExists: true, createdName: "ns-1"}, + {resource: "v1/Pod", namespace: "ns-1", name: "pod-1"}: {action: "skipped", itemExists: true}, + }, + }, + { + name: "mark item as skipped when pod exists in cluster and is different from backed up one, existing resource policy is not specified", + restore: defaultRestore().Result(), + backup: defaultBackup().Result(), + tarball: test.NewTarWriter(t). + AddItems("pods", builder.ForPod("ns-1", "pod-1").ObjectMeta(builder.WithLabels("app", "backed-up")).Result()). + Done(), + apiResources: []*test.APIResource{ + test.Pods(builder.ForPod("ns-1", "pod-1").ObjectMeta(builder.WithLabels("app", "in-cluster")).Result()), + }, + want: []*test.APIResource{ + test.Pods(builder.ForPod("ns-1", "pod-1").ObjectMeta(builder.WithLabels("app", "in-cluster")).Result()), + }, + wantWarnings: Result{ + Namespaces: map[string][]string{ + "ns-1": {"could not restore, Pod:pod-1 already exists. Warning: the in-cluster version is different than the backed-up version"}, + }, + }, + expectedRestoreItems: map[itemKey]restoredItemStatus{ + {resource: "v1/Namespace", namespace: "", name: "ns-1"}: {action: "created", itemExists: true, createdName: "ns-1"}, + {resource: "v1/Pod", namespace: "ns-1", name: "pod-1"}: {action: "skipped", itemExists: true}, + }, + }, { name: "service account secrets and image pull secrets are restored when service account already exists in cluster", restore: defaultRestore().Result(), @@ -1414,7 +1567,12 @@ func TestRestoreItems(t *testing.T) { nil, // volume snapshotter getter ) - assertEmptyResults(t, warnings, errs) + if tc.wantWarnings.IsEmpty() { + assertEmptyResults(t, warnings) + } else { + assertWantErrsOrWarnings(t, tc.wantWarnings, warnings) + } + assertEmptyResults(t, errs) assertRestoredItems(t, h, tc.want) if len(tc.expectedRestoreItems) > 0 { assert.Equal(t, tc.expectedRestoreItems, data.RestoredItems) @@ -2150,6 +2308,102 @@ func TestRestoreActionAdditionalItems(t *testing.T) { test.PVs(): nil, }, }, + { + name: "must-include annotation bypasses resource exclusion for additional items", + restore: defaultRestore().IncludedResources("pods").Result(), + backup: defaultBackup().Result(), + tarball: test.NewTarWriter(t). + AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). + AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result()). + Done(), + apiResources: []*test.APIResource{test.Pods(), test.PVs()}, + actions: []riav2.RestoreItemAction{ + &pluggableAction{ + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, + }, + }, nil + }, + }, + }, + want: map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-1"}, + test.PVs(): {"/pv-1"}, + }, + }, + { + name: "must-include annotation bypasses namespace exclusion for additional items", + restore: defaultRestore().IncludedNamespaces("ns-1").Result(), + backup: defaultBackup().Result(), + tarball: test.NewTarWriter(t).AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result()).Done(), + apiResources: []*test.APIResource{test.Pods()}, + actions: []riav2.RestoreItemAction{ + &pluggableAction{ + selector: velero.ResourceSelector{IncludedNamespaces: []string{"ns-1"}}, + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.Pods, Namespace: "ns-2", Name: "pod-2"}, + }, + }, nil + }, + }, + }, + want: map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-1", "ns-2/pod-2"}, + }, + }, + { + name: "must-include annotation bypasses IncludeClusterResources=false for additional items", + restore: defaultRestore().IncludeClusterResources(false).Result(), + backup: defaultBackup().Result(), + tarball: test.NewTarWriter(t). + AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). + AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result()). + Done(), + apiResources: []*test.APIResource{test.Pods(), test.PVs()}, + actions: []riav2.RestoreItemAction{ + &pluggableAction{ + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, + }, + }, nil + }, + }, + }, + want: map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-1"}, + test.PVs(): {"/pv-1"}, + }, + }, } for _, tc := range tests { @@ -2180,6 +2434,605 @@ func TestRestoreActionAdditionalItems(t *testing.T) { } } +// TestRestoreActionAdditionalItemsInvalidJSON verifies that an additional item whose file +// exists in the backup but does not contain valid JSON is reported as an error and skipped, +// rather than being passed to restoreItem as a nil object. +// +// archive.Unmarshal returns (nil, err) for malformed JSON, and restoreItem dereferences its +// obj argument immediately, so failing to skip the item panics the restore reconciler. +func TestRestoreActionAdditionalItemsInvalidJSON(t *testing.T) { + h := newHarness(t) + + for _, r := range []*test.APIResource{test.Pods(), test.PVs()} { + h.AddItems(t, r) + } + + // pv-1.json exists so the Stat check passes, but its contents are not valid JSON. + tarball := test.NewTarWriter(t). + AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). + Add("resources/persistentvolumes/cluster/pv-1.json", []byte("not-json")). + Done() + + actions := []riav2.RestoreItemAction{ + &pluggableAction{ + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: input.Item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, + }, + }, nil + }, + }, + } + + data := &Request{ + Log: h.log, + Restore: defaultRestore().Result(), + Backup: defaultBackup().Result(), + BackupReader: tarball, + } + + // A nil additional item passed on to restoreItem panics here rather than failing. + warnings, errs := h.restorer.Restore(data, actions, nil) + + assertWantErrsOrWarnings(t, Result{}, warnings) + assertWantErrsOrWarnings(t, Result{ + Namespaces: map[string][]string{ + "ns-1": {"error restoring additional item persistentvolumes/pv-1"}, + }, + }, errs) + + // The item that triggered the action is still restored, so the loop continued. + assertAPIContents(t, h, map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-1"}, + test.PVs(): {}, + }) +} + +// TestRestoreMustIncludeAdditionalItems covers restore must-include edge cases beyond the +// basic filter-bypass cases in TestRestoreActionAdditionalItems. +func TestRestoreMustIncludeAdditionalItems(t *testing.T) { + t.Run("must-include annotation is stripped from the restored item", func(t *testing.T) { + h := newHarness(t) + h.AddItems(t, test.Pods()) + + data := &Request{ + Log: h.log, + Restore: defaultRestore().Result(), + Backup: defaultBackup().Result(), + BackupReader: test.NewTarWriter(t). + AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). + Done(), + } + warnings, errs := h.restorer.Restore( + data, + []riav2.RestoreItemAction{ + &pluggableAction{ + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + annotations["keep-me"] = "yes" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{UpdatedItem: item}, nil + }, + }, + }, + nil, + ) + + assertEmptyResults(t, warnings, errs) + + got, err := h.DynamicClient.Resource(test.Pods().GVR()).Namespace("ns-1").Get(t.Context(), "pod-1", metav1.GetOptions{}) + require.NoError(t, err) + annotations := got.GetAnnotations() + assert.NotContains(t, annotations, velerov1api.MustIncludeAdditionalItemRestoreAnnotation) + assert.Equal(t, "yes", annotations["keep-me"]) + }) + + t.Run("non-true must-include annotation is stripped without bypassing filters", func(t *testing.T) { + h := newHarness(t) + h.AddItems(t, test.Pods()) + h.AddItems(t, test.PVs()) + + data := &Request{ + Log: h.log, + Restore: defaultRestore().IncludedResources("pods").Result(), + Backup: defaultBackup().Result(), + BackupReader: test.NewTarWriter(t). + AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). + AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result()). + Done(), + } + warnings, errs := h.restorer.Restore( + data, + []riav2.RestoreItemAction{ + &pluggableAction{ + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "True" + annotations["keep-me"] = "yes" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, + }, + }, nil + }, + }, + }, + nil, + ) + + assertEmptyResults(t, warnings, errs) + assertAPIContents(t, h, map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-1"}, + test.PVs(): nil, + }) + + got, err := h.DynamicClient.Resource(test.Pods().GVR()).Namespace("ns-1").Get(t.Context(), "pod-1", metav1.GetOptions{}) + require.NoError(t, err) + annotations := got.GetAnnotations() + assert.NotContains(t, annotations, velerov1api.MustIncludeAdditionalItemRestoreAnnotation) + assert.Equal(t, "yes", annotations["keep-me"]) + }) + + t.Run("SkipRestore supersedes must-include annotation and skips additional items", func(t *testing.T) { + h := newHarness(t) + h.AddItems(t, test.Pods()) + h.AddItems(t, test.PVs()) + + data := &Request{ + Log: h.log, + Restore: defaultRestore().IncludedResources("pods").Result(), + Backup: defaultBackup().Result(), + BackupReader: test.NewTarWriter(t). + AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). + AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result()). + Done(), + } + warnings, errs := h.restorer.Restore( + data, + []riav2.RestoreItemAction{ + &pluggableAction{ + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + SkipRestore: true, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, + }, + }, nil + }, + }, + }, + nil, + ) + + assertEmptyResults(t, warnings, errs) + assertAPIContents(t, h, map[*test.APIResource][]string{ + test.Pods(): nil, + test.PVs(): nil, + }) + }) + + t.Run("must-include does not restore additional items missing from the backup tarball", func(t *testing.T) { + h := newHarness(t) + h.AddItems(t, test.Pods()) + h.AddItems(t, test.PVs()) + + data := &Request{ + Log: h.log, + Restore: defaultRestore().IncludedResources("pods").Result(), + Backup: defaultBackup().Result(), + BackupReader: test.NewTarWriter(t). + AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). + Done(), + } + warnings, errs := h.restorer.Restore( + data, + []riav2.RestoreItemAction{ + &pluggableAction{ + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.PersistentVolumes, Name: "pv-missing"}, + }, + }, nil + }, + }, + }, + nil, + ) + + assertEmptyResults(t, errs) + assertNonEmptyResults(t, "warning", warnings) + assertAPIContents(t, h, map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-1"}, + test.PVs(): nil, + }) + }) + + t.Run("transitive must-include requires each RIA level to re-set the annotation", func(t *testing.T) { + h := newHarness(t) + h.AddItems(t, test.Pods()) + h.AddItems(t, test.PVs()) + h.AddItems(t, test.PVCs()) + + data := &Request{ + Log: h.log, + Restore: defaultRestore().IncludedResources("pods").Result(), + Backup: defaultBackup().Result(), + BackupReader: test.NewTarWriter(t). + AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). + AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result()). + AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-2", "pvc-1").Result()). + Done(), + } + warnings, errs := h.restorer.Restore( + data, + []riav2.RestoreItemAction{ + // Parent pod RIA force-includes the excluded PV. + &pluggableAction{ + selector: velero.ResourceSelector{IncludedResources: []string{"pods"}}, + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, + }, + }, nil + }, + }, + // Child PV RIA also re-sets the annotation to force-include an excluded PVC. + &pluggableAction{ + selector: velero.ResourceSelector{IncludedResources: []string{"persistentvolumes"}}, + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.PersistentVolumeClaims, Namespace: "ns-2", Name: "pvc-1"}, + }, + }, nil + }, + }, + }, + nil, + ) + + assertEmptyResults(t, warnings, errs) + assertAPIContents(t, h, map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-1"}, + test.PVs(): {"/pv-1"}, + test.PVCs(): {"ns-2/pvc-1"}, + }) + }) + + t.Run("without re-annotating, transitive additional items still respect filters", func(t *testing.T) { + h := newHarness(t) + h.AddItems(t, test.Pods()) + h.AddItems(t, test.PVs()) + h.AddItems(t, test.PVCs()) + + data := &Request{ + Log: h.log, + Restore: defaultRestore().IncludedResources("pods").Result(), + Backup: defaultBackup().Result(), + BackupReader: test.NewTarWriter(t). + AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()). + AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result()). + AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-2", "pvc-1").Result()). + Done(), + } + warnings, errs := h.restorer.Restore( + data, + []riav2.RestoreItemAction{ + &pluggableAction{ + selector: velero.ResourceSelector{IncludedResources: []string{"pods"}}, + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, + }, + }, nil + }, + }, + // Child PV RIA returns an additional PVC but does NOT set must-include. + &pluggableAction{ + selector: velero.ResourceSelector{IncludedResources: []string{"persistentvolumes"}}, + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: input.Item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.PersistentVolumeClaims, Namespace: "ns-2", Name: "pvc-1"}, + }, + }, nil + }, + }, + }, + nil, + ) + + assertEmptyResults(t, warnings, errs) + assertAPIContents(t, h, map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-1"}, + test.PVs(): {"/pv-1"}, + test.PVCs(): nil, + }) + }) + + t.Run("VS must-include restores excluded VolumeSnapshotContent additional item", func(t *testing.T) { + h := newHarness(t) + h.AddItems(t, test.VolumeSnapshots()) + h.AddItems(t, test.VolumeSnapshotContents()) + + data := &Request{ + Log: h.log, + Restore: defaultRestore().IncludedResources("volumesnapshots.snapshot.storage.k8s.io").IncludeClusterResources(true).Result(), + Backup: defaultBackup().Result(), + BackupReader: test.NewTarWriter(t). + AddItems("volumesnapshots.snapshot.storage.k8s.io", builder.ForVolumeSnapshot("ns-1", "vs-1").Result()). + AddItems("volumesnapshotcontents.snapshot.storage.k8s.io", builder.ForVolumeSnapshotContent("vsc-1").Result()). + Done(), + } + warnings, errs := h.restorer.Restore( + data, + []riav2.RestoreItemAction{ + &pluggableAction{ + selector: velero.ResourceSelector{IncludedResources: []string{"volumesnapshots.snapshot.storage.k8s.io"}}, + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: item, + AdditionalItems: []velero.ResourceIdentifier{ + {GroupResource: kuberesource.VolumeSnapshotContents, Name: "vsc-1"}, + }, + }, nil + }, + }, + }, + nil, + ) + + assertEmptyResults(t, warnings, errs) + assertAPIContents(t, h, map[*test.APIResource][]string{ + test.VolumeSnapshots(): {"ns-1/vs-1"}, + test.VolumeSnapshotContents(): {"/vsc-1"}, + }) + }) +} + +// TestRestoreInplaceSelectedNodeCarrierAnnotation verifies the engine translates the +// Velero-internal in-place restore carrier annotation into the Kubernetes selected-node +// annotation after all RestoreItemActions have run, and always strips the carrier. +func TestRestoreInplaceSelectedNodeCarrierAnnotation(t *testing.T) { + t.Run("carrier annotation is translated to selected-node and stripped", func(t *testing.T) { + h := newHarness(t) + h.AddItems(t, test.PVCs()) + + data := &Request{ + Log: h.log, + Restore: defaultRestore().Result(), + Backup: defaultBackup().Result(), + BackupReader: test.NewTarWriter(t). + AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-1", "pvc-1").Result()). + Done(), + } + warnings, errs := h.restorer.Restore( + data, + []riav2.RestoreItemAction{ + // Simulates the PVC CSI RIA setting the carrier during an in-place restore. + &pluggableAction{ + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.InplaceRestoreSelectedNodeAnnotation] = "node-1" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{UpdatedItem: item}, nil + }, + }, + // The real generic PVC RIA (velero.io/pvc), which unconditionally strips the + // Kubernetes selected-node annotation. Running it after the carrier-setting + // action proves the carrier survives the real strip regardless of action order. + &pluggableAction{ + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + clientset := k8sfake.NewSimpleClientset() + return riav1.NewPVCAction( + h.log, + clientset.CoreV1().ConfigMaps("velero"), + clientset.CoreV1().Nodes(), + ).Execute(input) + }, + }, + }, + nil, + ) + + assertEmptyResults(t, warnings, errs) + + got, err := h.DynamicClient.Resource(test.PVCs().GVR()).Namespace("ns-1").Get(t.Context(), "pvc-1", metav1.GetOptions{}) + require.NoError(t, err) + annotations := got.GetAnnotations() + assert.Equal(t, "node-1", annotations["volume.kubernetes.io/selected-node"]) + assert.NotContains(t, annotations, velerov1api.InplaceRestoreSelectedNodeAnnotation) + }) + + t.Run("empty carrier annotation is stripped without setting selected-node", func(t *testing.T) { + h := newHarness(t) + h.AddItems(t, test.PVCs()) + + data := &Request{ + Log: h.log, + Restore: defaultRestore().Result(), + Backup: defaultBackup().Result(), + BackupReader: test.NewTarWriter(t). + AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-1", "pvc-1").Result()). + Done(), + } + warnings, errs := h.restorer.Restore( + data, + []riav2.RestoreItemAction{ + &pluggableAction{ + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[velerov1api.InplaceRestoreSelectedNodeAnnotation] = "" + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{UpdatedItem: item}, nil + }, + }, + }, + nil, + ) + + assertEmptyResults(t, warnings, errs) + + got, err := h.DynamicClient.Resource(test.PVCs().GVR()).Namespace("ns-1").Get(t.Context(), "pvc-1", metav1.GetOptions{}) + require.NoError(t, err) + annotations := got.GetAnnotations() + assert.NotContains(t, annotations, "volume.kubernetes.io/selected-node") + assert.NotContains(t, annotations, velerov1api.InplaceRestoreSelectedNodeAnnotation) + }) + + t.Run("no carrier annotation leaves selected-node stripped (PVC-absent fallback)", func(t *testing.T) { + h := newHarness(t) + h.AddItems(t, test.PVCs()) + + data := &Request{ + Log: h.log, + Restore: defaultRestore().Result(), + Backup: defaultBackup().Result(), + BackupReader: test.NewTarWriter(t). + AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-1", "pvc-1"). + ObjectMeta(builder.WithAnnotations("volume.kubernetes.io/selected-node", "stale-node")).Result()). + Done(), + } + warnings, errs := h.restorer.Restore( + data, + []riav2.RestoreItemAction{ + // Simulates the generic PVC RIA stripping the annotation; no action sets the + // carrier (as when the target PVC does not exist and Velero falls back to + // provisioning a new PVC). + &pluggableAction{ + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + annotations := item.GetAnnotations() + delete(annotations, "volume.kubernetes.io/selected-node") + item.SetAnnotations(annotations) + return &velero.RestoreItemActionExecuteOutput{UpdatedItem: item}, nil + }, + }, + }, + nil, + ) + + assertEmptyResults(t, warnings, errs) + + got, err := h.DynamicClient.Resource(test.PVCs().GVR()).Namespace("ns-1").Get(t.Context(), "pvc-1", metav1.GetOptions{}) + require.NoError(t, err) + assert.NotContains(t, got.GetAnnotations(), "volume.kubernetes.io/selected-node") + }) + + t.Run("carrier annotation baked into backup metadata is not trusted when no action sets it", func(t *testing.T) { + h := newHarness(t) + h.AddItems(t, test.PVCs()) + + data := &Request{ + Log: h.log, + Restore: defaultRestore().Result(), + Backup: defaultBackup().Result(), + BackupReader: test.NewTarWriter(t). + AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-1", "pvc-1"). + ObjectMeta(builder.WithAnnotations(velerov1api.InplaceRestoreSelectedNodeAnnotation, "stale-node")).Result()). + Done(), + } + warnings, errs := h.restorer.Restore( + data, + // No action sets the carrier during this restore (as in the PVC-absent fallback + // path where a new PVC is dynamically provisioned), so the carrier from the + // backup metadata must be stripped and never translated into selected-node. + []riav2.RestoreItemAction{ + &pluggableAction{ + executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + item := input.Item.(*unstructured.Unstructured) + // The stale carrier from the backup must already be gone before + // RestoreItemActions execute. + assert.NotContains(t, item.GetAnnotations(), velerov1api.InplaceRestoreSelectedNodeAnnotation) + return &velero.RestoreItemActionExecuteOutput{UpdatedItem: item}, nil + }, + }, + }, + nil, + ) + + assertEmptyResults(t, warnings, errs) + + got, err := h.DynamicClient.Resource(test.PVCs().GVR()).Namespace("ns-1").Get(t.Context(), "pvc-1", metav1.GetOptions{}) + require.NoError(t, err) + annotations := got.GetAnnotations() + assert.NotContains(t, annotations, "volume.kubernetes.io/selected-node") + assert.NotContains(t, annotations, velerov1api.InplaceRestoreSelectedNodeAnnotation) + }) +} + // TestShouldRestore runs the ShouldRestore function for various permutations of // existing/nonexisting/being-deleted PVs, PVCs, and namespaces, and verifies the // result/error matches expectations. diff --git a/pkg/test/api_server.go b/pkg/test/api_server.go index 63975014a..c69dc5926 100644 --- a/pkg/test/api_server.go +++ b/pkg/test/api_server.go @@ -58,6 +58,9 @@ func NewAPIServer(t *testing.T) *APIServer { {Group: "velero.io", Version: "v2alpha1", Resource: "datauploads"}: "DataUploadsList", {Group: "mygroup.io", Version: "v1", Resource: "mycustomkinds"}: "MyCustomKindList", {Group: "mygroup.io", Version: "v1", Resource: "myclustercustomkinds"}: "MyClusterCustomKindList", + {Group: "storage.k8s.io", Version: "v1", Resource: "storageclasses"}: "StorageClassList", + {Group: "snapshot.storage.k8s.io", Version: "v1", Resource: "volumesnapshots"}: "VolumeSnapshotList", + {Group: "snapshot.storage.k8s.io", Version: "v1", Resource: "volumesnapshotcontents"}: "VolumeSnapshotContentList", }) discoveryClient = &DiscoveryClient{FakeDiscovery: kubeClient.Discovery().(*discoveryfake.FakeDiscovery)} ) diff --git a/pkg/test/resources.go b/pkg/test/resources.go index fe2ad6352..975359d47 100644 --- a/pkg/test/resources.go +++ b/pkg/test/resources.go @@ -220,3 +220,37 @@ func DataUploads(items ...metav1.Object) *APIResource { Items: items, } } + +func StorageClasses(items ...metav1.Object) *APIResource { + return &APIResource{ + Group: "storage.k8s.io", + Version: "v1", + Name: "storageclasses", + ShortName: "sc", + Kind: "StorageClass", + Namespaced: false, + Items: items, + } +} + +func VolumeSnapshotContents(items ...metav1.Object) *APIResource { + return &APIResource{ + Group: "snapshot.storage.k8s.io", + Version: "v1", + Name: "volumesnapshotcontents", + Kind: "VolumeSnapshotContent", + Namespaced: false, + Items: items, + } +} + +func VolumeSnapshots(items ...metav1.Object) *APIResource { + return &APIResource{ + Group: "snapshot.storage.k8s.io", + Version: "v1", + Name: "volumesnapshots", + Kind: "VolumeSnapshot", + Namespaced: true, + Items: items, + } +} diff --git a/pkg/types/node_agent.go b/pkg/types/node_agent.go index 42fe06f58..16e50b283 100644 --- a/pkg/types/node_agent.go +++ b/pkg/types/node_agent.go @@ -57,13 +57,44 @@ type BackupPVC struct { // ignored if ReadOnly is false SPCNoRelabeling bool `json:"spcNoRelabeling,omitempty"` + // ReadWriteOncePod sets the backupPVC's access mode to ReadWriteOncePod so the kubelet can use + // mount-level SELinux labeling (-o context) instead of per-file relabeling, when the CSI driver + // advertises SELinux mount support. + // ignored if ReadOnly is true + ReadWriteOncePod bool `json:"readWriteOncePod,omitempty"` + // Annotations permits setting annotations for the backupPVC Annotations map[string]string `json:"annotations,omitempty"` + + // SecretNames is a list of secret names to copy from the source PVC namespace + // to the Velero namespace before creating the backupPVC. The secrets are deleted + // after the DataUpload completes. This is needed for CSI drivers that require + // namespace-scoped secrets for volume provisioning (e.g., encrypted volumes). + SecretNames []string `json:"secretNames,omitempty"` + + // ConfigMapNames is a list of configmap names to copy from the source PVC namespace + // to the Velero namespace before creating the backupPVC. The configmaps are deleted + // after the DataUpload completes. This is needed for CSI drivers that require + // namespace-scoped configmaps for volume provisioning (e.g., tenant-specific + // Vault connection overrides for encrypted volumes). + ConfigMapNames []string `json:"configMapNames,omitempty"` } type RestorePVC struct { // IgnoreDelayBinding indicates to ignore delay binding the restorePVC when it is in WaitForFirstConsumer mode IgnoreDelayBinding bool `json:"ignoreDelayBinding,omitempty"` + + // SecretNames is a list of secret names to copy from the target namespace to the + // Velero namespace before creating the restorePVC. The secrets are deleted after the + // DataDownload completes. This is needed for CSI drivers that require namespace-scoped + // secrets for volume provisioning (e.g., encrypted volumes). + SecretNames []string `json:"secretNames,omitempty"` + + // ConfigMapNames is a list of configmap names to copy from the target namespace to the + // Velero namespace before creating the restorePVC. The configmaps are deleted after the + // DataDownload completes. This is needed for CSI drivers that require namespace-scoped + // configmaps for volume provisioning (e.g., tenant-specific Vault connection overrides). + ConfigMapNames []string `json:"configMapNames,omitempty"` } type CachePVC struct { diff --git a/pkg/uploader/block/snapshot.go b/pkg/uploader/block/snapshot.go index e30f5c1bb..ff844d197 100644 --- a/pkg/uploader/block/snapshot.go +++ b/pkg/uploader/block/snapshot.go @@ -121,7 +121,10 @@ func snapshotSource( return "", 0, errors.Wrapf(err, "Failed to run uploader backup for si %v", source) } - snap.Tags = make(map[string]string) + if snap.Tags == nil { + snap.Tags = make(map[string]string) + } + snap.Tags[uploader.CBTChangeIDTag] = cbtSource.ChangeID snap.Tags[uploader.CBTVolumeIDTag] = cbtSource.VolumeID if snapshotTags != nil { @@ -146,6 +149,12 @@ func snapshotSource( func getParentBackupInfo(ctx context.Context, rep udmrepo.BackupRepo, forceFull bool, parentSnapshot string, volumeID string, realSource string, snapshotTags map[string]string, log logrus.FieldLogger) parentBackupInfo { var previous *udmrepo.Snapshot + + // parentID names whichever snapshot ended up being the parent. On the discovery + // branch the parentSnapshot parameter is empty by definition, so logging it there + // produces messages that describe a decision without naming the object it was about. + parentID := parentSnapshot + if !forceFull { if parentSnapshot != "" { snap, err := rep.GetSnapshot(ctx, udmrepo.ID(parentSnapshot)) @@ -163,6 +172,7 @@ func getParentBackupInfo(ctx context.Context, rep udmrepo.BackupRepo, forceFull log.WithError(err).Warn("Failed to search previous snapshot, fallback to full backup") } else { previous = &snap + parentID = string(snap.RootObject.ID) log.Infof("Using previous snapshot %s", snap.RootObject.ID) } } @@ -173,21 +183,21 @@ func getParentBackupInfo(ctx context.Context, rep udmrepo.BackupRepo, forceFull parentInfo := parentBackupInfo{} if previous != nil { if previous.Tags == nil { - log.Warnf("No tag from parent snapshot %s, fallback to full backup", parentSnapshot) + log.Warnf("No tag from parent snapshot %s, fallback to full backup", parentID) } else if previous.Tags[uploader.CBTChangeIDTag] == "" { - log.Warnf("No ChangeID tag from parent snapshot %s, fallback to full backup", parentSnapshot) + log.Warnf("No ChangeID tag from parent snapshot %s, fallback to full backup", parentID) } else if previous.Tags[uploader.CBTVolumeIDTag] == "" { - log.Warnf("No VolumeID tag from parent snapshot %s, fallback to full backup", parentSnapshot) + log.Warnf("No VolumeID tag from parent snapshot %s, fallback to full backup", parentID) } else if previous.Tags[uploader.CBTVolumeIDTag] != volumeID { - log.Warnf("VolumeID %s from parent snapshot %s is not expected as %s, fallback to full backup", previous.Tags[uploader.CBTVolumeIDTag], parentSnapshot, volumeID) + log.Warnf("VolumeID %s from parent snapshot %s is not expected as %s, fallback to full backup", previous.Tags[uploader.CBTVolumeIDTag], parentID, volumeID) } else if obj, err := loadObjectFromSnapshot(ctx, rep, previous); err != nil { - log.WithError(err).Warnf("Failed to load object from parent snapshot %s, fallback to full backup", parentSnapshot) + log.WithError(err).Warnf("Failed to load object from parent snapshot %s, fallback to full backup", parentID) } else { parentInfo.parentObject = obj parentInfo.changeID = previous.Tags[uploader.CBTChangeIDTag] parentInfo.volumeID = previous.Tags[uploader.CBTVolumeIDTag] - log.Infof("Using parent snapshot %s, start time %v, end time %v, description %s", parentSnapshot, previous.StartTime, previous.EndTime, previous.Description) + log.Infof("Using parent snapshot %s, start time %v, end time %v, description %s", parentID, previous.StartTime, previous.EndTime, previous.Description) } } @@ -195,18 +205,44 @@ func getParentBackupInfo(ctx context.Context, rep udmrepo.BackupRepo, forceFull } // Restore restore specific sourcePath with given snapshotID and update progress -func Restore(ctx context.Context, blkUp Uploader, rep udmrepo.BackupRepo, snapshotID, dest string, uploaderCfg map[string]string, log logrus.FieldLogger) (int64, error) { +func Restore(ctx context.Context, blkUp Uploader, rep udmrepo.BackupRepo, snapshotID, dest string, incremental bool, cbtSource cbtservice.SourceInfo, cbtService cbtservice.Service, uploaderCfg map[string]string, log logrus.FieldLogger) (int64, error) { log.Info("Start to restore...") snapshot, err := rep.GetSnapshot(ctx, udmrepo.ID(snapshotID)) if err != nil { return 0, errors.Wrapf(err, "Unable to load snapshot %v", snapshotID) } + log.Infof("Restore from snapshot %s, incremental %v, cbt source %v, description %s, created time %v, tags %v", snapshotID, incremental, cbtSource, snapshot.Description, snapshot.EndTime, snapshot.Tags) - log.Infof("Restore from snapshot %s, description %s, created time %v, tags %v", snapshotID, snapshot.Description, snapshot.EndTime, snapshot.Tags) + var volumeSnapshot, changeID, volumeID string + if incremental { + if snapshot.Tags == nil { + log.Warnf("No tag from snapshot %s, fallback to full restore", snapshotID) + incremental = false + } else if snapshot.Tags[uploader.CBTChangeIDTag] == "" { + log.Warnf("No ChangeID tag from snapshot %s, fallback to full restore", snapshotID) + incremental = false + } else if snapshot.Tags[uploader.CBTVolumeIDTag] == "" { + log.Warnf("No VolumeID tag from snapshot %s, fallback to full restore", snapshotID) + incremental = false + } else if snapshot.Tags[uploader.CBTVolumeIDTag] != cbtSource.VolumeID { + log.Warnf("VolumeID %s from snapshot %s is not expected as %s, fallback to full restore", snapshot.Tags[uploader.CBTVolumeIDTag], snapshotID, cbtSource.VolumeID) + incremental = false + } else { + volumeSnapshot = cbtSource.Snapshot + changeID = snapshot.Tags[uploader.CBTChangeIDTag] + volumeID = snapshot.Tags[uploader.CBTVolumeIDTag] + } + } - bitmap := cbt.NewBitmap(blockSize, uint64(snapshot.TotalSize), "", "", "") - bitmap.SetFull() + bitmap := cbt.NewBitmap(blockSize, uint64(snapshot.TotalSize), volumeSnapshot, changeID, volumeID) + if incremental { + if err = cbt.SetBitmapOrFull(ctx, cbtService, bitmap); err != nil { + log.WithError(err).Warnf("Failed to create CBT with source %v, fallback to full restore", cbtSource) + } + } else { + bitmap.SetFull() + } destPath, err := filepath.Abs(dest) if err != nil { @@ -222,12 +258,22 @@ func Restore(ctx context.Context, blkUp Uploader, rep udmrepo.BackupRepo, snapsh defer destDev.Close() - size, err := blkUp.Restore(snapshot, destInfo{dev: destDev, path: destPath}, bitmap.Iterator(), uploaderCfg) + destSize, err := destDev.Seek(0, io.SeekEnd) + if err != nil { + return 0, errors.Wrapf(err, "error getting length of block device %s", dest) + } + + _, err = destDev.Seek(0, io.SeekStart) + if err != nil { + return 0, errors.Wrapf(err, "error reset pos of block device %s", dest) + } + + _, totalSize, err := blkUp.Restore(snapshot, destInfo{dev: destDev, path: destPath, size: destSize}, bitmap.Iterator(), uploaderCfg) if err != nil { return 0, errors.Wrapf(err, "error restoring to block dev %s", destPath) } - return size, nil + return totalSize, nil } func findPreviousSnapshot(ctx context.Context, rep udmrepo.BackupRepo, path string, snapshotTags map[string]string, noLaterThan *time.Time, log logrus.FieldLogger) (udmrepo.Snapshot, error) { diff --git a/pkg/uploader/block/snapshot_test.go b/pkg/uploader/block/snapshot_test.go index 8f6338311..d7e7d2ee2 100644 --- a/pkg/uploader/block/snapshot_test.go +++ b/pkg/uploader/block/snapshot_test.go @@ -21,16 +21,19 @@ package block import ( "context" "os" + "strings" "testing" "time" "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" + logrustest "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/velero/pkg/cbtservice" + cbtservicemocks "github.com/vmware-tanzu/velero/pkg/cbtservice/mocks" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" udmrepomocks "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/mocks" "github.com/vmware-tanzu/velero/pkg/uploader" @@ -46,9 +49,9 @@ func (m *mockUploader) Backup(src sourceInfo, parent udmrepo.ID, iter cbttypes.I return args.Get(0).(udmrepo.Snapshot), args.Get(1).(int64), args.Error(2) } -func (m *mockUploader) Restore(snap udmrepo.Snapshot, dest destInfo, iter cbttypes.Iterator, cfg map[string]string) (int64, error) { +func (m *mockUploader) Restore(snap udmrepo.Snapshot, dest destInfo, iter cbttypes.Iterator, cfg map[string]string) (int64, int64, error) { args := m.Called(snap, dest, iter, cfg) - return args.Get(0).(int64), args.Error(1) + return args.Get(0).(int64), args.Get(1).(int64), args.Error(2) } func testLog() logrus.FieldLogger { @@ -121,6 +124,23 @@ func TestBackup(t *testing.T) { assert.Positive(t, info.Size) }, }, + { + name: "success with CBT", + setupOpenDev: func(t *testing.T) *os.File { + t.Helper() + return tempFile(t, "test-block-data") + }, + setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { + blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(8), nil) + repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-001"), nil) + repo.On("Flush", mock.Anything).Return(nil) + }, + checkInfo: func(t *testing.T, info uploader.SnapshotInfo) { + t.Helper() + assert.Equal(t, "snap-001", info.ID) + }, + }, } for _, tc := range testCases { @@ -184,6 +204,7 @@ func TestSnapshotSource(t *testing.T) { expectedErrStr string expectedSnapID string expectedSize int64 + cbtService func(t *testing.T) cbtservice.Service }{ { name: "uploader Backup error", @@ -216,7 +237,10 @@ func TestSnapshotSource(t *testing.T) { { name: "success with nil cbtService falls back to full bitmap", setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { - blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + blkup.On("Backup", mock.Anything, mock.Anything, mock.MatchedBy(func(iter cbttypes.Iterator) bool { + // In full mode, the iterator should cover the whole range if it's a full backup + return iter != nil + }), mock.Anything). Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(512), nil) repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-success"), nil) repo.On("Flush", mock.Anything).Return(nil) @@ -239,6 +263,46 @@ func TestSnapshotSource(t *testing.T) { }, expectedSnapID: "snap-tags", }, + { + name: "success with cbtService getting allocated blocks", + cbtService: func(t *testing.T) cbtservice.Service { + t.Helper() + m := cbtservicemocks.NewService(t) + m.On("GetAllocatedBlocks", mock.Anything, "snap-1", mock.Anything). + Run(func(args mock.Arguments) { + record := args.Get(2).(func([]cbtservice.Range) error) + record([]cbtservice.Range{{Offset: 0, Length: 1024}}) + }).Return(nil) + return m + }, + setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { + blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(1024), nil) + repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-cbt-alloc"), nil) + repo.On("Flush", mock.Anything).Return(nil) + }, + expectedSnapID: "snap-cbt-alloc", + expectedSize: 1024, + }, + { + name: "cbtService error falls back to full", + cbtService: func(t *testing.T) cbtservice.Service { + t.Helper() + m := cbtservicemocks.NewService(t) + m.On("GetAllocatedBlocks", mock.Anything, "snap-1", mock.Anything). + Return(errors.New("CBT error")) + return m + }, + setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { + // Should be called with parentObject as empty because of fallback + blkup.On("Backup", mock.Anything, udmrepo.ID(""), mock.Anything, mock.Anything). + Return(udmrepo.Snapshot{}, int64(2048), nil) + repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-cbt-fallback"), nil) + repo.On("Flush", mock.Anything).Return(nil) + }, + expectedSnapID: "snap-cbt-fallback", + expectedSize: 2048, + }, } for _, tc := range testCases { @@ -249,14 +313,19 @@ func TestSnapshotSource(t *testing.T) { tc.setupMocks(mockBlkup, mockRepo) - cbtSrc := cbtservice.SourceInfo{ChangeID: "cid-1", VolumeID: "vid-1"} + cbtSrc := cbtservice.SourceInfo{Snapshot: "snap-1", ChangeID: "cid-1", VolumeID: "vid-1"} snapshotTags := map[string]string{"custom": "val"} + var cbtSvc cbtservice.Service + if tc.cbtService != nil { + cbtSvc = tc.cbtService(t) + } + snapID, size, err := snapshotSource( ctx, mockRepo, mockBlkup, baseSource, true, "", - cbtSrc, nil, + cbtSrc, cbtSvc, snapshotTags, map[string]string{}, testLog(), "Block Uploader", ) @@ -275,6 +344,58 @@ func TestSnapshotSource(t *testing.T) { } } +// TestGetParentBackupInfoLogsDiscoveredParentID pins that the parent-selection messages +// name the snapshot they are about. On the discovery branch the parentSnapshot parameter +// is empty by definition, so logging it there emits "Using parent snapshot , start time ..." +// - a decision logged without the identifier needed to act on it. +func TestGetParentBackupInfoLogsDiscoveredParentID(t *testing.T) { + const volumeID = "vol-123" + const realSource = "/test/source" + const rootObj = "root-obj-42" + + snapshotTags := map[string]string{ + uploader.SnapshotRequesterTag: "test-requester", + uploader.SnapshotUploaderTag: uploader.BlockType, + } + + logger, hook := logrustest.NewNullLogger() + logger.SetLevel(logrus.DebugLevel) + + repo := udmrepomocks.NewBackupRepo(t) + repo.On("ListSnapshot", mock.Anything, realSource). + Return([]udmrepo.Snapshot{{ + RootObject: udmrepo.ObjectMetadata{ID: rootObj}, + Tags: map[string]string{ + uploader.CBTChangeIDTag: "cid-abc", + uploader.CBTVolumeIDTag: volumeID, + uploader.SnapshotRequesterTag: "test-requester", + uploader.SnapshotUploaderTag: uploader.BlockType, + }, + }}, nil) + repo.On("ReadMetadata", mock.Anything, udmrepo.ID(rootObj)). + Return(&udmrepo.Metadata{ + SubObjects: []udmrepo.ObjectMetadata{{ID: udmrepo.ID("parent-obj")}}, + }, nil) + + info := getParentBackupInfo( + context.Background(), repo, + false, "", // no explicit parent -> discovery branch + volumeID, realSource, snapshotTags, logger, + ) + + require.Equal(t, udmrepo.ID("parent-obj"), info.parentObject) + + var found bool + for _, entry := range hook.AllEntries() { + if strings.HasPrefix(entry.Message, "Using parent snapshot ") { + found = true + assert.Contains(t, entry.Message, rootObj, + "parent-selection message must name the discovered snapshot, got %q", entry.Message) + } + } + require.True(t, found, "expected a \"Using parent snapshot\" message") +} + func TestGetParentBackupInfo(t *testing.T) { const volumeID = "vol-123" const realSource = "/test/source" @@ -547,6 +668,9 @@ func TestRestore(t *testing.T) { testCases := []struct { name string + incremental bool + cbtSource cbtservice.SourceInfo + cbtService func(t *testing.T) cbtservice.Service setupMocks func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) setupOpenDev func(t *testing.T) *os.File expectedErrStr string @@ -574,7 +698,7 @@ func TestRestore(t *testing.T) { repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-001")). Return(storedSnap, nil) blkup.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(int64(0), errors.New("restore I/O error")) + Return(int64(0), int64(0), errors.New("restore I/O error")) }, setupOpenDev: func(t *testing.T) *os.File { t.Helper() @@ -583,12 +707,12 @@ func TestRestore(t *testing.T) { expectedErrStr: "error restoring to block dev", }, { - name: "success returns size", + name: "success returns size (full restore)", setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-001")). Return(storedSnap, nil) blkup.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(int64(4096), nil) + Return(int64(4096), int64(4096), nil) }, setupOpenDev: func(t *testing.T) *os.File { t.Helper() @@ -596,6 +720,102 @@ func TestRestore(t *testing.T) { }, expectedSize: 4096, }, + { + name: "incremental restore success", + incremental: true, + cbtSource: cbtservice.SourceInfo{Snapshot: "snap-cbt", VolumeID: "vol-1"}, + cbtService: func(t *testing.T) cbtservice.Service { + t.Helper() + m := cbtservicemocks.NewService(t) + m.On("GetChangedBlocks", mock.Anything, "snap-cbt", "cid-1", mock.Anything). + Run(func(args mock.Arguments) { + record := args.Get(3).(func([]cbtservice.Range) error) + record([]cbtservice.Range{{Offset: 0, Length: 512}}) + }).Return(nil) + return m + }, + setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { + snapWithTags := udmrepo.Snapshot{ + Tags: map[string]string{ + uploader.CBTChangeIDTag: "cid-1", + uploader.CBTVolumeIDTag: "vol-1", + }, + TotalSize: 1024, + } + repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-001")).Return(snapWithTags, nil) + blkup.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(int64(512), int64(512), nil) + }, + setupOpenDev: func(t *testing.T) *os.File { + t.Helper() + return tempFile(t, "") + }, + expectedSize: 512, + }, + { + name: "incremental restore fallback - missing tags", + incremental: true, + setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { + repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-001")).Return(storedSnap, nil) + blkup.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(int64(4096), int64(4096), nil) + }, + setupOpenDev: func(t *testing.T) *os.File { + t.Helper() + return tempFile(t, "") + }, + expectedSize: 4096, + }, + { + name: "incremental restore fallback - VolumeID mismatch", + incremental: true, + cbtSource: cbtservice.SourceInfo{VolumeID: "vol-actual"}, + setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { + snapWithTags := udmrepo.Snapshot{ + Tags: map[string]string{ + uploader.CBTChangeIDTag: "cid-1", + uploader.CBTVolumeIDTag: "vol-expected", + }, + } + repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-001")).Return(snapWithTags, nil) + blkup.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(int64(4096), int64(4096), nil) + }, + setupOpenDev: func(t *testing.T) *os.File { + t.Helper() + return tempFile(t, "") + }, + expectedSize: 4096, + }, + { + name: "incremental restore fallback - CBT service error", + incremental: true, + cbtSource: cbtservice.SourceInfo{Snapshot: "snap-cbt", VolumeID: "vol-1"}, + cbtService: func(t *testing.T) cbtservice.Service { + t.Helper() + m := cbtservicemocks.NewService(t) + m.On("GetChangedBlocks", mock.Anything, "snap-cbt", "cid-1", mock.Anything). + Return(errors.New("CBT error")) + return m + }, + setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { + snapWithTags := udmrepo.Snapshot{ + Tags: map[string]string{ + uploader.CBTChangeIDTag: "cid-1", + uploader.CBTVolumeIDTag: "vol-1", + }, + TotalSize: 1024, + } + repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-001")).Return(snapWithTags, nil) + blkup.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(int64(1024), int64(1024), nil) + }, + setupOpenDev: func(t *testing.T) *os.File { + t.Helper() + return tempFile(t, "") + }, + expectedSize: 1024, + }, } for _, tc := range testCases { @@ -617,7 +837,12 @@ func TestRestore(t *testing.T) { } } - size, err := Restore(ctx, mockBlkup, mockRepo, "snap-001", "/dev/sdb", map[string]string{}, testLog()) + var cbtSvc cbtservice.Service + if tc.cbtService != nil { + cbtSvc = tc.cbtService(t) + } + + size, err := Restore(ctx, mockBlkup, mockRepo, "snap-001", "/dev/sdb", tc.incremental, tc.cbtSource, cbtSvc, map[string]string{}, testLog()) if tc.expectedErrStr != "" { require.Error(t, err) diff --git a/pkg/uploader/block/uploader.go b/pkg/uploader/block/uploader.go index 75e913cb7..e2d464872 100644 --- a/pkg/uploader/block/uploader.go +++ b/pkg/uploader/block/uploader.go @@ -17,11 +17,16 @@ limitations under the License. package block import ( + "bytes" "context" + "fmt" "io" "os" "runtime" + "strconv" "strings" + "sync" + "time" "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" @@ -35,8 +40,9 @@ import ( var ErrCanceled = errors.New("uploader is canceled") const ( - blockSize = (1 << 20) - bufferSize = 100 << 20 + blockSize = (1 << 20) + bufferSize = 100 << 20 + bdevSourceSizeTag = "bdev-source-size" ) type sourceInfo struct { @@ -48,18 +54,20 @@ type sourceInfo struct { type destInfo struct { dev *os.File path string + size int64 } type Uploader interface { Backup(sourceInfo, udmrepo.ID, cbt.Iterator, map[string]string) (udmrepo.Snapshot, int64, error) - Restore(udmrepo.Snapshot, destInfo, cbt.Iterator, map[string]string) (int64, error) + Restore(udmrepo.Snapshot, destInfo, cbt.Iterator, map[string]string) (int64, int64, error) } type blockUploader struct { - ctx context.Context - repoWriter udmrepo.BackupRepo - progress uploader.ProgressUpdater - log logrus.FieldLogger + ctx context.Context + repoWriter udmrepo.BackupRepo + progress uploader.ProgressUpdater + log logrus.FieldLogger + lastProgressUpdate time.Time } func NewUploader(ctx context.Context, repoWriter udmrepo.BackupRepo, progress uploader.ProgressUpdater, log logrus.FieldLogger) Uploader { @@ -84,7 +92,7 @@ func (blkup *blockUploader) Backup(source sourceInfo, parentObject udmrepo.ID, b } destObj, err := blkup.repoWriter.NewObjectWriter(blkup.ctx, udmrepo.ObjectWriteOptions{ - Description: "BDEV:" + getObjectName(source.realSource), + Description: fmt.Sprintf("BDEV:%s-%s", getObjectName(source.realSource), snapStart.Format("2006-01-02-15-04-05")), DataType: udmrepo.ObjectDataTypeData, AccessMode: udmrepo.ObjectDataAccessModeBlock, ParentObject: parentObject, @@ -134,12 +142,55 @@ func (blkup *blockUploader) Backup(source sourceInfo, parentObject udmrepo.ID, b Type: udmrepo.ObjectDataTypeMetadata, Permissions: 0o777, }, + Tags: map[string]string{ + bdevSourceSizeTag: strconv.FormatInt(source.size, 10), + }, }, backupSize, nil } -// TODO implement in following PRs -func (blkup *blockUploader) Restore(snapshot udmrepo.Snapshot, dest destInfo, bitmap cbt.Iterator, configs map[string]string) (int64, error) { - return 0, errors.New("not implemented") +func (blkup *blockUploader) Restore(snapshot udmrepo.Snapshot, dest destInfo, bitmap cbt.Iterator, configs map[string]string) (int64, int64, error) { + if bitmap == nil { + return 0, 0, errors.New("bitmap is not available") + } + + meta, err := blkup.repoWriter.ReadMetadata(blkup.ctx, snapshot.RootObject.ID) + if err != nil { + return 0, 0, errors.Wrapf(err, "error reading snapshot metadata for %s", snapshot.Description) + } + + if len(meta.SubObjects) != 1 { + return 0, 0, errors.Errorf("unexpected number of bdev object (%d) for snapshot %s", len(meta.SubObjects), snapshot.Description) + } + + sourceSize, err := getSourceSize(snapshot) + if err != nil { + sourceSize = meta.SubObjects[0].Size + blkup.log.Warnf("Failed to get source size from snapshot %s, use backup size %v", snapshot.Description, sourceSize) + } + + if sourceSize > meta.SubObjects[0].Size { + return 0, 0, errors.Errorf("unexpected size (%v vs. %v) for bdev object %s", meta.SubObjects[0].Size, sourceSize, meta.SubObjects[0].Name) + } + + if sourceSize > dest.size { + return 0, 0, errors.Errorf("dest dev(%s) size is too small (%v vs. %v)", dest.path, dest.size, sourceSize) + } + + reader, err := blkup.repoWriter.OpenObject(blkup.ctx, meta.SubObjects[0].ID, udmrepo.ObjectReadOptions{ + Prefetch: true, + PrefetchBudgetMB: 256, + }) + if err != nil { + return 0, 0, errors.Wrapf(err, "error opening bdev object %v", meta.SubObjects[0].Name) + } + defer reader.Close() + + size, err := blkup.restoreData(reader, dest.dev, bitmap, sourceSize, dest.path) + if err != nil { + return 0, 0, errors.Wrapf(err, "error restoring bdev object %s to volume %s", meta.SubObjects[0].Name, dest.path) + } + + return size, sourceSize, nil } func (blkup *blockUploader) backupObject(dev *os.File, dest udmrepo.ObjectWriter, bitmap cbt.Iterator, totalLength int64) (udmrepo.ID, int64, int64, error) { @@ -152,6 +203,17 @@ func (blkup *blockUploader) backupObject(dev *os.File, dest udmrepo.ObjectWriter return id, backupSize, objectSize, err } +func (blkup *blockUploader) UpdateProgress(p *uploader.Progress) { + if blkup.progress == nil { + return + } + + if time.Since(blkup.lastProgressUpdate) >= 10*time.Second || p.BytesDone == p.TotalBytes { + blkup.progress.UpdateProgress(p) + blkup.lastProgressUpdate = time.Now() + } +} + type readResult struct { buffer []byte offset int64 @@ -167,77 +229,114 @@ func (r *readResult) resetBuffer(list *freelist.FreeList) { func (blkup *blockUploader) backupData(reader io.ReaderAt, writer udmrepo.ObjectWriter, bitmap cbt.Iterator, totalLength int64) (int64, int64, error) { blockSize := bitmap.BlockSize() + totalCount := int64(bitmap.Count()) list := freelist.New(bufferSize, int(blockSize)) resultChan := make(chan readResult, list.Capacity()) - totalCount := bitmap.Count() - aligned := (totalLength + int64(blockSize) - 1) / int64(blockSize) * int64(blockSize) - quit := make(chan struct{}) - defer close(quit) + aligned := (totalLength + int64(blockSize) - 1) / int64(blockSize) * int64(blockSize) + wg := &sync.WaitGroup{} + var writeErr error + var written int64 + var lastPos int64 + + wg.Add(2) go func() { - defer close(resultChan) - - offset, valid := bitmap.Next() - var buffer []byte - for valid { - select { - case <-blkup.ctx.Done(): - return - case <-quit: - return - case buffer = <-list.Chunks(): - } - - length := blockSize - if offset+uint64(length) > uint64(totalLength) { - length = uint(uint64(totalLength) - offset) - clear(buffer) - } - - readBytes, err := reader.ReadAt(buffer[:length], int64(offset)) - if err == nil && readBytes <= 0 { - err = io.ErrUnexpectedEOF - } - - r := readResult{ - buffer: buffer, - offset: int64(offset), - err: err, - } - - if r.err != nil { - r.resetBuffer(list) - } - - resultChan <- r - - if r.err != nil { - return - } - - offset, valid = bitmap.Next() - } + defer wg.Done() + backupReadProc(blkup.ctx, reader, resultChan, quit, bitmap, list, totalLength) }() + go func() { + defer wg.Done() + defer close(quit) + written, lastPos, writeErr = backupWriteProc(blkup.ctx, writer, resultChan, list, aligned, totalCount, int(blockSize), blkup) + }() + + wg.Wait() + + if writeErr != nil { + return written, aligned, errors.Wrap(writeErr, "error writing data") + } + + if lastPos < aligned { + s, err := copyTailData(reader, writer, totalLength, int64(blockSize)) + if err != nil { + return written, aligned, errors.Wrapf(err, "unable to write tail data at %v", lastPos) + } + + written += s + + blkup.UpdateProgress(&uploader.Progress{BytesDone: aligned, TotalBytes: aligned}) + } + + return written, aligned, nil +} + +func backupReadProc(ctx context.Context, reader io.ReaderAt, resultChan chan readResult, quit chan struct{}, bitmap cbt.Iterator, list *freelist.FreeList, totalLength int64) { + defer close(resultChan) + + blockSize := bitmap.BlockSize() + offset, valid := bitmap.Next() + var buffer []byte + for valid { + select { + case <-ctx.Done(): + return + case <-quit: + return + case buffer = <-list.Chunks(): + } + + length := blockSize + if offset+uint64(length) > uint64(totalLength) { + length = uint(uint64(totalLength) - offset) + clear(buffer) + } + + readBytes, err := reader.ReadAt(buffer[:length], int64(offset)) + if err == nil && readBytes <= 0 { + err = io.ErrUnexpectedEOF + } + + r := readResult{ + buffer: buffer, + offset: int64(offset), + err: err, + } + + if r.err != nil { + r.resetBuffer(list) + } + + resultChan <- r + + if r.err != nil { + return + } + + offset, valid = bitmap.Next() + } +} + +func backupWriteProc(ctx context.Context, writer udmrepo.ObjectWriter, resultChan chan readResult, list *freelist.FreeList, totalLength int64, + totalCount int64, blockSize int, progress uploader.ProgressUpdater) (int64, int64, error) { var lastPos int64 var result readResult var written int64 var curCount int64 var writeErr error - var readerRunning bool - for curCount < int64(totalCount) { + for { select { - case <-blkup.ctx.Done(): + case <-ctx.Done(): writeErr = ErrCanceled - case result, readerRunning = <-resultChan: - if !readerRunning { - if blkup.ctx.Err() != nil { + case r, ok := <-resultChan: + if !ok { + if ctx.Err() != nil { writeErr = ErrCanceled - } else { - writeErr = io.ErrUnexpectedEOF } + } else { + result = r } } @@ -250,13 +349,17 @@ func (blkup *blockUploader) backupData(reader io.ReaderAt, writer udmrepo.Object break } + if result.buffer == nil { + break + } + n, err := writer.WriteAt(result.buffer, result.offset) if err != nil { writeErr = err break } - if blockSize != uint(n) { + if blockSize != n { writeErr = io.ErrShortWrite break } @@ -266,27 +369,20 @@ func (blkup *blockUploader) backupData(reader io.ReaderAt, writer udmrepo.Object result.resetBuffer(list) curCount++ - blkup.progress.UpdateProgress(&uploader.Progress{BytesDone: lastPos, TotalBytes: aligned}) + progress.UpdateProgress(&uploader.Progress{BytesDone: lastPos, TotalBytes: totalLength}) } result.resetBuffer(list) if writeErr != nil { - return written, aligned, writeErr + return written, lastPos, writeErr } - if lastPos < aligned { - s, err := copyTailData(reader, writer, totalLength, int64(blockSize)) - if err != nil { - return written, aligned, errors.Wrapf(err, "unable to write tail data at %v", lastPos) - } - - written += s - - blkup.progress.UpdateProgress(&uploader.Progress{BytesDone: aligned, TotalBytes: aligned}) + if curCount < totalCount { + return written, lastPos, io.ErrUnexpectedEOF } - return written, aligned, nil + return written, lastPos, nil } func copyTailData(source io.ReaderAt, writer udmrepo.ObjectWriter, totalLength int64, blockSize int64) (int64, error) { @@ -318,6 +414,240 @@ func getObjectName(source string) string { return strings.Trim(s, "-") } +func (blkup *blockUploader) restoreData(reader io.ReadSeeker, dest *os.File, bitmap cbt.Iterator, totalLength int64, destPath string) (int64, error) { + blockSize := bitmap.BlockSize() + totalCount := int64(bitmap.Count()) + list := freelist.New(bufferSize, int(blockSize)) + resultChan := make(chan readResult, list.Capacity()) + quit := make(chan struct{}) + var writeErr error + var written int64 + + wg := &sync.WaitGroup{} + + wg.Add(2) + + go func() { + defer wg.Done() + restoreReadProc(blkup.ctx, reader, resultChan, quit, bitmap, list) + }() + + go func() { + defer wg.Done() + defer close(quit) + written, writeErr = restoreWriteProc(blkup.ctx, dest, resultChan, list, totalLength, totalCount, int(blockSize), destPath, blkup, blkup.log) + }() + + wg.Wait() + + if writeErr != nil { + return written, errors.Wrap(writeErr, "error writing data") + } + + blkup.progress.UpdateProgress(&uploader.Progress{BytesDone: totalLength, TotalBytes: totalLength}) + + return written, nil +} + +func restoreReadProc(ctx context.Context, reader io.ReadSeeker, resultChan chan readResult, quit chan struct{}, bitmap cbt.Iterator, list *freelist.FreeList) { + defer close(resultChan) + + blockSize := bitmap.BlockSize() + offset, valid := bitmap.Next() + var buffer []byte + var nextPos = uint64(0) + for valid { + select { + case <-ctx.Done(): + return + case <-quit: + return + case buffer = <-list.Chunks(): + } + + var err error + + if nextPos != offset { + _, err = reader.Seek(int64(offset), io.SeekStart) + } + + if err == nil { + var length int + length, err = io.ReadFull(reader, buffer) + if err == nil && length <= 0 { + err = io.ErrUnexpectedEOF + } + } + + r := readResult{ + buffer: buffer, + offset: int64(offset), + err: err, + } + + if r.err != nil { + r.resetBuffer(list) + } + + resultChan <- r + + if r.err != nil { + return + } + + nextPos = offset + uint64(blockSize) + offset, valid = bitmap.Next() + } +} + +func restoreWriteProc(ctx context.Context, dest *os.File, resultChan chan readResult, list *freelist.FreeList, totalLength int64, totalCount int64, + blockSize int, destPath string, progress uploader.ProgressUpdater, log logrus.FieldLogger) (int64, error) { + zeroBlock := make([]byte, blockSize) + + var written int64 + var result readResult + var writeErr error + var zeroStart int64 = -1 + var zeroLength int64 + var curCount int64 + + for { + select { + case <-ctx.Done(): + writeErr = ErrCanceled + case r, ok := <-resultChan: + if !ok { + if ctx.Err() != nil { + writeErr = ErrCanceled + } + } else { + result = r + } + } + + if writeErr != nil { + break + } + + if result.err != nil { + writeErr = result.err + break + } + + if result.buffer == nil { + break + } + + length := min(int64(blockSize), totalLength-result.offset) + if bytes.Equal(result.buffer, zeroBlock) { + if zeroStart == -1 { + zeroStart = result.offset + zeroLength = length + } else if result.offset == zeroStart+zeroLength { + zeroLength += length + } else { + if err := flushZeroBlocks(dest, zeroStart, zeroLength, zeroBlock, destPath, log); err != nil { + writeErr = errors.Wrapf(err, "error flushing zero blocks from %v, length %v", zeroStart, zeroLength) + break + } + zeroStart = result.offset + zeroLength = length + } + } else { + if zeroStart != -1 { + if err := flushZeroBlocks(dest, zeroStart, zeroLength, zeroBlock, destPath, log); err != nil { + writeErr = errors.Wrapf(err, "error flushing zero blocks from %v, length %v", zeroStart, zeroLength) + break + } + + zeroStart = -1 + zeroLength = 0 + } + + n, err := dest.WriteAt(result.buffer[:length], result.offset) + if err != nil { + writeErr = err + break + } + + if length != int64(n) { + writeErr = io.ErrShortWrite + break + } + } + + written += length + curCount++ + + result.resetBuffer(list) + + progress.UpdateProgress(&uploader.Progress{BytesDone: result.offset + length, TotalBytes: totalLength}) + } + + result.resetBuffer(list) + + if writeErr != nil { + return written, writeErr + } + + if curCount < totalCount { + return written, io.ErrUnexpectedEOF + } + + if zeroStart != -1 { + if err := flushZeroBlocks(dest, zeroStart, zeroLength, zeroBlock, destPath, log); err != nil { + return written, errors.Wrapf(err, "error flushing zero blocks from %v, length %v", zeroStart, zeroLength) + } + } + + return written, nil +} + +func flushZeroBlocks(dest *os.File, start int64, length int64, zeroBlock []byte, destPath string, log logrus.FieldLogger) error { + err := blkZeroOut(dest, start, length) + if err == nil { + return nil + } + + log.WithError(err).Warnf("Failed to call zero out from dev %s, start %v, length %v. Fallback to conservative way", destPath, start, length) + + var written int64 + for written < length { + writeSize := min(len(zeroBlock), int(length-written)) + + n, err := dest.WriteAt(zeroBlock[:writeSize], start+written) + if err != nil { + return errors.Wrapf(err, "error writing zero buffer at %v, length %v", start+written, writeSize) + } + + if writeSize != n { + return errors.Errorf("short write zero buffer at %v, length %v", start+written, writeSize) + } + + written += int64(writeSize) + } + + return nil +} + +func getSourceSize(snapshot udmrepo.Snapshot) (int64, error) { + if snapshot.Tags == nil { + return 0, errors.New("source size tag is empty") + } + + s, found := snapshot.Tags[bdevSourceSizeTag] + if !found { + return 0, errors.New("source size tag is missing") + } + + size, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return 0, errors.Wrapf(err, "error parsing size from %s", s) + } + + return size, nil +} + func loadObjectFromSnapshot(ctx context.Context, rep udmrepo.BackupRepo, snapshot *udmrepo.Snapshot) (udmrepo.ID, error) { if snapshot == nil { return "", errors.New("snapshot is empty") diff --git a/pkg/uploader/block/uploader_test.go b/pkg/uploader/block/uploader_test.go index 88fd4771e..bb94eb3de 100644 --- a/pkg/uploader/block/uploader_test.go +++ b/pkg/uploader/block/uploader_test.go @@ -21,6 +21,7 @@ import ( "context" "io" "os" + "strings" "testing" "time" @@ -196,7 +197,7 @@ func TestBlockUploaderBackup(t *testing.T) { name: "canceled in progress", cancelInProgress: true, expectErr: true, - expectErrStr: "error backing up bdev /data/volume1: uploader is canceled", + expectErrStr: "error backing up bdev /data/volume1: error writing data: uploader is canceled", }, { name: "create object writer err", @@ -319,6 +320,7 @@ func TestBlockUploaderBackup(t *testing.T) { iterMock.On("Count").Return(uint64(1)) iterMock.On("Next").Return(uint64(0), true).Maybe() + objWriter.On("WriteAt", mock.Anything, mock.Anything).Return(0, context.Canceled).Maybe() objWriter.On("Result").Return(udmrepo.ID(""), errors.New("write failed")).Maybe() } else if tc.shortWrite { iterMock.On("BlockSize").Return(uint(1048576)) @@ -357,7 +359,7 @@ func TestBlockUploaderBackup(t *testing.T) { } repoWriter.On("NewObjectWriter", mock.Anything, mock.MatchedBy(func(opt udmrepo.ObjectWriteOptions) bool { - return opt.Description == "BDEV:data-volume1" && opt.BackupMode == backupMode + return strings.HasPrefix(opt.Description, "BDEV:data-volume1-") && opt.BackupMode == backupMode })).Return(objWriter, tc.createObjErr) } @@ -459,3 +461,280 @@ func TestLoadObjectFromSnapshot(t *testing.T) { }) } } + +func TestGetSourceSize(t *testing.T) { + testCases := []struct { + name string + snapshot udmrepo.Snapshot + expectErr bool + expected int64 + }{ + { + name: "nil tags", + snapshot: udmrepo.Snapshot{}, + expectErr: true, + }, + { + name: "missing tag", + snapshot: udmrepo.Snapshot{ + Tags: map[string]string{}, + }, + expectErr: true, + }, + { + name: "invalid tag value", + snapshot: udmrepo.Snapshot{ + Tags: map[string]string{ + bdevSourceSizeTag: "abc", + }, + }, + expectErr: true, + }, + { + name: "valid tag value", + snapshot: udmrepo.Snapshot{ + Tags: map[string]string{ + bdevSourceSizeTag: "1048576", + }, + }, + expectErr: false, + expected: 1048576, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + size, err := getSourceSize(tc.snapshot) + if tc.expectErr { + assert.Error(t, err) + } else { + require.NoError(t, err) + assert.Equal(t, tc.expected, size) + } + }) + } +} + +func TestFlushZeroBlocks(t *testing.T) { + t.Run("success via write fallback", func(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "zerotest-*") + require.NoError(t, err) + defer os.Remove(f.Name()) + defer f.Close() + + require.NoError(t, f.Truncate(2048)) + + log := logrus.New() + log.Out = io.Discard + + zeroBlock := make([]byte, 1024) + err = flushZeroBlocks(f, 0, 2048, zeroBlock, f.Name(), log) + + require.NoError(t, err) + + data, err := os.ReadFile(f.Name()) + require.NoError(t, err) + assert.Equal(t, make([]byte, 2048), data) + }) +} + +type errReader struct { + err error +} + +func (r *errReader) Read(p []byte) (n int, err error) { + return 0, r.err +} + +func (r *errReader) Seek(offset int64, whence int) (int64, error) { + return 0, nil +} + +func TestRestoreData(t *testing.T) { + t.Run("success", func(t *testing.T) { + ctx := context.Background() + progress := &mockProgressUpdater{} + progress.On("UpdateProgress", mock.Anything).Return() + blkup := &blockUploader{ + ctx: ctx, + progress: progress, + log: logrus.New(), + } + + f, err := os.CreateTemp(t.TempDir(), "restoretest-*") + require.NoError(t, err) + defer os.Remove(f.Name()) + defer f.Close() + + data := make([]byte, 1048576) + for i := range data { + data[i] = 1 + } + reader := bytes.NewReader(data) + + iterMock := cbtmocks.NewIterator(t) + iterMock.On("Count").Return(uint64(1)) + iterMock.On("Next").Return(uint64(0), true).Once() + iterMock.On("Next").Return(uint64(0), false) + iterMock.On("BlockSize").Return(uint(1048576)) + + written, err := blkup.restoreData(reader, f, iterMock, 1048576, f.Name()) + require.NoError(t, err) + assert.Equal(t, int64(1048576), written) + + f.Seek(0, 0) + writtenData, err := io.ReadAll(f) + require.NoError(t, err) + assert.Equal(t, data, writtenData) + }) + + t.Run("read err", func(t *testing.T) { + ctx := context.Background() + blkup := &blockUploader{ + ctx: ctx, + log: logrus.New(), + } + + f, err := os.CreateTemp(t.TempDir(), "restoretest-*") + require.NoError(t, err) + defer os.Remove(f.Name()) + defer f.Close() + + reader := &errReader{err: errors.New("read error")} + + iterMock := cbtmocks.NewIterator(t) + iterMock.On("Count").Return(uint64(1)) + iterMock.On("Next").Return(uint64(0), true).Once() + iterMock.On("Next").Return(uint64(0), false) + iterMock.On("BlockSize").Return(uint(1048576)) + + _, err = blkup.restoreData(reader, f, iterMock, 1048576, f.Name()) + require.Error(t, err) + assert.Contains(t, err.Error(), "read error") + }) +} + +func TestBlockUploaderRestore(t *testing.T) { + t.Run("missing metadata", func(t *testing.T) { + ctx := context.Background() + repoWriter := udmrepomocks.NewBackupRepo(t) + blkup := NewUploader(ctx, repoWriter, nil, logrus.New()) + + repoWriter.On("ReadMetadata", mock.Anything, udmrepo.ID("root-id")).Return(nil, errors.New("meta not found")) + + iterMock := cbtmocks.NewIterator(t) + _, _, err := blkup.Restore(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root-id"}}, destInfo{}, iterMock, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "meta not found") + }) + + t.Run("success", func(t *testing.T) { + ctx := context.Background() + repoWriter := udmrepomocks.NewBackupRepo(t) + progress := &mockProgressUpdater{} + progress.On("UpdateProgress", mock.Anything).Return() + + blkup := NewUploader(ctx, repoWriter, progress, logrus.New()) + + f, err := os.CreateTemp(t.TempDir(), "restoretest-*") + require.NoError(t, err) + defer os.Remove(f.Name()) + defer f.Close() + + meta := &udmrepo.Metadata{ + SubObjects: []udmrepo.ObjectMetadata{ + { + ID: "data-id", + Name: "bdev", + Size: 1048576, + }, + }, + } + + repoWriter.On("ReadMetadata", mock.Anything, udmrepo.ID("root-id")).Return(meta, nil) + + objReader := udmrepomocks.NewObjectReader(t) + objReader.On("Read", mock.Anything).Run(func(args mock.Arguments) { + p := args.Get(0).([]byte) + for i := range p { + p[i] = 1 + } + }).Return(1048576, io.EOF).Once() + objReader.On("Read", mock.Anything).Return(0, io.EOF) + objReader.On("Close").Return(nil) + + repoWriter.On("OpenObject", mock.Anything, udmrepo.ID("data-id"), mock.Anything).Return(objReader, nil) + + snap := udmrepo.Snapshot{ + Description: "test snapshot", + RootObject: udmrepo.ObjectMetadata{ID: "root-id"}, + Tags: map[string]string{ + bdevSourceSizeTag: "1048576", + }, + } + + dest := destInfo{ + dev: f, + size: 2048576, + path: f.Name(), + } + + iterMock := cbtmocks.NewIterator(t) + iterMock.On("Count").Return(uint64(1)) + iterMock.On("Next").Return(uint64(0), true).Once() + iterMock.On("Next").Return(uint64(0), false) + iterMock.On("BlockSize").Return(uint(1048576)) + + written, _, err := blkup.Restore(snap, dest, iterMock, nil) + require.NoError(t, err) + assert.Equal(t, int64(1048576), written) + }) + + t.Run("source size tag larger than object size", func(t *testing.T) { + ctx := context.Background() + repoWriter := udmrepomocks.NewBackupRepo(t) + blkup := NewUploader(ctx, repoWriter, nil, logrus.New()) + + meta := &udmrepo.Metadata{ + SubObjects: []udmrepo.ObjectMetadata{ + {ID: "data-id", Name: "bdev", Size: 1048576}, + }, + } + repoWriter.On("ReadMetadata", mock.Anything, udmrepo.ID("root-id")).Return(meta, nil) + + snap := udmrepo.Snapshot{ + RootObject: udmrepo.ObjectMetadata{ID: "root-id"}, + Tags: map[string]string{bdevSourceSizeTag: "2097152"}, + } + dest := destInfo{size: 4194304, path: "/dev/target"} + iterMock := cbtmocks.NewIterator(t) + + _, _, err := blkup.Restore(snap, dest, iterMock, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "unexpected size (1048576 vs. 2097152) for bdev object bdev") + }) + + t.Run("destination smaller than source size", func(t *testing.T) { + ctx := context.Background() + repoWriter := udmrepomocks.NewBackupRepo(t) + blkup := NewUploader(ctx, repoWriter, nil, logrus.New()) + + meta := &udmrepo.Metadata{ + SubObjects: []udmrepo.ObjectMetadata{ + {ID: "data-id", Name: "bdev", Size: 1048576}, + }, + } + repoWriter.On("ReadMetadata", mock.Anything, udmrepo.ID("root-id")).Return(meta, nil) + + snap := udmrepo.Snapshot{ + RootObject: udmrepo.ObjectMetadata{ID: "root-id"}, + Tags: map[string]string{bdevSourceSizeTag: "1048576"}, + } + dest := destInfo{size: 512, path: "/dev/small"} + iterMock := cbtmocks.NewIterator(t) + + _, _, err := blkup.Restore(snap, dest, iterMock, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "dest dev(/dev/small) size is too small") + }) +} diff --git a/pkg/uploader/kopia/shim.go b/pkg/uploader/kopia/shim.go index 4a3908185..465b76c04 100644 --- a/pkg/uploader/kopia/shim.go +++ b/pkg/uploader/kopia/shim.go @@ -56,7 +56,7 @@ func NewShimRepo(repo udmrepo.BackupRepo) repo.RepositoryWriter { // OpenObject open specific object func (sr *shimRepository) OpenObject(ctx context.Context, id object.ID) (object.Reader, error) { - reader, err := sr.udmRepo.OpenObject(ctx, udmrepo.ID(id.String())) + reader, err := sr.udmRepo.OpenObject(ctx, udmrepo.ID(id.String()), udmrepo.ObjectReadOptions{}) if err != nil { return nil, errors.Wrapf(err, "failed to open object with id %v", id) } diff --git a/pkg/uploader/kopia/shim_test.go b/pkg/uploader/kopia/shim_test.go index 7933ec6b8..3c7941405 100644 --- a/pkg/uploader/kopia/shim_test.go +++ b/pkg/uploader/kopia/shim_test.go @@ -81,7 +81,7 @@ func TestOpenObject(t *testing.T) { name: "Success", backupRepo: func() *mocks.BackupRepo { backupRepo := &mocks.BackupRepo{} - backupRepo.On("OpenObject", mock.Anything, mock.Anything).Return(&shimObjectReader{}, nil) + backupRepo.On("OpenObject", mock.Anything, mock.Anything, mock.Anything).Return(&shimObjectReader{}, nil) return backupRepo }(), }, @@ -89,7 +89,7 @@ func TestOpenObject(t *testing.T) { name: "Open object error", backupRepo: func() *mocks.BackupRepo { backupRepo := &mocks.BackupRepo{} - backupRepo.On("OpenObject", mock.Anything, mock.Anything).Return(&shimObjectReader{}, errors.New("Error open object")) + backupRepo.On("OpenObject", mock.Anything, mock.Anything, mock.Anything).Return(&shimObjectReader{}, errors.New("Error open object")) return backupRepo }(), isOpenObjectError: true, @@ -98,7 +98,7 @@ func TestOpenObject(t *testing.T) { name: "Get nil reader", backupRepo: func() *mocks.BackupRepo { backupRepo := &mocks.BackupRepo{} - backupRepo.On("OpenObject", mock.Anything, mock.Anything).Return(nil, nil) + backupRepo.On("OpenObject", mock.Anything, mock.Anything, mock.Anything).Return(nil, nil) return backupRepo }(), isReaderNil: true, diff --git a/pkg/uploader/kopia/snapshot.go b/pkg/uploader/kopia/snapshot.go index 217ff531f..fae7a517c 100644 --- a/pkg/uploader/kopia/snapshot.go +++ b/pkg/uploader/kopia/snapshot.go @@ -389,7 +389,7 @@ func (o *fileSystemRestoreOutput) Terminate() error { } // Restore restore specific sourcePath with given snapshotID and update progress -func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, +func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, snapshotID, dest string, incremental bool, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { log.Info("Start to restore...") @@ -421,7 +421,7 @@ func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, } restoreConcurrency := runtime.NumCPU() - + deleteExtra := false if len(uploaderCfg) > 0 { writeSparseFiles, err := uploaderutil.GetWriteSparseFiles(uploaderCfg) if err != nil { @@ -438,9 +438,14 @@ func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, if concurrency > 0 { restoreConcurrency = concurrency } + + deleteExtra, err = uploaderutil.GetDeleteExtraFiles(uploaderCfg) + if err != nil { + return 0, 0, errors.Wrap(err, "failed to get delete extra files config") + } } - log.Debugf("Restore filesystem output %v, concurrency %d", fsOutput, restoreConcurrency) + log.Debugf("Restore filesystem output %v, concurrency %d, incremental %v, delete extra %v", fsOutput, restoreConcurrency, incremental, deleteExtra) err = fsOutput.Init(ctx) if err != nil { @@ -448,14 +453,22 @@ func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, } var output RestoreOutput + // kopiaOutput is the output passed to Kopia's restore.Entry function. + // We must pass the unwrapped fsOutput (*restore.FilesystemOutput) directly for file system restores. + // This is because Kopia internally uses a strict type assertion (c.output.(*FilesystemOutput)) + // to determine if it should execute the deleteExtra logic. If we pass the wrapped + // fileSystemRestoreOutput, the type assertion fails and extra files are not deleted. + var kopiaOutput restore.Output if volMode == uploader.PersistentVolumeBlock { output = &BlockOutput{ FilesystemOutput: fsOutput, } + kopiaOutput = output } else { output = &fileSystemRestoreOutput{ FilesystemOutput: fsOutput, } + kopiaOutput = fsOutput } defer func() { @@ -464,8 +477,10 @@ func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, } }() - stat, err := restoreEntryFunc(kopiaCtx, rep, output, rootEntry, restore.Options{ + stat, err := restoreEntryFunc(kopiaCtx, rep, kopiaOutput, rootEntry, restore.Options{ Parallel: restoreConcurrency, + Incremental: incremental, + DeleteExtra: deleteExtra, RestoreDirEntryAtDepth: math.MaxInt32, Cancel: cancleCh, ProgressCallback: func(ctx context.Context, stats restore.Stats) { diff --git a/pkg/uploader/kopia/snapshot_test.go b/pkg/uploader/kopia/snapshot_test.go index 36f30d82c..e58c2bb88 100644 --- a/pkg/uploader/kopia/snapshot_test.go +++ b/pkg/uploader/kopia/snapshot_test.go @@ -681,6 +681,7 @@ func TestRestore(t *testing.T) { expectedCount int32 expectedError error volMode uploader.PersistentVolumeMode + incremental bool } // Define test cases @@ -818,7 +819,7 @@ func TestRestore(t *testing.T) { repoWriterMock.On("OpenObject", mock.Anything, mock.Anything).Return(em, nil) progress := new(Progress) - bytesRestored, fileCount, err := Restore(t.Context(), repoWriterMock, progress, tc.snapshotID, tc.dest, tc.volMode, map[string]string{}, logrus.New(), nil) + bytesRestored, fileCount, err := Restore(t.Context(), repoWriterMock, progress, tc.snapshotID, tc.dest, tc.incremental, tc.volMode, map[string]string{}, logrus.New(), nil) // Check if the returned error matches the expected error if tc.expectedError != nil { diff --git a/pkg/uploader/provider/block.go b/pkg/uploader/provider/block.go index 9135bb67b..2b5ad275f 100644 --- a/pkg/uploader/provider/block.go +++ b/pkg/uploader/provider/block.go @@ -134,7 +134,11 @@ func (bp *blockProvider) RunBackup( snapshotInfo, _, err := blockBackupFunc(ctx, blkUploader, bp.bkRepo, path, realSource, cbtParam.Source, forceFull, parentSnapshot, cbtParam.Service, uploaderCfg, tags, log) - if err == block.ErrCanceled { + // errors.Is, not ==: the sentinel is wrapped twice on its way here, by + // block/uploader.go ("error backing up bdev %s") and again by + // block/snapshot.go ("Failed to run uploader backup for si %v"), so an + // equality check never matches and cancellation gets reported as a failure. + if errors.Is(err, block.ErrCanceled) { log.Warn("Block backup is canceled") return snapshotInfo.ID, false, snapshotInfo.Size, snapshotInfo.IncrementalSize, ErrorCanceled } @@ -150,7 +154,7 @@ func (bp *blockProvider) RunBackup( }, ) - log.Infof("Block backup finished, snapshot ID %s, backup size %d", snapshotInfo.ID, snapshotInfo.Size) + log.Infof("Block backup finished, snapshot ID %s, backup size %v, incremental size %v", snapshotInfo.ID, snapshotInfo.Size, snapshotInfo.IncrementalSize) return snapshotInfo.ID, false, snapshotInfo.Size, snapshotInfo.IncrementalSize, nil } @@ -159,6 +163,8 @@ func (bp *blockProvider) RunRestore( ctx context.Context, snapshotID string, volumePath string, + incremental bool, + cbtParam CBTParam, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (int64, error) { @@ -174,9 +180,10 @@ func (bp *blockProvider) RunRestore( blkUploader := block.NewUploader(ctx, bp.bkRepo, updater, log) - size, err := blockRestoreFunc(ctx, blkUploader, bp.bkRepo, snapshotID, volumePath, uploaderCfg, log) + size, err := blockRestoreFunc(ctx, blkUploader, bp.bkRepo, snapshotID, volumePath, incremental, cbtParam.Source, cbtParam.Service, uploaderCfg, log) - if err == block.ErrCanceled { + // errors.Is, not ==: see the equivalent comment on the backup path above. + if errors.Is(err, block.ErrCanceled) { log.Warn("Block restore is canceled") return 0, ErrorCanceled } diff --git a/pkg/uploader/provider/block_test.go b/pkg/uploader/provider/block_test.go index ad8f68b52..970fc7cf6 100644 --- a/pkg/uploader/provider/block_test.go +++ b/pkg/uploader/provider/block_test.go @@ -372,6 +372,63 @@ func TestBlockProviderRunBackup(t *testing.T) { } } +// TestBlockProviderCancelThroughWrappedError pins that cancellation is recognized +// after the sentinel has been wrapped, which is the only way it ever arrives in +// production: block/uploader.go wraps it with "error backing up bdev %s" and +// block/snapshot.go wraps that with "Failed to run uploader backup for si %v". +// +// Asserting on the message is useless here — provider.ErrorCanceled and +// block.ErrCanceled carry the *same* text ("uploader is canceled"), so a substring +// check passes whether or not the sentinel was actually recognized. The assertion +// has to be on identity. +func TestBlockProviderCancelThroughWrappedError(t *testing.T) { + t.Run("backup", func(t *testing.T) { + orig := blockBackupFunc + defer func() { blockBackupFunc = orig }() + blockBackupFunc = func(_ context.Context, _ block.Uploader, _ udmrepo.BackupRepo, _ string, _ string, _ cbtservice.SourceInfo, _ bool, _ string, _ cbtservice.Service, _ map[string]string, _ map[string]string, _ logrus.FieldLogger) (uploader.SnapshotInfo, bool, error) { + return uploader.SnapshotInfo{ID: "snap-cancel", Size: 2048, IncrementalSize: 1024}, false, + errors.Wrapf( + errors.Wrapf(block.ErrCanceled, "error backing up bdev %s", "ns/pvc"), + "Failed to run uploader backup for si %v", "si") + } + + bp := &blockProvider{ + requestorType: "test", + bkRepo: udmrepomocks.NewBackupRepo(t), + log: logrus.New(), + } + + _, _, _, _, err := bp.RunBackup( + t.Context(), "/dev/sda", "ns/pvc", map[string]string{}, false, "", + CBTParam{}, uploader.PersistentVolumeBlock, map[string]string{}, + &FakeBackupProgressUpdater{}, + ) + + require.ErrorIs(t, err, ErrorCanceled, + "a wrapped block.ErrCanceled must surface as provider.ErrorCanceled; otherwise the "+ + "DataUpload is marked Failed and the Backup PartiallyFailed for a user-requested cancel") + }) + + t.Run("restore", func(t *testing.T) { + orig := blockRestoreFunc + defer func() { blockRestoreFunc = orig }() + blockRestoreFunc = func(_ context.Context, _ block.Uploader, _ udmrepo.BackupRepo, _ string, _ string, _ bool, _ cbtservice.SourceInfo, _ cbtservice.Service, _ map[string]string, _ logrus.FieldLogger) (int64, error) { + return 0, errors.Wrap(block.ErrCanceled, "error restoring bdev") + } + + bp := &blockProvider{ + requestorType: "test", + bkRepo: udmrepomocks.NewBackupRepo(t), + log: logrus.New(), + } + + _, err := bp.RunRestore(t.Context(), "snap-1", "/dev/sda", false, CBTParam{}, + uploader.PersistentVolumeBlock, map[string]string{}, &blockMockProgressUpdater{}) + + require.ErrorIs(t, err, ErrorCanceled) + }) +} + func TestBlockProviderRunRestore(t *testing.T) { testCases := []struct { name string @@ -439,9 +496,9 @@ func TestBlockProviderRunRestore(t *testing.T) { var capturedSnapshotID string var capturedVolumePath string - blockRestoreFunc = func(_ context.Context, _ block.Uploader, _ udmrepo.BackupRepo, snapshotID string, volumePath string, _ map[string]string, _ logrus.FieldLogger) (int64, error) { + blockRestoreFunc = func(ctx context.Context, blkUp block.Uploader, rep udmrepo.BackupRepo, snapshotID string, dest string, incremental bool, cbtSource cbtservice.SourceInfo, cbtService cbtservice.Service, uploaderCfg map[string]string, log logrus.FieldLogger) (int64, error) { capturedSnapshotID = snapshotID - capturedVolumePath = volumePath + capturedVolumePath = dest return tc.mockRestoreSize, tc.mockRestoreErr } @@ -454,6 +511,8 @@ func TestBlockProviderRunRestore(t *testing.T) { t.Context(), tc.snapshotID, tc.volumePath, + false, + CBTParam{}, uploader.PersistentVolumeBlock, map[string]string{}, tc.updater, diff --git a/pkg/uploader/provider/kopia.go b/pkg/uploader/provider/kopia.go index 682b2053e..c9d9948bf 100644 --- a/pkg/uploader/provider/kopia.go +++ b/pkg/uploader/provider/kopia.go @@ -211,6 +211,8 @@ func (kp *kopiaProvider) RunRestore( ctx context.Context, snapshotID string, volumePath string, + incremental bool, + _ CBTParam, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (int64, error) { @@ -234,7 +236,7 @@ func (kp *kopiaProvider) RunRestore( // We use the cancel channel to control the restore cancel, so don't pass a context with cancel to Kopia restore. // Otherwise, Kopia restore will not response to the cancel control but return an arbitrary error. // Kopia restore cancel is not designed as well as Kopia backup which uses the context to control backup cancel all the way. - size, fileCount, err := kopiaRestoreFunc(context.Background(), repoWriter, progress, snapshotID, volumePath, volMode, uploaderCfg, log, restoreCancel) + size, fileCount, err := kopiaRestoreFunc(context.Background(), repoWriter, progress, snapshotID, volumePath, incremental, volMode, uploaderCfg, log, restoreCancel) if err != nil { return 0, errors.Wrapf(err, "Failed to run kopia restore") diff --git a/pkg/uploader/provider/kopia_test.go b/pkg/uploader/provider/kopia_test.go index bfb544c26..a29a3c424 100644 --- a/pkg/uploader/provider/kopia_test.go +++ b/pkg/uploader/provider/kopia_test.go @@ -119,20 +119,21 @@ func TestRunBackup(t *testing.T) { func TestRunRestore(t *testing.T) { testCases := []struct { name string - hookRestoreFunc func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) + hookRestoreFunc func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, incremental bool, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) notError bool volMode uploader.PersistentVolumeMode + incremental bool }{ { name: "normal restore", - hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { + hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, incremental bool, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { return 0, 0, nil }, notError: true, }, { name: "normal block mode restore", - hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { + hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, incremental bool, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { return 0, 0, nil }, volMode: uploader.PersistentVolumeBlock, @@ -140,7 +141,7 @@ func TestRunRestore(t *testing.T) { }, { name: "failed to restore", - hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { + hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, incremental bool, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { return 0, 0, errors.New("failed to restore") }, notError: false, @@ -157,7 +158,7 @@ func TestRunRestore(t *testing.T) { tc.volMode = uploader.PersistentVolumeFilesystem } kopiaRestoreFunc = tc.hookRestoreFunc - _, err := kp.RunRestore(t.Context(), "", "/var", tc.volMode, map[string]string{}, &updater) + _, err := kp.RunRestore(t.Context(), "", "/var", tc.incremental, CBTParam{}, tc.volMode, map[string]string{}, &updater) if tc.notError { assert.NoError(t, err) } else { diff --git a/pkg/uploader/provider/mocks/Provider.go b/pkg/uploader/provider/mocks/Provider.go index 71e60b84e..5bd3dda54 100644 --- a/pkg/uploader/provider/mocks/Provider.go +++ b/pkg/uploader/provider/mocks/Provider.go @@ -223,8 +223,8 @@ func (_c *Provider_RunBackup_Call) RunAndReturn(run func(ctx context.Context, pa } // RunRestore provides a mock function for the type Provider -func (_mock *Provider) RunRestore(ctx context.Context, snapshotID string, volumePath string, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater) (int64, error) { - ret := _mock.Called(ctx, snapshotID, volumePath, volMode, uploaderConfig, updater) +func (_mock *Provider) RunRestore(ctx context.Context, snapshotID string, volumePath string, incremental bool, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater) (int64, error) { + ret := _mock.Called(ctx, snapshotID, volumePath, incremental, cbtParam, volMode, uploaderConfig, updater) if len(ret) == 0 { panic("no return value specified for RunRestore") @@ -232,16 +232,16 @@ func (_mock *Provider) RunRestore(ctx context.Context, snapshotID string, volume var r0 int64 var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) (int64, error)); ok { - return returnFunc(ctx, snapshotID, volumePath, volMode, uploaderConfig, updater) + if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, bool, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) (int64, error)); ok { + return returnFunc(ctx, snapshotID, volumePath, incremental, cbtParam, volMode, uploaderConfig, updater) } - if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) int64); ok { - r0 = returnFunc(ctx, snapshotID, volumePath, volMode, uploaderConfig, updater) + if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, bool, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) int64); ok { + r0 = returnFunc(ctx, snapshotID, volumePath, incremental, cbtParam, volMode, uploaderConfig, updater) } else { r0 = ret.Get(0).(int64) } - if returnFunc, ok := ret.Get(1).(func(context.Context, string, string, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) error); ok { - r1 = returnFunc(ctx, snapshotID, volumePath, volMode, uploaderConfig, updater) + if returnFunc, ok := ret.Get(1).(func(context.Context, string, string, bool, provider.CBTParam, uploader.PersistentVolumeMode, map[string]string, uploader.ProgressUpdater) error); ok { + r1 = returnFunc(ctx, snapshotID, volumePath, incremental, cbtParam, volMode, uploaderConfig, updater) } else { r1 = ret.Error(1) } @@ -257,14 +257,16 @@ type Provider_RunRestore_Call struct { // - ctx context.Context // - snapshotID string // - volumePath string +// - incremental bool +// - cbtParam provider.CBTParam // - volMode uploader.PersistentVolumeMode // - uploaderConfig map[string]string // - updater uploader.ProgressUpdater -func (_e *Provider_Expecter) RunRestore(ctx interface{}, snapshotID interface{}, volumePath interface{}, volMode interface{}, uploaderConfig interface{}, updater interface{}) *Provider_RunRestore_Call { - return &Provider_RunRestore_Call{Call: _e.mock.On("RunRestore", ctx, snapshotID, volumePath, volMode, uploaderConfig, updater)} +func (_e *Provider_Expecter) RunRestore(ctx interface{}, snapshotID interface{}, volumePath interface{}, incremental interface{}, cbtParam interface{}, volMode interface{}, uploaderConfig interface{}, updater interface{}) *Provider_RunRestore_Call { + return &Provider_RunRestore_Call{Call: _e.mock.On("RunRestore", ctx, snapshotID, volumePath, incremental, cbtParam, volMode, uploaderConfig, updater)} } -func (_c *Provider_RunRestore_Call) Run(run func(ctx context.Context, snapshotID string, volumePath string, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater)) *Provider_RunRestore_Call { +func (_c *Provider_RunRestore_Call) Run(run func(ctx context.Context, snapshotID string, volumePath string, incremental bool, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater)) *Provider_RunRestore_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -278,17 +280,25 @@ func (_c *Provider_RunRestore_Call) Run(run func(ctx context.Context, snapshotID if args[2] != nil { arg2 = args[2].(string) } - var arg3 uploader.PersistentVolumeMode + var arg3 bool if args[3] != nil { - arg3 = args[3].(uploader.PersistentVolumeMode) + arg3 = args[3].(bool) } - var arg4 map[string]string + var arg4 provider.CBTParam if args[4] != nil { - arg4 = args[4].(map[string]string) + arg4 = args[4].(provider.CBTParam) } - var arg5 uploader.ProgressUpdater + var arg5 uploader.PersistentVolumeMode if args[5] != nil { - arg5 = args[5].(uploader.ProgressUpdater) + arg5 = args[5].(uploader.PersistentVolumeMode) + } + var arg6 map[string]string + if args[6] != nil { + arg6 = args[6].(map[string]string) + } + var arg7 uploader.ProgressUpdater + if args[7] != nil { + arg7 = args[7].(uploader.ProgressUpdater) } run( arg0, @@ -297,6 +307,8 @@ func (_c *Provider_RunRestore_Call) Run(run func(ctx context.Context, snapshotID arg3, arg4, arg5, + arg6, + arg7, ) }) return _c @@ -307,7 +319,7 @@ func (_c *Provider_RunRestore_Call) Return(n int64, err error) *Provider_RunRest return _c } -func (_c *Provider_RunRestore_Call) RunAndReturn(run func(ctx context.Context, snapshotID string, volumePath string, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater) (int64, error)) *Provider_RunRestore_Call { +func (_c *Provider_RunRestore_Call) RunAndReturn(run func(ctx context.Context, snapshotID string, volumePath string, incremental bool, cbtParam provider.CBTParam, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater) (int64, error)) *Provider_RunRestore_Call { _c.Call.Return(run) return _c } diff --git a/pkg/uploader/provider/provider.go b/pkg/uploader/provider/provider.go index 26b7b84f2..9d06578d8 100644 --- a/pkg/uploader/provider/provider.go +++ b/pkg/uploader/provider/provider.go @@ -64,6 +64,8 @@ type Provider interface { ctx context.Context, snapshotID string, volumePath string, + incremental bool, + cbtParam CBTParam, volMode uploader.PersistentVolumeMode, uploaderConfig map[string]string, updater uploader.ProgressUpdater) (int64, error) diff --git a/pkg/uploader/util/uploader_config.go b/pkg/uploader/util/uploader_config.go index c221741bf..3736bcfca 100644 --- a/pkg/uploader/util/uploader_config.go +++ b/pkg/uploader/util/uploader_config.go @@ -28,6 +28,7 @@ const ( ParallelFilesUpload = "ParallelFilesUpload" WriteSparseFiles = "WriteSparseFiles" RestoreConcurrency = "ParallelFilesDownload" + DeleteExtraFiles = "DeleteExtraFiles" ) func StoreBackupConfig(config *velerov1api.UploaderConfigForBackup) map[string]string { @@ -47,6 +48,13 @@ func StoreRestoreConfig(config *velerov1api.UploaderConfigForRestore) map[string if config.ParallelFilesDownload > 0 { data[RestoreConcurrency] = strconv.Itoa(config.ParallelFilesDownload) } + + if config.DeleteExtraFiles != nil { + data[DeleteExtraFiles] = strconv.FormatBool(*config.DeleteExtraFiles) + } else { + data[DeleteExtraFiles] = strconv.FormatBool(false) + } + return data } @@ -85,3 +93,15 @@ func GetRestoreConcurrency(uploaderCfg map[string]string) (int, error) { } return 0, nil } + +func GetDeleteExtraFiles(uploaderCfg map[string]string) (bool, error) { + deleteExtraFiles, ok := uploaderCfg[DeleteExtraFiles] + if ok { + deleteExtraFilesBool, err := strconv.ParseBool(deleteExtraFiles) + if err != nil { + return false, errors.Wrap(err, "failed to parse DeleteExtraFiles config") + } + return deleteExtraFilesBool, nil + } + return false, nil +} diff --git a/pkg/uploader/util/uploader_config_test.go b/pkg/uploader/util/uploader_config_test.go index 46df8b714..e9628d938 100644 --- a/pkg/uploader/util/uploader_config_test.go +++ b/pkg/uploader/util/uploader_config_test.go @@ -58,6 +58,7 @@ func TestStoreRestoreConfig(t *testing.T) { }, expectedData: map[string]string{ WriteSparseFiles: "true", + DeleteExtraFiles: "false", }, }, { @@ -67,6 +68,7 @@ func TestStoreRestoreConfig(t *testing.T) { }, expectedData: map[string]string{ WriteSparseFiles: "false", + DeleteExtraFiles: "false", }, }, { @@ -76,6 +78,7 @@ func TestStoreRestoreConfig(t *testing.T) { }, expectedData: map[string]string{ WriteSparseFiles: "false", // Assuming default value is false for nil case + DeleteExtraFiles: "false", }, }, { @@ -86,6 +89,37 @@ func TestStoreRestoreConfig(t *testing.T) { expectedData: map[string]string{ RestoreConcurrency: "5", WriteSparseFiles: "false", + DeleteExtraFiles: "false", + }, + }, + { + name: "DeleteExtraFiles is true", + config: &velerov1api.UploaderConfigForRestore{ + DeleteExtraFiles: &boolTrue, + }, + expectedData: map[string]string{ + WriteSparseFiles: "false", + DeleteExtraFiles: "true", + }, + }, + { + name: "DeleteExtraFiles is false", + config: &velerov1api.UploaderConfigForRestore{ + DeleteExtraFiles: &boolFalse, + }, + expectedData: map[string]string{ + WriteSparseFiles: "false", + DeleteExtraFiles: "false", + }, + }, + { + name: "DeleteExtraFiles is nil", + config: &velerov1api.UploaderConfigForRestore{ + DeleteExtraFiles: nil, + }, + expectedData: map[string]string{ + WriteSparseFiles: "false", + DeleteExtraFiles: "false", // Assuming default value is false for nil case }, }, } @@ -240,3 +274,51 @@ func TestGetRestoreConcurrency(t *testing.T) { }) } } + +func TestGetDeleteExtraFiles(t *testing.T) { + tests := []struct { + name string + uploaderCfg map[string]string + expectedResult bool + expectedError error + }{ + { + name: "Valid DeleteExtraFiles (true)", + uploaderCfg: map[string]string{DeleteExtraFiles: "true"}, + expectedResult: true, + expectedError: nil, + }, + { + name: "Valid DeleteExtraFiles (false)", + uploaderCfg: map[string]string{DeleteExtraFiles: "false"}, + expectedResult: false, + expectedError: nil, + }, + { + name: "Invalid DeleteExtraFiles (not a boolean)", + uploaderCfg: map[string]string{DeleteExtraFiles: "invalid"}, + expectedResult: false, + expectedError: errors.Wrap(errors.New("strconv.ParseBool: parsing \"invalid\": invalid syntax"), "failed to parse DeleteExtraFiles config"), + }, + { + name: "Missing DeleteExtraFiles", + uploaderCfg: map[string]string{}, + expectedResult: false, + expectedError: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result, err := GetDeleteExtraFiles(test.uploaderCfg) + + if result != test.expectedResult { + t.Errorf("Expected result %t, but got %t", test.expectedResult, result) + } + + if (err == nil && test.expectedError != nil) || (err != nil && test.expectedError == nil) || (err != nil && test.expectedError != nil && err.Error() != test.expectedError.Error()) { + t.Errorf("Expected error '%v', but got '%v'", test.expectedError, err) + } + }) + } +} diff --git a/pkg/util/azure/credential.go b/pkg/util/azure/credential.go index f36eb43a6..aeaff74f0 100644 --- a/pkg/util/azure/credential.go +++ b/pkg/util/azure/credential.go @@ -37,8 +37,7 @@ func NewCredential(creds map[string]string, options policy.ClientOptions) (azcor // config credential if len(creds[CredentialKeyClientSecret]) > 0 || len(creds[CredentialKeyClientCertificate]) > 0 || - len(creds[CredentialKeyClientCertificatePath]) > 0 || - len(creds[CredentialKeyUsername]) > 0 { + len(creds[CredentialKeyClientCertificatePath]) > 0 { return newConfigCredential(creds, configCredentialOptions{ ClientOptions: options, AdditionallyAllowedTenants: additionalTenants, diff --git a/pkg/util/azure/credential_test.go b/pkg/util/azure/credential_test.go index 40dd5e2c6..d92ee6a8f 100644 --- a/pkg/util/azure/credential_test.go +++ b/pkg/util/azure/credential_test.go @@ -69,6 +69,28 @@ func TestNewCredential(t *testing.T) { assert.IsType(t, &azidentity.WorkloadIdentityCredential{}, tokenCredential) os.Clearenv() + // a leftover AZURE_USERNAME must not hijack credential selection. Username/password + // handling was removed from newConfigCredential in #9041, so routing on it sends the + // caller into a function that cannot serve it and short-circuits the workload + // identity and managed identity branches below. + os.Setenv(CredentialKeyTenantID, "tenantid") + os.Setenv(CredentialKeyClientID, "clientid") + os.Setenv("AZURE_FEDERATED_TOKEN_FILE", "/tmp/token") + creds = map[string]string{CredentialKeyUsername: "username"} + tokenCredential, err = NewCredential(creds, options) + require.NoError(t, err) + assert.IsType(t, &azidentity.WorkloadIdentityCredential{}, tokenCredential) + os.Clearenv() + + // ... and must not short-circuit managed identity either + creds = map[string]string{ + CredentialKeyClientID: "clientid", + CredentialKeyUsername: "username", + } + tokenCredential, err = NewCredential(creds, options) + require.NoError(t, err) + assert.IsType(t, &azidentity.ManagedIdentityCredential{}, tokenCredential) + // managed identity credential creds = map[string]string{CredentialKeyClientID: "clientid"} tokenCredential, err = NewCredential(creds, options) diff --git a/pkg/util/csi/cbt.go b/pkg/util/csi/cbt.go new file mode 100644 index 000000000..00342996d --- /dev/null +++ b/pkg/util/csi/cbt.go @@ -0,0 +1,80 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + "fmt" + "strings" + + "github.com/cockroachdb/errors" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + "github.com/vmware-tanzu/velero/pkg/util" +) + +// CBTInfo define the info for CBT +type CBTInfo struct { + ChangeID string + VolumeID string + SnapshotID string +} + +// GetCBTInfo returns the CBT info for a snapshot +func GetCBTInfo(ctx context.Context, kubeClient kubernetes.Interface, log logrus.FieldLogger, vs *snapshotv1api.VolumeSnapshot, vsc *snapshotv1api.VolumeSnapshotContent, sourcePVName string) (CBTInfo, error) { + cbtInfo := CBTInfo{} + if vs == nil || vsc == nil { + return cbtInfo, errors.New("vs or vsc is nil") + } + + cbtInfo.SnapshotID = vs.Name + + if vs.Annotations != nil && + (vs.Annotations[util.VSphereCNSChangeIDAnno] != "" || + vs.Annotations[util.VSphereCNSSnapshotAnno] != "") { + cbtInfo.ChangeID = vs.Annotations[util.VSphereCNSChangeIDAnno] + + splitSnapshotAnno := strings.Split(vs.Annotations[util.VSphereCNSSnapshotAnno], "+") + if len(splitSnapshotAnno) >= 2 { + cbtInfo.VolumeID = splitSnapshotAnno[0] + } + log.Debugf("volumeID %s and changeID %s are read from VKS annotations.", cbtInfo.VolumeID, cbtInfo.ChangeID) + } else { + pv, err := kubeClient.CoreV1().PersistentVolumes().Get(ctx, sourcePVName, metav1.GetOptions{}) + if err != nil { + return cbtInfo, fmt.Errorf("failed to get pv %s: %w", sourcePVName, err) + } + + if vsc.Status != nil && vsc.Status.SnapshotHandle != nil { + cbtInfo.ChangeID = *vsc.Status.SnapshotHandle + } + + if pv.Spec.CSI != nil && pv.Spec.CSI.VolumeHandle != "" { + cbtInfo.VolumeID = pv.Spec.CSI.VolumeHandle + } + log.Debugf("volumeID %s and changeID %s are read from PV and VS's handles.", cbtInfo.VolumeID, cbtInfo.ChangeID) + } + + if cbtInfo.VolumeID == "" { + return cbtInfo, fmt.Errorf("volumeID must not be empty for CBT") + } + + return cbtInfo, nil +} diff --git a/pkg/util/csi/volume_snapshot.go b/pkg/util/csi/volume_snapshot.go index b78455bc8..330f0a73a 100644 --- a/pkg/util/csi/volume_snapshot.go +++ b/pkg/util/csi/volume_snapshot.go @@ -112,6 +112,7 @@ func WaitVolumeSnapshotReady( // GetVolumeSnapshotContentForVolumeSnapshot returns the VolumeSnapshotContent // object associated with the VolumeSnapshot. func GetVolumeSnapshotContentForVolumeSnapshot( + ctx context.Context, volSnap *snapshotv1api.VolumeSnapshot, snapshotClient snapshotter.SnapshotV1Interface, ) (*snapshotv1api.VolumeSnapshotContent, error) { @@ -120,7 +121,7 @@ func GetVolumeSnapshotContentForVolumeSnapshot( } vsc, err := snapshotClient.VolumeSnapshotContents().Get( - context.TODO(), + ctx, *volSnap.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}, ) @@ -186,6 +187,12 @@ func EnsureDeleteVS(ctx context.Context, snapshotClient snapshotter.SnapshotV1In if err != nil { if errors.Is(err, context.DeadlineExceeded) { + // updated is only set once the VS has been retrieved successfully, so it + // is still nil when the deadline is exceeded before that happens, e.g. + // when the first Get times out. No finalizers are available to report. + if updated == nil { + return errors.Errorf("timeout to assure VolumeSnapshot %s is deleted", vsName) + } return errors.Errorf("timeout to assure VolumeSnapshot %s is deleted, finalizers in VS %v", vsName, updated.Finalizers) } else { return errors.Wrapf(err, "error to assure VolumeSnapshot is deleted, %s", vsName) @@ -245,6 +252,12 @@ func EnsureDeleteVSC(ctx context.Context, snapshotClient snapshotter.SnapshotV1I if err != nil { if errors.Is(err, context.DeadlineExceeded) { + // updated is only set once the VSC has been retrieved successfully, so it + // is still nil when the deadline is exceeded before that happens, e.g. + // when the first Get times out. No finalizers are available to report. + if updated == nil { + return errors.Errorf("timeout to assure VolumeSnapshotContent %s is deleted", vscName) + } return errors.Errorf("timeout to assure VolumeSnapshotContent %s is deleted, finalizers in VSC %v", vscName, updated.Finalizers) } else { return errors.Wrapf(err, "error to assure VolumeSnapshotContent is deleted, %s", vscName) @@ -309,14 +322,16 @@ func patchVSC( } func GetVolumeSnapshotClass( + ctx context.Context, provisioner string, backup *velerov1api.Backup, pvc *corev1api.PersistentVolumeClaim, log logrus.FieldLogger, crClient crclient.Client, + policySnapshotClass string, ) (*snapshotv1api.VolumeSnapshotClass, error) { snapshotClasses := new(snapshotv1api.VolumeSnapshotClassList) - err := crClient.List(context.TODO(), snapshotClasses) + err := crClient.List(ctx, snapshotClasses) if err != nil { return nil, errors.Wrap(err, "error listing VolumeSnapshotClass") } @@ -331,6 +346,16 @@ func GetVolumeSnapshotClass( return snapshotClass, nil } + // If a snapshot class is specified by volume policy, use that + snapshotClass, err = GetVolumeSnapshotClassFromVolumePolicy( + policySnapshotClass, provisioner, snapshotClasses) + if err != nil { + log.Debugf("Didn't find VolumeSnapshotClass from volume policy: %v", err) + } + if snapshotClass != nil { + return snapshotClass, nil + } + // If there is no annotation in PVC, attempt to fetch it from backup annotations snapshotClass, err = GetVolumeSnapshotClassFromBackupAnnotationsForDriver( backup, provisioner, snapshotClasses) @@ -412,6 +437,34 @@ func GetVolumeSnapshotClassFromBackupAnnotationsForDriver( ) } +// GetVolumeSnapshotClassFromVolumePolicy returns a VolumeSnapshotClass +// specified by a volume policy's snapshotClass parameter. If +// policySnapshotClass is empty, it returns nil (no match). +func GetVolumeSnapshotClassFromVolumePolicy( + policySnapshotClass string, + provisioner string, + snapshotClasses *snapshotv1api.VolumeSnapshotClassList, +) (*snapshotv1api.VolumeSnapshotClass, error) { + if policySnapshotClass == "" { + return nil, nil + } + for _, sc := range snapshotClasses.Items { + if strings.EqualFold(policySnapshotClass, sc.ObjectMeta.Name) { + if !strings.EqualFold(sc.Driver, provisioner) { + return nil, errors.Errorf( + "VolumeSnapshotClass %s specified by volume policy is not for driver %s", + sc.ObjectMeta.Name, provisioner, + ) + } + return &sc, nil + } + } + return nil, errors.Errorf( + "No CSI VolumeSnapshotClass found with name %s specified by volume policy for driver %s", + policySnapshotClass, provisioner, + ) +} + // GetVolumeSnapshotClassForStorageClass returns a VolumeSnapshotClass // for the supplied volume provisioner/ driver name. func GetVolumeSnapshotClassForStorageClass( @@ -478,13 +531,14 @@ func IsVolumeSnapshotContentHasDeleteSecret(vsc *snapshotv1api.VolumeSnapshotCon // IsVolumeSnapshotExists returns whether a specific volumesnapshot object exists. func IsVolumeSnapshotExists( + ctx context.Context, ns, name string, crClient crclient.Client, ) bool { vs := new(snapshotv1api.VolumeSnapshot) err := crClient.Get( - context.TODO(), + ctx, crclient.ObjectKey{Namespace: ns, Name: name}, vs, ) @@ -493,24 +547,26 @@ func IsVolumeSnapshotExists( } func SetVolumeSnapshotContentDeletionPolicy( + ctx context.Context, vscName string, crClient crclient.Client, policy snapshotv1api.DeletionPolicy, ) (*snapshotv1api.VolumeSnapshotContent, error) { vsc := new(snapshotv1api.VolumeSnapshotContent) - if err := crClient.Get(context.TODO(), crclient.ObjectKey{Name: vscName}, vsc); err != nil { + if err := crClient.Get(ctx, crclient.ObjectKey{Name: vscName}, vsc); err != nil { return nil, err } originVSC := vsc.DeepCopy() vsc.Spec.DeletionPolicy = policy - return vsc, crClient.Patch(context.TODO(), vsc, crclient.MergeFrom(originVSC)) + return vsc, crClient.Patch(ctx, vsc, crclient.MergeFrom(originVSC)) } // CleanupVolumeSnapshot deletes the VolumeSnapshot and the associated VolumeSnapshotContent. It will make sure the // physical snapshot is also deleted. func CleanupVolumeSnapshot( + ctx context.Context, volSnap *snapshotv1api.VolumeSnapshot, crClient crclient.Client, log logrus.FieldLogger, @@ -518,7 +574,7 @@ func CleanupVolumeSnapshot( log.Infof("Deleting Volumesnapshot %s/%s", volSnap.Namespace, volSnap.Name) vs := new(snapshotv1api.VolumeSnapshot) err := crClient.Get( - context.TODO(), + ctx, crclient.ObjectKey{Name: volSnap.Name, Namespace: volSnap.Namespace}, vs, ) @@ -531,6 +587,7 @@ func CleanupVolumeSnapshot( // we patch the DeletionPolicy of the VolumeSnapshotContent to set it to Delete. // This ensures that the volume snapshot in the storage provider is also deleted. _, err := SetVolumeSnapshotContentDeletionPolicy( + ctx, *vs.Status.BoundVolumeSnapshotContentName, crClient, snapshotv1api.VolumeSnapshotContentDelete, @@ -540,7 +597,7 @@ func CleanupVolumeSnapshot( vs.Namespace, vs.Name) } } - err = crClient.Delete(context.TODO(), vs) + err = crClient.Delete(ctx, vs) if err != nil { log.Debugf("Failed to delete volumesnapshot %s/%s: %v", vs.Namespace, vs.Name, err) } else { @@ -550,6 +607,7 @@ func CleanupVolumeSnapshot( } func DeleteReadyVolumeSnapshot( + ctx context.Context, vs snapshotv1api.VolumeSnapshot, client crclient.Client, logger logrus.FieldLogger, @@ -571,6 +629,7 @@ func DeleteReadyVolumeSnapshot( // Patch the DeletionPolicy of the VolumeSnapshotContent to set it to Retain. // This ensures that the volume snapshot in the storage provider is kept. if vsc, err = SetVolumeSnapshotContentDeletionPolicy( + ctx, *vs.Status.BoundVolumeSnapshotContentName, client, snapshotv1api.VolumeSnapshotContentRetain, @@ -580,11 +639,11 @@ func DeleteReadyVolumeSnapshot( return } - if err := client.Delete(context.TODO(), vsc); err != nil { + if err := client.Delete(ctx, vsc); err != nil { logger.WithError(err).Warnf("Failed to delete the VolumeSnapshotContent %s", vsc.Name) } } - if err := client.Delete(context.TODO(), &vs); err != nil { + if err := client.Delete(ctx, &vs); err != nil { logger.WithError(err).Warnf("Failed to delete VolumeSnapshot %s", vs.Namespace+"/"+vs.Name) } else { logger.Infof("Deleted VolumeSnapshot %s and VolumeSnapshotContent %s", @@ -654,7 +713,7 @@ func WaitUntilVSCHandleIsReady( if vsc.Status != nil && vsc.Status.Error != nil { log.Warnf("VolumeSnapshotContent %s has error: %v", - vsc.Name, *vsc.Status.Error.Message) + vsc.Name, stringptr.GetString(vsc.Status.Error.Message)) } return false, nil } @@ -705,10 +764,10 @@ func WaitUntilVSCHandleIsReady( vsc.Status.Error != nil { log.Errorf( "Timed out awaiting reconciliation of VolumeSnapshot, VolumeSnapshotContent %s has error: %v", - vsc.Name, *vsc.Status.Error.Message) + vsc.Name, stringptr.GetString(vsc.Status.Error.Message)) return nil, errors.Errorf("CSI got timed out with error: %v", - *vsc.Status.Error.Message) + stringptr.GetString(vsc.Status.Error.Message)) } else { log.Errorf( "Timed out awaiting reconciliation of volumesnapshot %s/%s", diff --git a/pkg/util/csi/volume_snapshot_test.go b/pkg/util/csi/volume_snapshot_test.go index 67a07d135..895935b4b 100644 --- a/pkg/util/csi/volume_snapshot_test.go +++ b/pkg/util/csi/volume_snapshot_test.go @@ -17,6 +17,7 @@ limitations under the License. package csi import ( + "context" "errors" "testing" "time" @@ -201,7 +202,7 @@ func TestWaitVolumeSnapshotReady(t *testing.T) { fakeClient := snapshotFake.NewSimpleClientset(test.clientObj...) vs, err := WaitVolumeSnapshotReady(t.Context(), fakeClient.SnapshotV1(), test.vsName, test.namespace, time.Millisecond, velerotest.NewLogger()) - if err != nil { + if test.err != "" { require.EqualError(t, err, test.err) } else { require.NoError(t, err) @@ -286,8 +287,8 @@ func TestGetVolumeSnapshotContentForVolumeSnapshot(t *testing.T) { t.Run(test.name, func(t *testing.T) { fakeClient := snapshotFake.NewSimpleClientset(test.clientObj...) - vs, err := GetVolumeSnapshotContentForVolumeSnapshot(test.snapshotObj, fakeClient.SnapshotV1()) - if err != nil { + vs, err := GetVolumeSnapshotContentForVolumeSnapshot(context.TODO(), test.snapshotObj, fakeClient.SnapshotV1()) + if test.err != "" { require.EqualError(t, err, test.err) } else { require.NoError(t, err) @@ -376,6 +377,29 @@ func TestEnsureDeleteVS(t *testing.T) { }, err: "timeout to assure VolumeSnapshot fake-vs is deleted, finalizers in VS []", }, + { + name: "wait timeout before the VS is ever retrieved", + vsName: "fake-vs", + namespace: "fake-ns", + clientObj: []runtime.Object{vsObjWithFinalizer}, + reactors: []reactor{ + { + verb: "delete", + resource: "volumesnapshots", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, nil + }, + }, + { + verb: "get", + resource: "volumesnapshots", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, context.DeadlineExceeded + }, + }, + }, + err: "timeout to assure VolumeSnapshot fake-vs is deleted", + }, { name: "success", vsName: "fake-vs", @@ -393,7 +417,7 @@ func TestEnsureDeleteVS(t *testing.T) { } err := EnsureDeleteVS(t.Context(), fakeSnapshotClient.SnapshotV1(), test.vsName, test.namespace, time.Millisecond) - if err != nil { + if test.err != "" { assert.EqualError(t, err, test.err) } else { assert.NoError(t, err) @@ -487,6 +511,28 @@ func TestEnsureDeleteVSC(t *testing.T) { }, err: "timeout to assure VolumeSnapshotContent fake-vsc is deleted, finalizers in VSC []", }, + { + name: "wait timeout before the VSC is ever retrieved", + vscName: "fake-vsc", + clientObj: []runtime.Object{vscObjWithFinalizer}, + reactors: []reactor{ + { + verb: "delete", + resource: "volumesnapshotcontents", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, nil + }, + }, + { + verb: "get", + resource: "volumesnapshotcontents", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, context.DeadlineExceeded + }, + }, + }, + err: "timeout to assure VolumeSnapshotContent fake-vsc is deleted", + }, { name: "success", vscName: "fake-vsc", @@ -1032,7 +1078,8 @@ func TestGetVolumeSnapshotClass(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { actualSnapshotClass, actualError := GetVolumeSnapshotClass( - tc.driverName, tc.backup, tc.pvc, logrus.New(), fakeClient) + context.TODO(), + tc.driverName, tc.backup, tc.pvc, logrus.New(), fakeClient, "") if tc.expectError { require.Error(t, actualError) assert.Nil(t, actualSnapshotClass) @@ -1043,6 +1090,93 @@ func TestGetVolumeSnapshotClass(t *testing.T) { } } +func TestGetVolumeSnapshotClassFromVolumePolicy(t *testing.T) { + vscArray1 := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{Name: "vsc-array-1"}, + Driver: "infinibox-csi-driver", + } + vscArray2 := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{Name: "vsc-array-2"}, + Driver: "infinibox-csi-driver", + } + vscOther := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{Name: "vsc-other"}, + Driver: "other-csi-driver", + } + + snapshotClasses := &snapshotv1api.VolumeSnapshotClassList{ + Items: []snapshotv1api.VolumeSnapshotClass{*vscArray1, *vscArray2, *vscOther}, + } + + testCases := []struct { + name string + policySnapshotClass string + provisioner string + expectedVSC *snapshotv1api.VolumeSnapshotClass + expectError bool + }{ + { + name: "empty policy returns nil", + policySnapshotClass: "", + provisioner: "infinibox-csi-driver", + expectedVSC: nil, + expectError: false, + }, + { + name: "matching VSC with correct driver", + policySnapshotClass: "vsc-array-1", + provisioner: "infinibox-csi-driver", + expectedVSC: vscArray1, + expectError: false, + }, + { + name: "matching VSC with correct driver second array", + policySnapshotClass: "vsc-array-2", + provisioner: "infinibox-csi-driver", + expectedVSC: vscArray2, + expectError: false, + }, + { + name: "VSC exists but wrong driver", + policySnapshotClass: "vsc-other", + provisioner: "infinibox-csi-driver", + expectError: true, + }, + { + name: "VSC does not exist", + policySnapshotClass: "non-existent", + provisioner: "infinibox-csi-driver", + expectError: true, + }, + { + name: "case-insensitive name matching", + policySnapshotClass: "VSC-ARRAY-1", + provisioner: "infinibox-csi-driver", + expectedVSC: vscArray1, + expectError: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualVSC, actualError := GetVolumeSnapshotClassFromVolumePolicy( + tc.policySnapshotClass, tc.provisioner, snapshotClasses) + if tc.expectError { + require.Error(t, actualError) + assert.Nil(t, actualVSC) + return + } + if tc.expectedVSC == nil { + assert.Nil(t, actualVSC) + } else { + require.NotNil(t, actualVSC) + assert.Equal(t, tc.expectedVSC.Name, actualVSC.Name) + assert.Equal(t, tc.expectedVSC.Driver, actualVSC.Driver) + } + }) + } +} + func TestGetVolumeSnapshotClassForStorageClass(t *testing.T) { hostpathClass := &snapshotv1api.VolumeSnapshotClass{ ObjectMeta: metav1.ObjectMeta{ @@ -1371,7 +1505,7 @@ func TestIsVolumeSnapshotExists(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - actual := IsVolumeSnapshotExists(tc.vs.Namespace, tc.vs.Name, fakeClient) + actual := IsVolumeSnapshotExists(context.TODO(), tc.vs.Namespace, tc.vs.Name, fakeClient) assert.Equal(t, tc.expected, actual) }) } @@ -1442,7 +1576,7 @@ func TestSetVolumeSnapshotContentDeletionPolicy(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { fakeClient := velerotest.NewFakeControllerRuntimeClient(t, tc.objs...) - _, err := SetVolumeSnapshotContentDeletionPolicy(tc.inputVSCName, fakeClient, tc.policy) + _, err := SetVolumeSnapshotContentDeletionPolicy(context.TODO(), tc.inputVSCName, fakeClient, tc.policy) if tc.expectError { assert.Error(t, err) } else { @@ -1499,7 +1633,7 @@ func TestDeleteVolumeSnapshots(t *testing.T) { ) logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatText) - DeleteReadyVolumeSnapshot(tc.vs, client, logger) + DeleteReadyVolumeSnapshot(context.TODO(), tc.vs, client, logger) vsList := new(snapshotv1api.VolumeSnapshotList) err := client.List( @@ -1632,6 +1766,34 @@ func TestWaitUntilVSCHandleIsReady(t *testing.T) { }, } + errNoMessageVsc := "err-no-message-vsc" + vscWithErrorNoMessage := &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: errNoMessageVsc, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + VolumeSnapshotRef: corev1api.ObjectReference{ + Name: "vol-snap-1", + APIVersion: snapshotv1api.SchemeGroupVersion.String(), + }, + }, + Status: &snapshotv1api.VolumeSnapshotContentStatus{ + SnapshotHandle: nil, + // Error is set while Message is left nil. Both are optional in the + // CSI API, so the error-reporting paths must not dereference Message. + Error: &snapshotv1api.VolumeSnapshotError{Message: nil}, + }, + } + vsForErrorNoMessageVsc := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-for-err-no-message", + Namespace: "default", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &errNoMessageVsc, + }, + } + objs := []runtime.Object{ vscObj, validVS, @@ -1642,6 +1804,8 @@ func TestWaitUntilVSCHandleIsReady(t *testing.T) { vsForNilStatusVsc, vscWithNilStatusField, vsForNilStatusFieldVsc, + vscWithErrorNoMessage, + vsForErrorNoMessageVsc, } fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) testCases := []struct { @@ -1676,6 +1840,12 @@ func TestWaitUntilVSCHandleIsReady(t *testing.T) { }, }, }, + { + name: "waitEnabled should return an error rather than panic when the volumesnapshotcontent has an error without a message", + volSnap: vsForErrorNoMessageVsc, + exepctedVSC: nil, + expectError: true, + }, } for _, tc := range testCases { diff --git a/pkg/util/datamover/datamover.go b/pkg/util/datamover/datamover.go index 59dd1499b..7815f7a4a 100644 --- a/pkg/util/datamover/datamover.go +++ b/pkg/util/datamover/datamover.go @@ -20,6 +20,7 @@ limitations under the License. package datamover const ( + DataMoverTypeEmpty = "" // DataMoverTypeVelero refers to the default built-in data mover. The default // data mover may change among releases; see GetDefaultBuiltInDataMover. DataMoverTypeVelero = "velero" @@ -32,7 +33,20 @@ const ( // IsBuiltInDataMover reports whether the given data mover value refers to a // Velero built-in data mover (an empty value or the default "velero" alias). func IsBuiltInDataMover(dataMover string) bool { - return dataMover == "" || dataMover == DataMoverTypeVelero + return IsVeleroBlockDataMover(dataMover) || IsVeleroFSDataMover(dataMover) +} + +// IsVeleroFSDataMover checks whether the given data mover belongs to fs type. +func IsVeleroFSDataMover(dataMover string) bool { + if dataMover == "" || dataMover == DataMoverTypeVelero { + dataMover = DataMoverTypeVeleroFs + } + return dataMover == DataMoverTypeVeleroFs +} + +// IsVeleroBlockDataMover checks whether the given data mover belongs to block type. +func IsVeleroBlockDataMover(dataMover string) bool { + return dataMover == DataMoverTypeVeleroBlock } // GetDefaultBuiltInDataMover returns the data mover used when the default diff --git a/pkg/util/datamover/datamover_test.go b/pkg/util/datamover/datamover_test.go index 8576aed0e..94585e8f9 100644 --- a/pkg/util/datamover/datamover_test.go +++ b/pkg/util/datamover/datamover_test.go @@ -38,6 +38,16 @@ func TestIsBuiltInDataMover(t *testing.T) { dataMover: "velero", want: true, }, + { + name: "velero-fs dataMover is builtin", + dataMover: "velero-fs", + want: true, + }, + { + name: "velero-block dataMover is builtin", + dataMover: "velero-block", + want: true, + }, { name: "kopia dataMover is not builtin", dataMover: "kopia", @@ -54,3 +64,61 @@ func TestIsBuiltInDataMover(t *testing.T) { func TestGetDefaultBuiltInDataMover(t *testing.T) { assert.Equal(t, DataMoverTypeVeleroFs, GetDefaultBuiltInDataMover()) } + +func TestIsFSDataMover(t *testing.T) { + testcases := []struct { + name string + dataMover string + want bool + }{ + { + name: "empty dataMover is fs", + dataMover: "", + want: true, + }, + { + name: "velero dataMover is fs", + dataMover: "velero", + want: true, + }, + { + name: "velero-fs dataMover is fs", + dataMover: "velero-fs", + want: true, + }, + { + name: "velero-block dataMover is not fs", + dataMover: "velero-block", + want: false, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(tt *testing.T) { + assert.Equal(tt, tc.want, IsVeleroFSDataMover(tc.dataMover)) + }) + } +} + +func TestIsBlockDataMover(t *testing.T) { + testcases := []struct { + name string + dataMover string + want bool + }{ + { + name: "velero-block dataMover is block", + dataMover: "velero-block", + want: true, + }, + { + name: "velero-fs dataMover is not block", + dataMover: "velero-fs", + want: false, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(tt *testing.T) { + assert.Equal(tt, tc.want, IsVeleroBlockDataMover(tc.dataMover)) + }) + } +} diff --git a/pkg/util/kube/node.go b/pkg/util/kube/node.go index 3426e508f..6fc2974c9 100644 --- a/pkg/util/kube/node.go +++ b/pkg/util/kube/node.go @@ -30,7 +30,7 @@ import ( const ( NodeOSLinux = "linux" NodeOSWindows = "windows" - NodeOSLabel = "kubernetes.io/os" + NodeOSLabel = corev1api.LabelOSStable ) var realNodeOSMap = map[string]string{ diff --git a/pkg/util/kube/node_test.go b/pkg/util/kube/node_test.go index 612b8f977..41e7b7806 100644 --- a/pkg/util/kube/node_test.go +++ b/pkg/util/kube/node_test.go @@ -35,8 +35,8 @@ import ( func TestIsLinuxNode(t *testing.T) { nodeNoOSLabel := builder.ForNode("fake-node").Result() - nodeWindows := builder.ForNode("fake-node").Labels(map[string]string{"kubernetes.io/os": "windows"}).Result() - nodeLinux := builder.ForNode("fake-node").Labels(map[string]string{"kubernetes.io/os": "linux"}).Result() + nodeWindows := builder.ForNode("fake-node").Labels(map[string]string{corev1api.LabelOSStable: "windows"}).Result() + nodeLinux := builder.ForNode("fake-node").Labels(map[string]string{corev1api.LabelOSStable: "linux"}).Result() scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) @@ -90,8 +90,8 @@ func TestIsLinuxNode(t *testing.T) { } func TestWithLinuxNode(t *testing.T) { - nodeWindows := builder.ForNode("fake-node-1").Labels(map[string]string{"kubernetes.io/os": "windows"}).Result() - nodeLinux := builder.ForNode("fake-node-2").Labels(map[string]string{"kubernetes.io/os": "linux"}).Result() + nodeWindows := builder.ForNode("fake-node-1").Labels(map[string]string{corev1api.LabelOSStable: "windows"}).Result() + nodeLinux := builder.ForNode("fake-node-2").Labels(map[string]string{corev1api.LabelOSStable: "linux"}).Result() scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) @@ -135,8 +135,8 @@ func TestWithLinuxNode(t *testing.T) { func TestGetNodeOSType(t *testing.T) { nodeNoOSLabel := builder.ForNode("fake-node").Result() - nodeWindows := builder.ForNode("fake-node").Labels(map[string]string{"kubernetes.io/os": "windows"}).Result() - nodeLinux := builder.ForNode("fake-node").Labels(map[string]string{"kubernetes.io/os": "linux"}).Result() + nodeWindows := builder.ForNode("fake-node").Labels(map[string]string{corev1api.LabelOSStable: "windows"}).Result() + nodeLinux := builder.ForNode("fake-node").Labels(map[string]string{corev1api.LabelOSStable: "linux"}).Result() scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) tests := []struct { @@ -185,8 +185,8 @@ func TestGetNodeOSType(t *testing.T) { func TestHasNodeWithOS(t *testing.T) { nodeNoOSLabel := builder.ForNode("fake-node-1").Result() - nodeWindows := builder.ForNode("fake-node-2").Labels(map[string]string{"kubernetes.io/os": "windows"}).Result() - nodeLinux := builder.ForNode("fake-node-3").Labels(map[string]string{"kubernetes.io/os": "linux"}).Result() + nodeWindows := builder.ForNode("fake-node-2").Labels(map[string]string{corev1api.LabelOSStable: "windows"}).Result() + nodeLinux := builder.ForNode("fake-node-3").Labels(map[string]string{corev1api.LabelOSStable: "linux"}).Result() scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) diff --git a/pkg/util/kube/pod.go b/pkg/util/kube/pod.go index 3ced95feb..25857cd02 100644 --- a/pkg/util/kube/pod.go +++ b/pkg/util/kube/pod.go @@ -129,6 +129,12 @@ func EnsureDeletePod(ctx context.Context, podGetter corev1client.CoreV1Interface if err != nil { if errors.Is(err, context.DeadlineExceeded) { + // updated is only set once the pod has been retrieved successfully, so it + // is still nil when the deadline is exceeded before that happens, e.g. + // when the first Get times out. No finalizers are available to report. + if updated == nil { + return errors.Errorf("timeout to assure pod %s is deleted", pod) + } return errors.Errorf("timeout to assure pod %s is deleted, finalizers in pod %v", pod, updated.Finalizers) } else { return errors.Wrapf(err, "error to assure pod is deleted, %s", pod) @@ -318,9 +324,26 @@ func ExitPodWithMessage(logger logrus.FieldLogger, succeed bool, message string, funcExit(exitCode) } +// deepCopy returns a deep copy of the LoadAffinity, so that the returned value +// can be safely modified without affecting the source. +func (a *LoadAffinity) deepCopy() *LoadAffinity { + if a == nil { + return nil + } + + result := &LoadAffinity{ + StorageClass: a.StorageClass, + } + a.NodeSelector.DeepCopyInto(&result.NodeSelector) + + return result +} + // GetLoadAffinityByStorageClass retrieves the LoadAffinity from the parameter affinityList. // The function first try to find by the scName. If there is no such LoadAffinity, // it will try to get the LoadAffinity whose StorageClass has no value. +// The returned LoadAffinity is a deep copy of the matched element, so that the +// callers can modify it without corrupting the shared node-agent configuration. func GetLoadAffinityByStorageClass( affinityList []*LoadAffinity, scName string, @@ -331,7 +354,7 @@ func GetLoadAffinityByStorageClass( for _, affinity := range affinityList { if affinity.StorageClass == scName { logger.WithField("StorageClass", scName).Info("Found pod's affinity setting per StorageClass.") - return affinity + return affinity.deepCopy() } if affinity.StorageClass == "" && globalAffinity == nil { @@ -345,5 +368,5 @@ func GetLoadAffinityByStorageClass( logger.Info("No Affinity is found for pod.") } - return globalAffinity + return globalAffinity.deepCopy() } diff --git a/pkg/util/kube/pod_test.go b/pkg/util/kube/pod_test.go index 1d54071c3..ec82fd1f8 100644 --- a/pkg/util/kube/pod_test.go +++ b/pkg/util/kube/pod_test.go @@ -106,6 +106,29 @@ func TestEnsureDeletePod(t *testing.T) { }, err: "timeout to assure pod fake-pod is deleted, finalizers in pod []", }, + { + name: "wait timeout before the pod is ever retrieved", + podName: "fake-pod", + namespace: "fake-ns", + clientObj: []runtime.Object{podObjectWithFinalizer}, + reactors: []reactor{ + { + verb: "delete", + resource: "pods", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, nil + }, + }, + { + verb: "get", + resource: "pods", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, context.DeadlineExceeded + }, + }, + }, + err: "timeout to assure pod fake-pod is deleted", + }, { name: "wait fail", podName: "fake-pod", @@ -1374,7 +1397,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/arch", + Key: corev1api.LabelArchStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"amd64"}, }, @@ -1386,7 +1409,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"Linux"}, }, @@ -1399,7 +1422,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"Linux"}, }, @@ -1414,7 +1437,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"Linux"}, }, @@ -1425,7 +1448,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/arch", + Key: corev1api.LabelArchStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"amd64"}, }, @@ -1436,7 +1459,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"Windows"}, }, @@ -1449,7 +1472,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"Linux"}, }, @@ -1475,7 +1498,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"Linux"}, }, @@ -1487,7 +1510,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/arch", + Key: corev1api.LabelArchStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"amd64"}, }, @@ -1501,7 +1524,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"Linux"}, }, @@ -1517,7 +1540,7 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { - Key: "kubernetes.io/arch", + Key: corev1api.LabelArchStable, Operator: metav1.LabelSelectorOpIn, Values: []string{"amd64"}, }, @@ -1545,3 +1568,82 @@ func TestGetLoadAffinityByStorageClass(t *testing.T) { }) } } + +func TestGetLoadAffinityByStorageClassReturnsCopy(t *testing.T) { + newAffinityList := func() []*LoadAffinity { + return []*LoadAffinity{ + { + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"pool": "backup"}, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: corev1api.LabelArchStable, + Operator: metav1.LabelSelectorOpIn, + Values: []string{"amd64"}, + }, + }, + }, + }, + { + NodeSelector: metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: corev1api.LabelArchStable, + Operator: metav1.LabelSelectorOpIn, + Values: []string{"arm64"}, + }, + }, + }, + StorageClass: "storage-class-01", + }, + } + } + + tests := []struct { + name string + scName string + }{ + { + name: "global affinity", + scName: "no-such-storage-class", + }, + { + name: "affinity matched by StorageClass", + scName: "storage-class-01", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + affinityList := newAffinityList() + + // Simulate the exposers, which append an OS related term to the returned + // affinity on every expose call. The source list must not be affected. + for range 3 { + result := GetLoadAffinityByStorageClass(affinityList, test.scName, velerotest.NewLogger()) + require.NotNil(t, result) + + result.NodeSelector.MatchExpressions = append(result.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: NodeOSLabel, + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{NodeOSWindows}, + }) + + assert.Len(t, result.NodeSelector.MatchExpressions, 2) + } + + assert.Equal(t, newAffinityList(), affinityList) + + // The other fields must be copied as well. + result := GetLoadAffinityByStorageClass(affinityList, test.scName, velerotest.NewLogger()) + require.NotNil(t, result) + result.StorageClass = "modified" + result.NodeSelector.MatchExpressions[0].Values[0] = "modified" + if result.NodeSelector.MatchLabels != nil { + result.NodeSelector.MatchLabels["pool"] = "modified" + } + + assert.Equal(t, newAffinityList(), affinityList) + }) + } +} diff --git a/pkg/util/kube/pvc_pv.go b/pkg/util/kube/pvc_pv.go index 7db9df3e4..49d0bbc60 100644 --- a/pkg/util/kube/pvc_pv.go +++ b/pkg/util/kube/pvc_pv.go @@ -35,6 +35,7 @@ import ( corev1client "k8s.io/client-go/kubernetes/typed/core/v1" crclient "sigs.k8s.io/controller-runtime/pkg/client" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" storagev1api "k8s.io/api/storage/v1" storagev1 "k8s.io/client-go/kubernetes/typed/storage/v1" ) @@ -95,6 +96,10 @@ func WaitPVCBound(ctx context.Context, pvcGetter corev1client.CoreV1Interface, return false, nil } + if tmpPVC.Status.Phase != corev1api.ClaimBound { + return false, nil + } + updated = tmpPVC return true, nil @@ -112,6 +117,16 @@ func WaitPVCBound(ctx context.Context, pvcGetter corev1client.CoreV1Interface, return pv, err } +// DeletePVCIfAny deletes a PVC by namespace and name if it exists, and log an error when the deletion fails +func DeletePVCIfAny(ctx context.Context, client corev1client.CoreV1Interface, pvcName, pvcNamespace string, ensureTimeout time.Duration, log logrus.FieldLogger) { + if err := EnsureDeletePVC(ctx, client, pvcName, pvcNamespace, ensureTimeout); err != nil { + if apierrors.IsNotFound(err) { + return + } + log.Warnf("failed to delete pvc %s/%s with err %v", pvcNamespace, pvcName, err) + } +} + // DeletePVIfAny deletes a PV by name if it exists, and log an error when the deletion fails func DeletePVIfAny(ctx context.Context, pvGetter corev1client.CoreV1Interface, pvName string, log logrus.FieldLogger) { err := pvGetter.PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}) @@ -124,6 +139,47 @@ func DeletePVIfAny(ctx context.Context, pvGetter corev1client.CoreV1Interface, p } } +// EnsureDeleteVolumeSnapshotIfAny deletes a VolumeSnapshot by namespace and name if it exists, and log an error when the deletion fails +func EnsureDeleteVolumeSnapshotIfAny(ctx context.Context, client crclient.Client, namespace, name string, ensureTimeout time.Duration, log logrus.FieldLogger) { + if err := client.Delete(ctx, &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + }); err != nil && !apierrors.IsNotFound(err) { + log.WithError(err).Errorf("Failed to delete the VolumeSnapshot %s/%s", namespace, name) + } + + if ensureTimeout == 0 { + return + } + + var updated *snapshotv1api.VolumeSnapshot + err := wait.PollUntilContextTimeout(ctx, waitInternal, ensureTimeout, true, func(ctx context.Context) (bool, error) { + if err := client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, updated); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + + return false, errors.Wrapf(err, "error to get VolumeSnapshot %s/%s", namespace, name) + } + + return false, nil + }) + + if err != nil { + if errors.Is(err, context.DeadlineExceeded) { + if updated == nil { + log.WithError(err).Errorf("Timeout to assure VolumeSnapshot %s/%s is deleted", namespace, name) + } else { + log.WithError(err).Errorf("Timeout to assure VolumeSnapshot %s/%s is deleted, finalizers in VolumeSnapshot %v", namespace, name, updated.Finalizers) + } + } else { + log.WithError(err).Errorf("Error to assure VolumeSnapshot %s/%s is deleted", namespace, name) + } + } +} + // EnsureDeletePVC asserts the existence of a PVC by name, deletes it and waits for its disappearance and returns errors on any failure // If timeout is 0, it doesn't wait and return nil func EnsureDeletePVC(ctx context.Context, pvcGetter corev1client.CoreV1Interface, pvcName string, namespace string, timeout time.Duration) error { @@ -153,6 +209,12 @@ func EnsureDeletePVC(ctx context.Context, pvcGetter corev1client.CoreV1Interface if err != nil { if errors.Is(err, context.DeadlineExceeded) { + // updated is only set once the PVC has been retrieved successfully, so it + // is still nil when the deadline is exceeded before that happens, e.g. + // when the first Get times out. No finalizers are available to report. + if updated == nil { + return errors.Errorf("timeout to assure pvc %s is deleted", pvcName) + } return errors.Errorf("timeout to assure pvc %s is deleted, finalizers in pvc %v", pvcName, updated.Finalizers) } else { return errors.Wrapf(err, "error to ensure pvc deleted for %s", pvcName) @@ -189,6 +251,12 @@ func EnsureDeletePV(ctx context.Context, pvGetter corev1client.CoreV1Interface, if err != nil { if errors.Is(err, context.DeadlineExceeded) { + // updated is only set once the PV has been retrieved successfully, so it + // is still nil when the deadline is exceeded before that happens, e.g. + // when the first Get times out. No finalizers are available to report. + if updated == nil { + return errors.Errorf("timeout to assure pv %s is deleted", pvName) + } return errors.Errorf("timeout to assure pv %s is deleted, finalizers in pv %v", pvName, updated.Finalizers) } else { return errors.Wrapf(err, "error to ensure pv deleted for %s", pvName) diff --git a/pkg/util/kube/pvc_pv_test.go b/pkg/util/kube/pvc_pv_test.go index 9b93f2971..fb6eb4947 100644 --- a/pkg/util/kube/pvc_pv_test.go +++ b/pkg/util/kube/pvc_pv_test.go @@ -17,6 +17,7 @@ limitations under the License. package kube import ( + "context" "testing" "time" @@ -61,6 +62,9 @@ func TestWaitPVCBound(t *testing.T) { Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "fake-pv", }, + Status: corev1api.PersistentVolumeClaimStatus{ + Phase: corev1api.ClaimBound, + }, } pvObj := &corev1api.PersistentVolume{ @@ -303,6 +307,105 @@ func TestWaitPVCConsumed(t *testing.T) { } func TestDeletePVCIfAny(t *testing.T) { + pvcObject := &corev1api.PersistentVolumeClaim{ + TypeMeta: metav1.TypeMeta{ + Kind: "fake-kind-1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "fake-namespace", + Name: "fake-pvc", + }, + } + + tests := []struct { + name string + pvcName string + pvcNamespace string + kubeClientObj []runtime.Object + kubeReactors []reactor + logMessage string + logLevel string + ensureTimeout time.Duration + }{ + { + name: "pvc not found", + pvcName: "fake-pvc", + pvcNamespace: "fake-namespace", + }, + { + name: "failed to delete pvc", + pvcName: "fake-pvc", + pvcNamespace: "fake-namespace", + kubeReactors: []reactor{ + { + verb: "delete", + resource: "persistentvolumeclaims", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, errors.New("fake-delete-error") + }, + }, + }, + kubeClientObj: []runtime.Object{ + pvcObject, + }, + logMessage: "failed to delete pvc fake-namespace/fake-pvc with err error to delete pvc fake-pvc: fake-delete-error", + logLevel: "level=warning", + }, + { + name: "delete pvc success", + pvcName: "fake-pvc", + pvcNamespace: "fake-namespace", + kubeClientObj: []runtime.Object{ + pvcObject, + }, + }, + { + name: "delete pvc success but wait fail", + pvcName: "fake-pvc", + pvcNamespace: "fake-namespace", + kubeClientObj: []runtime.Object{ + pvcObject, + }, + kubeReactors: []reactor{ + { + verb: "delete", + resource: "persistentvolumeclaims", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, pvcObject, nil + }, + }, + }, + ensureTimeout: time.Second, + logMessage: "failed to delete pvc fake-namespace/fake-pvc with err timeout to assure pvc fake-pvc is deleted, finalizers in pvc []", + logLevel: "level=warning", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) + + for _, reactor := range test.kubeReactors { + fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) + } + + var kubeClient kubernetes.Interface = fakeKubeClient + + logMessage := "" + DeletePVCIfAny(t.Context(), kubeClient.CoreV1(), test.pvcName, test.pvcNamespace, test.ensureTimeout, velerotest.NewSingleLogger(&logMessage)) + + if len(test.logMessage) > 0 { + assert.Contains(t, logMessage, test.logMessage) + } + + if len(test.logLevel) > 0 { + assert.Contains(t, logMessage, test.logLevel) + } + }) + } +} + +func TestDeletePVAndPVCIfAny(t *testing.T) { pvObject := &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", @@ -687,6 +790,30 @@ func TestEnsureDeletePVC(t *testing.T) { }, err: "timeout to assure pvc fake-pvc is deleted, finalizers in pvc []", }, + { + name: "wait timeout before the pvc is ever retrieved", + pvcName: "fake-pvc", + namespace: "fake-ns", + clientObj: []runtime.Object{pvcObjectWithFinalizer}, + timeout: time.Millisecond, + reactors: []reactor{ + { + verb: "delete", + resource: "persistentvolumeclaims", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, pvcObject, nil + }, + }, + { + verb: "get", + resource: "persistentvolumeclaims", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, context.DeadlineExceeded + }, + }, + }, + err: "timeout to assure pvc fake-pvc is deleted", + }, } for _, test := range tests { @@ -1729,7 +1856,7 @@ func TestDiagnosePV(t *testing.T) { func TestGetPVCAttachingNodeOS(t *testing.T) { storageClass := "fake-storage-class" nodeNoOSLabel := builder.ForNode("fake-node").Result() - nodeWindows := builder.ForNode("fake-node").Labels(map[string]string{"kubernetes.io/os": "windows"}).Result() + nodeWindows := builder.ForNode("fake-node").Labels(map[string]string{corev1api.LabelOSStable: "windows"}).Result() pvcObj := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ @@ -2059,6 +2186,13 @@ func TestEnsureDeletePV(t *testing.T) { }, } + pvObjWithFinalizer := &corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-pv", + Finalizers: []string{"fake-finalizer-1", "fake-finalizer-2"}, + }, + } + tests := []struct { name string pvName string @@ -2134,6 +2268,29 @@ func TestEnsureDeletePV(t *testing.T) { }, expectedErr: "timeout to assure pv fake-pv is deleted, finalizers in pv []", }, + { + name: "wait timeout before the pv is ever retrieved", + pvName: "fake-pv", + timeout: time.Millisecond, + kubeClientObj: []runtime.Object{pvObjWithFinalizer}, + kubeReactors: []reactor{ + { + verb: "delete", + resource: "persistentvolumes", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, nil + }, + }, + { + verb: "get", + resource: "persistentvolumes", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, context.DeadlineExceeded + }, + }, + }, + expectedErr: "timeout to assure pv fake-pv is deleted", + }, } for _, test := range tests { diff --git a/pkg/util/kube/secrets.go b/pkg/util/kube/secrets.go index f1d19b84e..e949b0e97 100644 --- a/pkg/util/kube/secrets.go +++ b/pkg/util/kube/secrets.go @@ -18,9 +18,14 @@ package kube import ( "context" + "reflect" "github.com/cockroachdb/errors" + "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" kbclient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -49,3 +54,151 @@ func GetSecretKey(client kbclient.Client, namespace string, selector *corev1api. return key, nil } + +// ErrSecretCollision is returned when a secret or configmap with the same name but different +// data already exists in the target namespace, indicating another owner is using it. +var ErrSecretCollision = errors.New("secret collision: same name exists with different data") + +// labelsMatch reports whether all entries in want are present in have with matching values. +func labelsMatch(have, want map[string]string) bool { + for k, v := range want { + if have[k] != v { + return false + } + } + return true +} + +// CopySecret copies a secret from sourceNamespace to targetNamespace, applying the given labels. +// If a secret with the same name already exists in the target with identical data and matching +// labels, it is a no-op. If the data matches but the labels differ, or the data differs, it +// returns ErrSecretCollision. +func CopySecret(ctx context.Context, client corev1client.CoreV1Interface, secretName, sourceNamespace, targetNamespace string, labels map[string]string, log logrus.FieldLogger) error { + srcSecret, err := client.Secrets(sourceNamespace).Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "error getting secret %s/%s", sourceNamespace, secretName) + } + + newSecret := &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: targetNamespace, + Labels: labels, + }, + Type: srcSecret.Type, + Data: srcSecret.Data, + } + + _, err = client.Secrets(targetNamespace).Create(ctx, newSecret, metav1.CreateOptions{}) + if err == nil { + log.Infof("Copied secret %s from %s to %s", secretName, sourceNamespace, targetNamespace) + return nil + } + + if !apierrors.IsAlreadyExists(err) { + return errors.Wrapf(err, "error creating secret %s in %s", secretName, targetNamespace) + } + + existing, err := client.Secrets(targetNamespace).Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "error getting existing secret %s/%s", targetNamespace, secretName) + } + + if reflect.DeepEqual(existing.Data, srcSecret.Data) && labelsMatch(existing.Labels, labels) { + log.Infof("Secret %s already exists in %s with same data and labels, skipping copy", secretName, targetNamespace) + return nil + } + + log.Infof("Secret %s already exists in %s owned by a different owner, collision detected", secretName, targetNamespace) + return ErrSecretCollision +} + +// DeleteSecretsWithLabel deletes all secrets in a namespace matching a label key=value pair. +// Uses UID preconditions to avoid deleting a recreated object with the same name. +func DeleteSecretsWithLabel(ctx context.Context, client corev1client.CoreV1Interface, namespace, labelKey, labelValue string, log logrus.FieldLogger) { + secrets, err := client.Secrets(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: labelKey + "=" + labelValue, + }) + if err != nil { + log.WithError(err).Errorf("Failed to list secrets with label %s=%s in %s", labelKey, labelValue, namespace) + return + } + + for i := range secrets.Items { + uid := secrets.Items[i].UID + err := client.Secrets(namespace).Delete(ctx, secrets.Items[i].Name, metav1.DeleteOptions{ + Preconditions: &metav1.Preconditions{UID: &uid}, + }) + if err != nil && !apierrors.IsNotFound(err) { + log.WithError(err).Errorf("Failed to delete secret %s/%s", namespace, secrets.Items[i].Name) + } + } +} + +// CopyConfigMap copies a configmap from sourceNamespace to targetNamespace, applying the given +// labels. If a configmap with the same name already exists in the target with identical data and +// matching labels, it is a no-op. If the data matches but the labels differ, or the data differs, +// it returns ErrSecretCollision. +func CopyConfigMap(ctx context.Context, client corev1client.CoreV1Interface, cmName, sourceNamespace, targetNamespace string, labels map[string]string, log logrus.FieldLogger) error { + srcCM, err := client.ConfigMaps(sourceNamespace).Get(ctx, cmName, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "error getting configmap %s/%s", sourceNamespace, cmName) + } + + newCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: targetNamespace, + Labels: labels, + }, + Data: srcCM.Data, + BinaryData: srcCM.BinaryData, + } + + _, err = client.ConfigMaps(targetNamespace).Create(ctx, newCM, metav1.CreateOptions{}) + if err == nil { + log.Infof("Copied configmap %s from %s to %s", cmName, sourceNamespace, targetNamespace) + return nil + } + + if !apierrors.IsAlreadyExists(err) { + return errors.Wrapf(err, "error creating configmap %s in %s", cmName, targetNamespace) + } + + existing, err := client.ConfigMaps(targetNamespace).Get(ctx, cmName, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "error getting existing configmap %s/%s", targetNamespace, cmName) + } + + if reflect.DeepEqual(existing.Data, srcCM.Data) && + reflect.DeepEqual(existing.BinaryData, srcCM.BinaryData) && + labelsMatch(existing.Labels, labels) { + log.Infof("ConfigMap %s already exists in %s with same data and labels, skipping copy", cmName, targetNamespace) + return nil + } + + log.Infof("ConfigMap %s already exists in %s owned by a different owner, collision detected", cmName, targetNamespace) + return ErrSecretCollision +} + +// DeleteConfigMapsWithLabel deletes all configmaps in a namespace matching a label key=value pair. +// Uses UID preconditions to avoid deleting a recreated object with the same name. +func DeleteConfigMapsWithLabel(ctx context.Context, client corev1client.CoreV1Interface, namespace, labelKey, labelValue string, log logrus.FieldLogger) { + cms, err := client.ConfigMaps(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: labelKey + "=" + labelValue, + }) + if err != nil { + log.WithError(err).Errorf("Failed to list configmaps with label %s=%s in %s", labelKey, labelValue, namespace) + return + } + + for i := range cms.Items { + uid := cms.Items[i].UID + err := client.ConfigMaps(namespace).Delete(ctx, cms.Items[i].Name, metav1.DeleteOptions{ + Preconditions: &metav1.Preconditions{UID: &uid}, + }) + if err != nil && !apierrors.IsNotFound(err) { + log.WithError(err).Errorf("Failed to delete configmap %s/%s", namespace, cms.Items[i].Name) + } + } +} diff --git a/pkg/util/kube/secrets_copy_test.go b/pkg/util/kube/secrets_copy_test.go new file mode 100644 index 000000000..ea294eb4e --- /dev/null +++ b/pkg/util/kube/secrets_copy_test.go @@ -0,0 +1,362 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kube + +import ( + "context" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1api "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" +) + +const testCopyLabel = "velero.io/backup-pvc-secret" + +func TestCopySecret(t *testing.T) { + log := logrus.New() + + tests := []struct { + name string + secretName string + sourceNS string + targetNS string + ownerName string + objects []k8sruntime.Object + expectErr bool + errContains string + }{ + { + name: "successfully copies secret to target namespace", + secretName: "ceph-csi-kms-token", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{ + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-token", Namespace: "app-ns"}, + Data: map[string][]byte{"token": []byte("vault-token-a")}, + Type: corev1api.SecretTypeOpaque, + }, + }, + }, + { + name: "returns error when source secret does not exist", + secretName: "missing-secret", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{}, + expectErr: true, + errContains: "error getting secret", + }, + { + name: "no-op when target already has secret with same data and same owner", + secretName: "ceph-csi-kms-token", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{ + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-token", Namespace: "app-ns"}, + Data: map[string][]byte{"token": []byte("same-token")}, + Type: corev1api.SecretTypeOpaque, + }, + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ceph-csi-kms-token", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-123"}, + }, + Data: map[string][]byte{"token": []byte("same-token")}, + Type: corev1api.SecretTypeOpaque, + }, + }, + }, + { + name: "returns collision when same data but different owner", + secretName: "ceph-csi-kms-token", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-456", + objects: []k8sruntime.Object{ + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-token", Namespace: "app-ns"}, + Data: map[string][]byte{"token": []byte("same-token")}, + Type: corev1api.SecretTypeOpaque, + }, + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ceph-csi-kms-token", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-123"}, + }, + Data: map[string][]byte{"token": []byte("same-token")}, + Type: corev1api.SecretTypeOpaque, + }, + }, + expectErr: true, + errContains: "collision", + }, + { + name: "returns collision error when target has secret with different data", + secretName: "ceph-csi-kms-token", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{ + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-token", Namespace: "app-ns"}, + Data: map[string][]byte{"token": []byte("token-a")}, + Type: corev1api.SecretTypeOpaque, + }, + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-token", Namespace: "velero"}, + Data: map[string][]byte{"token": []byte("token-b")}, + Type: corev1api.SecretTypeOpaque, + }, + }, + expectErr: true, + errContains: "secret collision", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeClient := fake.NewSimpleClientset(tt.objects...) + + err := CopySecret(context.Background(), fakeClient.CoreV1(), + tt.secretName, tt.sourceNS, tt.targetNS, + map[string]string{testCopyLabel: tt.ownerName}, log) + + if tt.expectErr { + require.Error(t, err) + if tt.errContains != "" { + assert.Contains(t, err.Error(), tt.errContains) + } + return + } + + require.NoError(t, err) + + copied, getErr := fakeClient.CoreV1().Secrets(tt.targetNS).Get( + context.Background(), tt.secretName, metav1.GetOptions{}) + require.NoError(t, getErr) + assert.NotNil(t, copied) + }) + } +} + +func TestDeleteSecretsWithLabel(t *testing.T) { + log := logrus.New() + + fakeClient := fake.NewSimpleClientset( + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-1", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-123"}, + }, + }, + &corev1api.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-2", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-456"}, + }, + }, + ) + + DeleteSecretsWithLabel(context.Background(), fakeClient.CoreV1(), "velero", + testCopyLabel, "du-123", log) + + _, err := fakeClient.CoreV1().Secrets("velero").Get( + context.Background(), "secret-1", metav1.GetOptions{}) + require.Error(t, err, "secret-1 should be deleted") + + _, err = fakeClient.CoreV1().Secrets("velero").Get( + context.Background(), "secret-2", metav1.GetOptions{}) + assert.NoError(t, err, "secret-2 should still exist") +} + +func TestCopyConfigMap(t *testing.T) { + log := logrus.New() + + tests := []struct { + name string + cmName string + sourceNS string + targetNS string + ownerName string + objects []k8sruntime.Object + expectErr bool + errContains string + }{ + { + name: "successfully copies configmap to target namespace", + cmName: "ceph-csi-kms-config", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{ + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-config", Namespace: "app-ns"}, + Data: map[string]string{"vaultAddress": "https://vault.example.com"}, + }, + }, + }, + { + name: "returns error when source configmap does not exist", + cmName: "missing-cm", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{}, + expectErr: true, + errContains: "error getting configmap", + }, + { + name: "no-op when target already has configmap with same data and same owner", + cmName: "ceph-csi-kms-config", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{ + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-config", Namespace: "app-ns"}, + Data: map[string]string{"vaultAddress": "https://vault.example.com"}, + }, + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ceph-csi-kms-config", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-123"}, + }, + Data: map[string]string{"vaultAddress": "https://vault.example.com"}, + }, + }, + }, + { + name: "returns collision when same data but different owner", + cmName: "ceph-csi-kms-config", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-456", + objects: []k8sruntime.Object{ + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-config", Namespace: "app-ns"}, + Data: map[string]string{"vaultAddress": "https://vault.example.com"}, + }, + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ceph-csi-kms-config", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-123"}, + }, + Data: map[string]string{"vaultAddress": "https://vault.example.com"}, + }, + }, + expectErr: true, + errContains: "collision", + }, + { + name: "copies configmap with BinaryData", + cmName: "ceph-csi-kms-config", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{ + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-config", Namespace: "app-ns"}, + BinaryData: map[string][]byte{"ca.crt": []byte("binary-ca-bundle")}, + }, + }, + }, + { + name: "returns collision error when target has configmap with different data", + cmName: "ceph-csi-kms-config", + sourceNS: "app-ns", + targetNS: "velero", + ownerName: "du-123", + objects: []k8sruntime.Object{ + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-config", Namespace: "app-ns"}, + Data: map[string]string{"vaultAddress": "https://vault-a.example.com"}, + }, + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "ceph-csi-kms-config", Namespace: "velero"}, + Data: map[string]string{"vaultAddress": "https://vault-b.example.com"}, + }, + }, + expectErr: true, + errContains: "secret collision", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeClient := fake.NewSimpleClientset(tt.objects...) + + err := CopyConfigMap(context.Background(), fakeClient.CoreV1(), + tt.cmName, tt.sourceNS, tt.targetNS, + map[string]string{testCopyLabel: tt.ownerName}, log) + + if tt.expectErr { + require.Error(t, err) + if tt.errContains != "" { + assert.Contains(t, err.Error(), tt.errContains) + } + return + } + + require.NoError(t, err) + + copied, getErr := fakeClient.CoreV1().ConfigMaps(tt.targetNS).Get( + context.Background(), tt.cmName, metav1.GetOptions{}) + require.NoError(t, getErr) + assert.NotNil(t, copied) + }) + } +} + +func TestDeleteConfigMapsWithLabel(t *testing.T) { + log := logrus.New() + + fakeClient := fake.NewSimpleClientset( + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cm-1", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-123"}, + }, + }, + &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cm-2", Namespace: "velero", + Labels: map[string]string{testCopyLabel: "du-456"}, + }, + }, + ) + + DeleteConfigMapsWithLabel(context.Background(), fakeClient.CoreV1(), "velero", + testCopyLabel, "du-123", log) + + _, err := fakeClient.CoreV1().ConfigMaps("velero").Get( + context.Background(), "cm-1", metav1.GetOptions{}) + require.Error(t, err, "cm-1 should be deleted") + + _, err = fakeClient.CoreV1().ConfigMaps("velero").Get( + context.Background(), "cm-2", metav1.GetOptions{}) + assert.NoError(t, err, "cm-2 should still exist") +} diff --git a/pkg/util/kube/utils.go b/pkg/util/kube/utils.go index d76dad4a3..c3d0b2046 100644 --- a/pkg/util/kube/utils.go +++ b/pkg/util/kube/utils.go @@ -103,7 +103,10 @@ func EnsureNamespaceExistsAndIsReady(namespace *corev1api.Namespace, client core return true, err } if clusterNS != nil && (clusterNS.GetDeletionTimestamp() != nil || clusterNS.Status.Phase == corev1api.NamespaceTerminating) { - if resourceDeletionStatusTracker.Contains(clusterNS.Kind, clusterNS.Name, clusterNS.Name) { + // Use namespace.Kind (not clusterNS.Kind) so this key matches the one Add() + // writes below: client.Get() strips TypeMeta (Kind=""), but getNamespace() + // sets Kind="Namespace". Mismatched keys made Contains never match. + if resourceDeletionStatusTracker.Contains(namespace.Kind, namespace.Name, namespace.Name) { namespaceAlreadyInDeletionTracker = true return true, errors.Errorf("namespace %s is already present in the polling set, skipping execution", namespace.Name) } diff --git a/pkg/util/kube/utils_test.go b/pkg/util/kube/utils_test.go index 23db12a41..cc53b31b5 100644 --- a/pkg/util/kube/utils_test.go +++ b/pkg/util/kube/utils_test.go @@ -154,6 +154,39 @@ func TestEnsureNamespaceExistsAndIsReady(t *testing.T) { } } +// TestEnsureNamespaceExistsAndIsReadyTerminatingTrackerKindMismatch verifies the +// tracker skip-path fires when Add and Contains see different Kind values, as they +// do in production: getNamespace() sets Kind="Namespace" but client.Get() strips it. +func TestEnsureNamespaceExistsAndIsReadyTerminatingTrackerKindMismatch(t *testing.T) { + // Passed-in namespace mirrors getNamespace(): Kind is set. + namespace := &corev1api.Namespace{ + TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + } + + // clusterNS mirrors client.Get(): Kind stripped, phase Terminating. + clusterNS := &corev1api.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Status: corev1api.NamespaceStatus{Phase: corev1api.NamespaceTerminating}, + } + + nsClient := &velerotest.FakeNamespaceClient{} + defer nsClient.AssertExpectations(t) + nsClient.On("Get", "test", metav1.GetOptions{}).Return(clusterNS, nil) + + // Seed the tracker as production Add() does. + tracker := NewResourceDeletionStatusTracker() + tracker.Add(namespace.Kind, namespace.Name, namespace.Name) + + result, nsCreated, err := EnsureNamespaceExistsAndIsReady(namespace, nsClient, time.Millisecond, tracker) + + assert.False(t, result) + assert.False(t, nsCreated) + // Skip-path must fire, not the full terminating-resource-timeout wait. + require.ErrorContains(t, err, "skipping polling for terminating namespace") + assert.NotContains(t, err.Error(), "timed out waiting for terminating namespace") +} + // TestGetVolumeDirectorySuccess tests that the GetVolumeDirectory function // returns a volume's name or a volume's name plus '/mount' when a PVC is present. func TestGetVolumeDirectorySuccess(t *testing.T) { diff --git a/pkg/util/velero/restore/util.go b/pkg/util/velero/restore/util.go index e0812884b..96a368e50 100644 --- a/pkg/util/velero/restore/util.go +++ b/pkg/util/velero/restore/util.go @@ -5,8 +5,14 @@ import ( ) func IsResourcePolicyValid(resourcePolicy string) bool { - if resourcePolicy == string(api.PolicyTypeNone) || resourcePolicy == string(api.PolicyTypeUpdate) { - return true - } - return false + return resourcePolicy == "" || + resourcePolicy == string(api.ResourcePolicyTypeNone) || + resourcePolicy == string(api.ResourcePolicyTypeUpdate) +} + +func IsVolumeDataPolicyValid(volumeDataPolicy string) bool { + return volumeDataPolicy == "" || + volumeDataPolicy == string(api.VolumeDataPolicyTypeNone) || + volumeDataPolicy == string(api.VolumeDataPolicyTypeFull) || + volumeDataPolicy == string(api.VolumeDataPolicyTypeIncremental) } diff --git a/pkg/util/velero/restore/util_test.go b/pkg/util/velero/restore/util_test.go index be72ff8ba..bcd447d4b 100644 --- a/pkg/util/velero/restore/util_test.go +++ b/pkg/util/velero/restore/util_test.go @@ -9,7 +9,16 @@ import ( ) func TestIsResourcePolicyValid(t *testing.T) { - require.True(t, IsResourcePolicyValid(string(velerov1api.PolicyTypeNone))) - require.True(t, IsResourcePolicyValid(string(velerov1api.PolicyTypeUpdate))) - require.False(t, IsResourcePolicyValid("")) + require.True(t, IsResourcePolicyValid(string(velerov1api.ResourcePolicyTypeNone))) + require.True(t, IsResourcePolicyValid(string(velerov1api.ResourcePolicyTypeUpdate))) + require.True(t, IsResourcePolicyValid("")) + require.False(t, IsResourcePolicyValid("invalid")) +} + +func TestIsVolumeDataPolicyValid(t *testing.T) { + require.True(t, IsVolumeDataPolicyValid(string(velerov1api.VolumeDataPolicyTypeNone))) + require.True(t, IsVolumeDataPolicyValid(string(velerov1api.VolumeDataPolicyTypeFull))) + require.True(t, IsVolumeDataPolicyValid(string(velerov1api.VolumeDataPolicyTypeIncremental))) + require.True(t, IsVolumeDataPolicyValid("")) + require.False(t, IsVolumeDataPolicyValid("invalid")) } diff --git a/pkg/util/volumehelper/volume_policy_helper.go b/pkg/util/volumehelper/volume_policy_helper.go index 95f104994..a148b0432 100644 --- a/pkg/util/volumehelper/volume_policy_helper.go +++ b/pkg/util/volumehelper/volume_policy_helper.go @@ -27,4 +27,6 @@ type VolumeHelper interface { ShouldPerformFSBackup(volume corev1api.Volume, pod corev1api.Pod) (bool, error) ShouldPerformCustomAction(obj runtime.Unstructured, groupResource schema.GroupResource, matchParams map[string]any) (bool, error) GetActionParameters(obj runtime.Unstructured, groupResource schema.GroupResource) (bool, string, map[string]any, error) + GetSnapshotClass(obj runtime.Unstructured, groupResource schema.GroupResource) (string, error) + GetDataMoverFromActionParameters(obj runtime.Unstructured, groupResource schema.GroupResource) string } diff --git a/site/config.yaml b/site/config.yaml index 6eddc1e14..9edc08541 100644 --- a/site/config.yaml +++ b/site/config.yaml @@ -8,7 +8,7 @@ frontmatter: params: author: Velero Authors logo: Velero.svg - cncf_logo: cncf-white.svg + cncf_logo: cncf-color.svg hero: backgroundColor: med-blue versioning: true @@ -63,6 +63,9 @@ params: - title: Twitter fa_icon: fab fa-twitter url: https://twitter.com/projectvelero + - title: LinkedIn + fa_icon: fab fa-linkedin + url: https://www.linkedin.com/company/project-velero - title: Slack fa_icon: fab fa-slack url: https://kubernetes.slack.com/messages/velero diff --git a/site/content/_index.md b/site/content/_index.md index 5d34a41a3..7d27dc709 100644 --- a/site/content/_index.md +++ b/site/content/_index.md @@ -10,7 +10,7 @@ hero: content: Velero is an open source tool to safely backup and restore, perform disaster recovery, and migrate Kubernetes cluster resources and persistent volumes. cta_link1: text: Latest Release Information - url: /blog/Velero-1.11/ + url: https://github.com/velero-io/velero/releases/latest cta_link2: text: Download Velero url: https://github.com/velero-io/velero/releases/latest @@ -32,7 +32,7 @@ secondary_ctas: url: /blog/Velero-is-an-Open-Source-Tool-to-Back-up-and-Migrate-Kubernetes-Clusters/ # Velero.io word list : ignore content: Learn about Velero and how to protect your Kubernetes resources and volumes. cta2: - title: How Do You Use Velero? - url: https://github.com/velero-io/velero/issues/1327 - content: See how Velero is helping others and tell the world how you use Velero. + title: Join the Velero Community + url: /community/ + content: Connect with other Velero users on Slack, attend community meetings, and contribute to the project. --- \ No newline at end of file diff --git a/site/content/community/_index.md b/site/content/community/_index.md index 41755f9db..e8b2ca2af 100644 --- a/site/content/community/_index.md +++ b/site/content/community/_index.md @@ -9,14 +9,16 @@ If you’re a newcomer, check out the “[Good first issue](https://github.com/v If you are ready to jump in and test, add code, or help with documentation, follow the instructions on our [Start contributing](https://velero.io/docs/main/start-contributing/) documentation for guidance on how to setup Velero for development. -You can follow the work we do, see our milestones, and our backlog on our [GitHub project boards](https://github.com/velero-io/velero/projects). +You can follow the work we do via our [GitHub milestones](https://github.com/velero-io/velero/milestones) and the project [Roadmap](https://github.com/velero-io/velero/wiki/Roadmap). * Follow us on Twitter at [@projectvelero](https://twitter.com/projectvelero) +* Follow us on LinkedIn at [Project Velero](https://www.linkedin.com/company/project-velero) * Join our Kubernetes Slack channel and talk to over 800 other community members: [#velero-users](https://kubernetes.slack.com/messages/velero-users) * Join the Velero community meetings Bi-weekly community meeting alternating every week between Beijing Friendly timezone and EST/Europe Friendly Timezone - * Beijing/US friendly - we start at 8am Beijing Time(bound to CST) / 8pm EDT(7pm EST) / 5pm PDT(4pm PST) / 2am CEST(1am CET) - [Convert to your time zone](https://dateful.com/convert/beijing-china?t=8am) - [Zoom Link](https://broadcom.zoom.us/j/93945566592?pwd=rovF20vuI73kR6v67QBMpQuJOtM6sr.1&jst=2) - * US/Europe friendly - we start at 10am ET(bound to ET) / 7am PT / 3pm CET / 10pm(11pm) CST - [Convert to your time zone](https://dateful.com/convert/est-edt-eastern-time?t=10) - [Google meet link](https://meet.google.com/dyr-djtj-sko) + * Beijing/US friendly - we start at 8am Beijing Time(bound to CST) / 8pm EDT(7pm EST) / 5pm PDT(4pm PST) / 2am CEST(1am CET) - [Convert to your time zone](https://dateful.com/convert/beijing-china?t=8am) - [Zoom Link](https://zoom-lfx.platform.linuxfoundation.org/meeting/98821524848?password=579eadc1-f4aa-45aa-93c6-f7ea69d73b1a) + * US/Europe friendly - we start at 10am ET(bound to ET) / 7am PT / 3pm CET / 10pm(11pm) CST - [Convert to your time zone](https://dateful.com/convert/est-edt-eastern-time?t=10) - [Zoom Link](https://zoom-lfx.platform.linuxfoundation.org/meeting/95078224949?password=5f97cd2a-b140-4ede-add8-26a0816a8606) +* [Project meeting calendar](https://zoom-lfx.platform.linuxfoundation.org/meetings/velero?view=week) ([subscribe via iCal](https://webcal.prod.itx.linuxfoundation.org/lfx/lfpdCDzBbgNRCLpey8)) * Read and comment on the [meeting notes](https://hackmd.io/fCDVjqGuTG23CoOWQpoEVg) * See previous community meetings on our [YouTube Channel](https://www.youtube.com/playlist?list=PL7bmigfV0EqQRysvqvqOtRNk4L5S7uqwM) * Have a question to discuss in the community meeting? Please add it to our [Q&A Discussion board](https://github.com/velero-io/velero/discussions/categories/community-support-q-a) diff --git a/site/content/docs/main/api-types/schedule.md b/site/content/docs/main/api-types/schedule.md index ef3df4324..ef2d14b07 100644 --- a/site/content/docs/main/api-types/schedule.md +++ b/site/content/docs/main/api-types/schedule.md @@ -155,11 +155,13 @@ spec: uploaderConfig: # ParallelFilesUpload is the number of files parallel uploads to perform when using the uploader. parallelFilesUpload: 10 - # The labels you want on backup objects, created from this schedule (instead of copying the labels you have on schedule object itself). - # When this field is set, the labels from the Schedule resource are not copied to the Backup resource. + # The labels/annotations you want on backup objects, created from this schedule (instead of copying the labels/annotations you have on schedule object itself). + # When this field is set, the labels/annotations from the Schedule resource are not copied to the Backup resource. metadata: labels: labelname: somelabelvalue + annotations: + annotationname: someannotationvalue # Actions to perform at different times during a backup. The only hook supported is # executing a command in a container in a pod using the pod exec API. Optional. hooks: diff --git a/site/content/docs/main/backup-restore-windows.md b/site/content/docs/main/backup-restore-windows.md index 9d700f472..b0b8ef1ae 100644 --- a/site/content/docs/main/backup-restore-windows.md +++ b/site/content/docs/main/backup-restore-windows.md @@ -17,38 +17,38 @@ For volume backups, CSI and CSI snapshot should be supported by the storage. As mentioned in [Image building][2], a hybrid image is provided for all platforms, so you don't need to set different images for linux and Windows clusters, you can always use the all-in-one image, e.g., `velero/velero:v1.16.0` or `velero/velero:main`. In order to backup/restore volumes for stateful workloads, Velero node-agent needs to run in the Windows nodes. Velero provides a dedicated daemonset for Windows nodes, called `node-agent-windows`. -Therefore, in a typical cluster with linux and Windows nodes, there are two daemonsets for Velero node-agent, the existing `node-agent` deamonset for linux nodes, and the `node-agent-windows` daemonset for Windows nodes. -If you want to install `node-agent` deamonset, specify `--use-node-agent` parameter in `velero install` command; and if you want to install `node-agent-windows` daemonset, specify `--use-node-agent-windows` parameter. +Therefore, in a typical cluster with linux and Windows nodes, there are two daemonsets for Velero node-agent, the existing `node-agent` daemonset for linux nodes, and the `node-agent-windows` daemonset for Windows nodes. +If you want to install `node-agent` daemonset, specify `--use-node-agent` parameter in `velero install` command; and if you want to install `node-agent-windows` daemonset, specify `--use-node-agent-windows` parameter. ## Resource backup restore -Resource backup/restore for Windows workloads are done by Velero server as same as linux workloads. +Resource backup/restore for Windows workloads is done by the Velero server the same as for linux workloads. -Since Velero server is running in linux nodes only, all the existing plugins, i.e., BIA, RIA, BackupStore plugins, could be started by Velero in a cluster with Windows nodes. However, whether or how the plugins are functional to Windows workloads are decided by the plugins themselves. -It is recommended that plugin providers do a well round test with Velero in Windows cluster environments, and: +Since Velero server is running in linux nodes only, all the existing plugins, i.e., BIA, RIA, BackupStore plugins, could be started by Velero in a cluster with Windows nodes. However, whether or how the plugins are functional for Windows workloads is decided by the plugins themselves. +It is recommended that plugin providers do a thorough test with Velero in Windows cluster environments, and: - If they need to support Windows workloads, make the necessary modification to ensure their plugins work well with Windows workloads - If they don't want to support Windows workloads, or part of the Windows workloads, they need to ensure the plugins won't cause any failure or crash when they process the undesired Windows workload items ## Volume backup restore -Below are the status of supportive of Windows workload volumes for different backup methods: -- CSI snapshot data movement: block volumes (i.e., vSphere CNS Block Volume, Azure Disk, AWS EBS, GCP Persistent Disk, etc.) are full supported; file volumes (i.e., vSphere CNS File Volume, Azure File, AWS EFS, GCP Filestore, etc.) are not tested or officially supported. This is the same with linux workloads -- CSI snapshot backup: block volumes (i.e., vSphere CNS Block Volume, Azure Disk, AWS EBS, GCP Persistent Disk, etc.) are full supported; file volumes (i.e., vSphere CNS File Volume, Azure File, AWS EFS, GCP Filestore, etc.) are not tested or officially supported. This is the same with linux workloads -- native snapshot backup: supported as same as linux workloads +Below is the support status for Windows workload volumes for different backup methods: +- CSI snapshot data movement: block volumes (i.e., vSphere CNS Block Volume, Azure Disk, AWS EBS, GCP Persistent Disk, etc.) are fully supported; file volumes (i.e., vSphere CNS File Volume, Azure File, AWS EFS, GCP Filestore, etc.) are not tested or officially supported. This is the same for linux workloads +- CSI snapshot backup: block volumes (i.e., vSphere CNS Block Volume, Azure Disk, AWS EBS, GCP Persistent Disk, etc.) are fully supported; file volumes (i.e., vSphere CNS File Volume, Azure File, AWS EFS, GCP Filestore, etc.) are not tested or officially supported. This is the same for linux workloads +- native snapshot backup: supported the same as for linux workloads - file system backup: at present, NOT supported -For volume backups/restores conducted through Velero plugins, the supportive status is decided by the plugin themselves. +For volume backups/restores conducted through Velero plugins, the support status is decided by the plugins themselves. ### CSI snapshot data movement During backup, Velero automatically identifies the OS type of the workload and schedules data mover pods to the right nodes. Specifically, for a linux workload, linux nodes in the cluster will be used; for a Windows workload, Windows nodes in the cluster will be used. You could view the OS type that a data mover pod is running with from the DataUpload status's `nodeOS` field. -Velero takes several measures to deduce the OS type for volumes of workloads, from PVCs, VolumeAttach CRs, nodes and storage classes. If Velero fails to deduce the OS type, it fallbacks to linux, then the data mover pods will be scheduled to linux nodes. As a result, the data mover pods may not be able to start and the corresponding DataUploads will be cancelled because of timeout, so the backup will be partially failed. +Velero takes several measures to deduce the OS type for volumes of workloads, from PVCs, VolumeAttach CRs, nodes and storage classes. If Velero fails to deduce the OS type, it falls back to linux, then the data mover pods will be scheduled to linux nodes. As a result, the data mover pods may not be able to start and the corresponding DataUploads will be cancelled because of timeout, so the backup will be partially failed. Therefore, it is highly recommended you provide a dedicated storage class for Windows workloads volumes, and set `csi.storage.k8s.io/fstype` correctly. E.g., for linux workload volumes, set `csi.storage.k8s.io/fstype=ext4`; for Windows workload volumes set `csi.storage.k8s.io/fstype=ntfs`. Specifically, if you have X number of storage classes for linux workloads, you need to create another X number of storage classes for Windows workloads. -This is helpful for Velero to deduce the right OS type successfully all the time, especially when you are backing up below kind of volumes belonging to a Windows workload: +This is helpful for Velero to deduce the right OS type successfully all the time, especially when you are backing up the following kinds of volumes belonging to a Windows workload: - The PVC is with Immediate mode - There is no pod mounting the PVC at the time of backup diff --git a/site/content/docs/main/contributions/minio.md b/site/content/docs/main/contributions/minio.md index 41d0e997f..125f7e191 100644 --- a/site/content/docs/main/contributions/minio.md +++ b/site/content/docs/main/contributions/minio.md @@ -74,7 +74,7 @@ These instructions start the Velero server and a Minio instance that is accessib ``` velero install \ --provider aws \ - --plugins velero/velero-plugin-for-aws:v1.2.1 \ + --plugins velero/velero-plugin-for-aws:v1.14.0 \ --bucket velero \ --secret-file ./credentials-velero \ --use-volume-snapshots=false \ diff --git a/site/content/docs/main/csi-snapshot-data-movement.md b/site/content/docs/main/csi-snapshot-data-movement.md index 154abb198..9c9bc6184 100644 --- a/site/content/docs/main/csi-snapshot-data-movement.md +++ b/site/content/docs/main/csi-snapshot-data-movement.md @@ -67,7 +67,7 @@ On source cluster, Velero needs to manipulate CSI snapshots through the CSI volu To integrate Velero with the CSI volume snapshot APIs, you must enable the `EnableCSI` feature flag. -From release-1.14, the `github.com/vmware-tanzu/velero-plugin-for-csi` repository, which is the Velero CSI plugin, is merged into the `github.com/velero-io/velero` repository. +From release-1.14, the `github.com/velero-io/velero-plugin-for-csi` repository, which is the Velero CSI plugin, is merged into the `github.com/velero-io/velero` repository. The reasons to merge the CSI plugin are: * The VolumeSnapshot data mover depends on the CSI plugin, it's reasonabe to integrate them. * This change reduces the Velero deploying complexity. @@ -170,6 +170,7 @@ kubectl -n velero get datadownloads -l velero.io/restore-name=YOUR_RESTORE_NAME that anyone who has access to your backup storage can decrypt your backup data**. Make sure that you limit access to the backup storage appropriately. - [Velero built-in data mover] Even though the backup data could be incrementally preserved, for a single file data, Velero built-in data mover leverages on deduplication to find the difference to be saved. This means that large files (such as ones storing a database) will take a long time to scan for data deduplication, even if the actual difference is small. +- [Velero built-in data mover] On volumes where the underlying filesystem enforces mount-constant identity (Azure Files SMB/CIFS, Azure Blob via blobfuse, GCP Cloud Storage FUSE, and similar), data download's `chown`/`chmod` can report success while changing nothing, silently losing file ownership (and on FUSE mounts, permission bits) with no error surfaced anywhere. See [File Ownership and Permission Preservation](file-system-backup.md#file-ownership-and-permission-preservation) for details and remediation. ## Troubleshooting diff --git a/site/content/docs/main/csi.md b/site/content/docs/main/csi.md index fddc5f258..68d2c5f67 100644 --- a/site/content/docs/main/csi.md +++ b/site/content/docs/main/csi.md @@ -8,7 +8,7 @@ Integrating Container Storage Interface (CSI) snapshot support into Velero enabl By supporting CSI snapshot APIs, Velero can support any volume provider that has a CSI driver, without requiring a Velero-specific plugin to be available. This page gives an overview of how to add support for CSI snapshots to Velero. ## Notice -From release-1.14, the `github.com/vmware-tanzu/velero-plugin-for-csi` repository, which is the Velero CSI plugin, is merged into the `github.com/velero-io/velero` repository. +From release-1.14, the `github.com/velero-io/velero-plugin-for-csi` repository, which is the Velero CSI plugin, is merged into the `github.com/velero-io/velero` repository. The reasons to merge the CSI plugin are: * The VolumeSnapshot data mover depends on the CSI plugin, it's reasonabe to integrate them. * This change reduces the Velero deploying complexity. @@ -86,8 +86,23 @@ This section documents some of the choices made during implementing the CSI snap ``` Note: Please ensure all your annotations are in lowercase. And follow the following format: `velero.io/csi-volumesnapshot-class_ = ` - 3. **Choosing VolumeSnapshotClass for a particular PVC:** - If you want to use a particular VolumeSnapshotClass for a particular PVC, you can add a annotation to the PVC to indicate which VolumeSnapshotClass to use. This overrides any annotation added to backup or schedule. For example, if you want to use the VolumeSnapshotClass `test-snapclass` for a particular PVC, you can create a PVC like this: + 3. **Choosing VolumeSnapshotClass via Volume Policy:** + If you want to use a particular VolumeSnapshotClass based on conditions like StorageClass, you can specify the `snapshotClass` parameter in a volume policy's `snapshot` action. This is useful when multiple storage arrays share the same CSI driver but require different VolumeSnapshotClasses. For example: + ```yaml + version: v1 + volumePolicies: + - conditions: + storageClass: + - nutanix-files + action: + type: snapshot + parameters: + snapshotClass: nutanix-files-snapclass + ``` + This overrides backup/schedule annotations and VolumeSnapshotClass labels, but is overridden by PVC-level annotations. See the [resource filtering documentation](resource-filtering.md) for more volume policy examples. + + 4. **Choosing VolumeSnapshotClass for a particular PVC:** + If you want to use a particular VolumeSnapshotClass for a particular PVC, you can add a annotation to the PVC to indicate which VolumeSnapshotClass to use. This overrides any other method of selecting a VolumeSnapshotClass. For example, if you want to use the VolumeSnapshotClass `test-snapclass` for a particular PVC, you can create a PVC like this: ```yaml apiVersion: v1 kind: PersistentVolumeClaim diff --git a/site/content/docs/main/custom-plugins.md b/site/content/docs/main/custom-plugins.md index b0881d579..106ebfd0f 100644 --- a/site/content/docs/main/custom-plugins.md +++ b/site/content/docs/main/custom-plugins.md @@ -65,6 +65,32 @@ order in which item action plugins are invoked. However, if a single binary impl they may be invoked in the order in which they are registered but it is best to not depend on this implementation. This is not guaranteed officially and the implementation can change at any time. +### Must-include additional items (Restore Item Actions) + +Restore Item Actions may return `AdditionalItems` that Velero restores as dependencies of the current item. +By default those additional items must still pass the restore's global resource and namespace include/exclude +filters (and `IncludeClusterResources=false` for cluster-scoped resources). + +To force-restore hard dependencies despite those filters, set the following annotation on the `UpdatedItem` +returned from `Execute()`: + +``` +restore.velero.io/must-include-additional-items: "true" +``` + +Behavior: +- Only the string value `"true"` enables the bypass. +- The annotation applies blanket to all `AdditionalItems` from that RIA invocation (not per-item). +- Velero strips the annotation before applying the item to the cluster. +- `SkipRestore: true` takes precedence: if set, the annotation is never inspected and `AdditionalItems` are not processed. +- Must-include only bypasses filters; the additional item must still exist in the backup tarball. +- When an additional item targets an excluded namespace, Velero may still create that target namespace so the item can be restored. +- Cluster-scoped additional items are restored even when `IncludeClusterResources=false`. +- Transitive force-include requires each RIA level to re-set the annotation on its own `UpdatedItem`. + +This mirrors the backup-side annotation `backup.velero.io/must-include-additional-items` used by Backup Item Actions. +Installing an RIA that sets this annotation is a trust decision: the plugin can restore resources outside the operator's restore filters. + ## Plugin Logging Velero provides a [logger][2] that can be used by plugins to log structured information to the main Velero server log or diff --git a/site/content/docs/main/customize-installation.md b/site/content/docs/main/customize-installation.md index e42d6a3f8..4b4a11f30 100644 --- a/site/content/docs/main/customize-installation.md +++ b/site/content/docs/main/customize-installation.md @@ -40,7 +40,7 @@ When installing with the `--use-node-agent` flag, the node-agent will mount the By default, `velero install` does not enable the use of File System Backup (FSB) to take backups of all pod volumes. You must apply an [annotation](file-system-backup.md/#using-opt-in-pod-volume-backup) to every pod which contains volumes for Velero to use FSB for the backup. -If you are planning to only use FSB for volume backups, you can run the `velero install` command with the `--default-volumes-to-fs-backup` flag. This will default all pod volumes backups to use FSB without having to apply annotations to pods. Note that when this flag is set during install, Velero will always try to use FSB to perform the backup, even want an individual backup to use volume snapshots, by setting the `--snapshot-volumes` flag in the `backup create` command. Alternatively, you can set the `--default-volumes-to-fs-backup` on an individual backup to to make sure Velero uses FSB for each volume being backed up. +If you are planning to only use FSB for volume backups, you can run the `velero install` command with the `--default-volumes-to-fs-backup` flag. This will default all pod volume backups to use FSB without having to apply annotations to pods. Note that when this flag is set during install, Velero will always try to use FSB to perform the backup. If you want an individual backup to use volume snapshots instead, set the `--snapshot-volumes` flag in the `backup create` command. Alternatively, you can set the `--default-volumes-to-fs-backup` flag on an individual backup to make sure Velero uses FSB for each volume being backed up. ## Update an existing installation @@ -219,7 +219,7 @@ kubectl patch daemonset node-agent -n velero --patch \ '{"spec":{"template":{"spec":{"containers":[{"name": "node-agent", "resources": {"limits":{"cpu": "1", "memory": "1024Mi"}, "requests": {"cpu": "1", "memory": "512Mi"}}}]}}}}' ``` -Additionally, you may want to update the the default File System Backup operation timeout (default 240 minutes) to allow larger backups more time to complete. You can adjust this timeout by adding the `- --fs-backup-timeout` argument to the Velero Deployment spec. +Additionally, you may want to update the default File System Backup operation timeout (default 240 minutes) to allow larger backups more time to complete. You can adjust this timeout by adding the `- --fs-backup-timeout` argument to the Velero Deployment spec. **NOTE:** Changes made to this timeout value will revert back to the default value if you re-run the Velero install command. @@ -348,6 +348,21 @@ By default, only one backup is processed in the `InProgress` phase at a time. Th Enabling parallel backups can provide a significant performance benefit for backups which contain a large number of Kubernetes resources or ones which contain a large number of smaller volumes. Backups dominated by large volumes will not see as much benefit, since the majority of time for those backups is spent waiting for the async phase to complete. A larger `concurrent-backups` configuration may require additional memory and CPU resources for the velero container. +## Limiting Resource Backup Data Cache Size +For Kubernetes resource data (non volume data), for some operations like Restores or Backup Deletions, etc., Velero uses local cache (in the root file system of the cluster node) to download and extract the data from the backup storage location, Velero sets a limit for the cache size. If the cache size exceeds the limit, the specific operation would fail. +By default Velero sets the limit as 16GB, if your backup data is large, you can change the Velero server parameter `max-backup-extraction-size`. Here is an example to set the limit to 32GB: + +```yaml +containers: + - name: velero + image: velero/velero:latest + command: + - /velero + args: + - server + - --max-backup-extraction-size=32768 +``` + ## Additional options Run `velero install --help` or see the [Helm chart documentation](https://vmware-tanzu.github.io/helm-charts/) for the full set of installation options. @@ -356,7 +371,7 @@ Run `velero install --help` or see the [Helm chart documentation](https://vmware ### Enabling shell autocompletion -**Velero CLI** provides autocompletion support for `Bash` and `Zsh`, which can save you a lot of typing. +**Velero CLI** provides autocompletion support for `Bash`, `Zsh`, and `Fish`, which can save you a lot of typing. In addition to command and flag names, the CLI dynamically completes resource names (backups, restores, schedules, etc.) by querying the cluster. Below are the procedures to set up autocompletion for `Bash` (including the difference between `Linux` and `macOS`) and `Zsh`. @@ -501,6 +516,7 @@ By far, `velero install` supports the following parameters to specify the extern * --backup-repository-configmap: [backup repository configuration document][15] * --node-agent-configmap: [node-agent concurrency configuration document][16], and there are some other documents specify other parts of node-agent-config. * --repo-maintenance-job-configmap: [repository maintenance configuration document][17] +* --default-resource-modifier-configmap: [default restore resource modifier document][18]. When set, the referenced ConfigMap's resource modifier rules apply automatically to all restores that don't specify a per-restore modifier. From v1.17, Velero adds verification for the ConfigMaps in CLI and server side, which means `velero install` CLI will fail and velero server and node-agent pod will exit if the specified ConfigMaps don't exist or are invalid. @@ -539,3 +555,4 @@ The new workflow is: [15]: backup-repository-configuration.md [16]: node-agent-concurrency.md [17]: repository-maintenance.md +[18]: restore-resource-modifiers.md#default-resource-modifiers diff --git a/site/content/docs/main/data-movement-backup-pvc-configuration.md b/site/content/docs/main/data-movement-backup-pvc-configuration.md index 9d48b0a5f..afcd7e50d 100644 --- a/site/content/docs/main/data-movement-backup-pvc-configuration.md +++ b/site/content/docs/main/data-movement-backup-pvc-configuration.md @@ -34,12 +34,28 @@ default the source PVC's storage class will be used. the SELinux point of view, this will be considered a "Super Privileged Container" which means that selinux enforcement will be disabled and volume relabeling will not occur. This field is ignored if `readOnly` is `false`. +- `readWriteOncePod`: This is a boolean value. If set to `true`, then `ReadWriteOncePod` will be the only value set to the backupPVC's access modes. On + SELinux-enabled clusters the kubelet applies the SELinux label to a `ReadWriteOncePod` volume at mount time (`-o context=`) instead of recursively + relabeling every file on the volume, which can take hours on volumes with a high file count. It requires a CSI driver that advertises SELinux mount + support (`CSIDriver.spec.seLinuxMount: true`) and a storage class that supports creating `ReadWriteOncePod` PVCs from a snapshot. This field is ignored + if `readOnly` is `true`. + The users can specify the ConfigMap name during velero installation by CLI: `velero install --node-agent-configmap=` - `annotations`: permits to set annotations on the backupPVC itself. typically useful for some CSI provider which cannot mount a VolumeSnapshot without a custom annotation. +- `secretNames`: a list of secret names to copy from the source PVC's namespace to the Velero namespace before the backupPVC is + created, and delete after the DataUpload completes. This is needed for CSI drivers that require namespace-scoped secrets to + provision the volume, for example ODF/ceph-csi encrypted volumes that fetch a KMS token secret (`ceph-csi-kms-token`) from the + PVC's namespace. Without this, the backupPVC created in the Velero namespace fails to provision because the secret only exists + in the source namespace. + +- `configMapNames`: a list of configmap names to copy from the source PVC's namespace to the Velero namespace before the backupPVC + is created, and delete after the DataUpload completes. This is needed for CSI drivers that require namespace-scoped configmaps to + provision the volume, for example a tenant-specific ceph-csi KMS connection override configmap (`ceph-csi-kms-config`). + A sample of `backupPVC` config as part of the ConfigMap would look like: ```json { @@ -60,11 +76,24 @@ A sample of `backupPVC` config as part of the ConfigMap would look like: "storage-class-4": { "readOnly": true, "spcNoRelabeling": true + }, + "ocs-storagecluster-ceph-rbd-encrypted": { + "secretNames": ["ceph-csi-kms-token"], + "configMapNames": ["ceph-csi-kms-config"] + }, + "storage-class-5": { + "readWriteOncePod": true } } } ``` +**Note on encrypted volumes:** the copied secrets/configmaps are labeled `velero.io/backup-pvc-secret=` and +deleted when the DataUpload completes (or on failure). If concurrent DataUploads from different namespaces need a secret with the same +name but different content in the Velero namespace, they conflict. For ceph-csi, +this can be avoided by configuring a unique +[`tenantTokenName` per tenant](https://github.com/ceph/ceph-csi/blob/devel/docs/design/proposals/encryption-with-vault-tokens.md#example-of-the-kms-configuration-file-for-vault-tokens). + **Note:** - Users should make sure that the storage class specified in `backupPVC` config should exist in the cluster and can be used by the `backupPVC`, otherwise the corresponding DataUpload CR will stay in `Accepted` phase until timeout (data movement prepare timeout value is 30m by default). @@ -73,6 +102,10 @@ A sample of `backupPVC` config as part of the ConfigMap would look like: timeout (data movement prepare timeout value is 30m by default). - In an SELinux-enabled cluster, any time users set `readOnly=true` they must also set `spcNoRelabeling=true`. There is no need to set `spcNoRelabeling=true` if the volume is not readOnly. +- `readWriteOncePod` and `readOnly` are mutually exclusive. If both are set to `true`, `readOnly` wins, `readWriteOncePod` is ignored and a warning is logged. +- `readWriteOncePod` is an alternative to `readOnly`+`spcNoRelabeling` for SELinux-enabled clusters whose storage does not support `ReadOnlyMany` +(for example Ceph RBD in Filesystem mode or LVM). Users must make sure the storage class used for `backupPVC` supports creating a `ReadWriteOncePod` PVC from +a snapshot, otherwise the corresponding DataUpload CR will stay in `Accepted` phase until timeout. - If any of the above problems occur, then the DataUpload CR is `canceled` after timeout, and the backupPod and backupPVC will be deleted, and the backup will be marked as `PartiallyFailed`. diff --git a/site/content/docs/main/data-movement-restore-pvc-configuration.md b/site/content/docs/main/data-movement-restore-pvc-configuration.md index 1cb8fa14d..1728d346e 100644 --- a/site/content/docs/main/data-movement-restore-pvc-configuration.md +++ b/site/content/docs/main/data-movement-restore-pvc-configuration.md @@ -12,6 +12,10 @@ Velero introduces a new section in the node agent configuration ConfigMap (the n - `ignoreDelayBinding`: If this flag is set, the data movement restore will ignore the delay binding requirements from `WaitForFirstConsumer` mode, create the restore pod and provision the volume associated to an arbitrary node. When multiple volume restores happen in parallel, the restore pods will be spread evenly to all the nodes. +- `secretNames`: a list of secret names to copy from the target (restore) namespace to the Velero namespace before the restorePVC is created, and delete after the DataDownload completes. This is needed for CSI drivers that require namespace-scoped secrets to provision the volume, for example ODF/ceph-csi encrypted volumes that fetch a KMS token secret (`ceph-csi-kms-token`) from the PVC's namespace. Without this, the restorePVC created in the Velero namespace fails to provision because the secret only exists in the target namespace. + +- `configMapNames`: a list of configmap names to copy from the target (restore) namespace to the Velero namespace before the restorePVC is created, and delete after the DataDownload completes. This is needed for CSI drivers that require namespace-scoped configmaps to provision the volume, for example a tenant-specific ceph-csi KMS connection override configmap (`ceph-csi-kms-config`). + The users can specify the ConfigMap name during velero installation by CLI: `velero install --node-agent-configmap=` @@ -20,11 +24,15 @@ A sample of `restorePVC` config as part of the ConfigMap would look like: ```json { "restorePVC": { - "ignoreDelayBinding": true + "ignoreDelayBinding": true, + "secretNames": ["ceph-csi-kms-token"], + "configMapNames": ["ceph-csi-kms-config"] } } ``` +**Note on encrypted volumes:** unlike `backupPVC` (which is keyed per source storage class), `restorePVC` is a single config that applies to all restore PVCs. The copied secrets/configmaps are labeled `velero.io/backup-pvc-secret=` and deleted when the DataDownload completes (or on failure). + **Note:** - If `ignoreDelayBinding` is set, the restored volume is provisioned in the storage areas associated to an arbitrary node, if the restored pod cannot be scheduled to that node, e.g., because of topology constraints, the data mover restore still completes, but the workload is not usable since the restored pod cannot mount the restored volume - At present, node selection is not supported for data mover restore, so the restored volume may be attached to any node in the cluster; once node selection is supported and enabled, the restored volume will be attached to one of the selected nodes only. In this way, node selection and `ignoreDelayBinding` can work together even though the environment is with topology constraints diff --git a/site/content/docs/main/file-system-backup.md b/site/content/docs/main/file-system-backup.md index 139b91438..907fc08e8 100644 --- a/site/content/docs/main/file-system-backup.md +++ b/site/content/docs/main/file-system-backup.md @@ -367,7 +367,98 @@ For this reason, FSB can only backup volumes that are mounted by a pod and not d (without running pods), some Velero users overcame this limitation running a staging pod (i.e. a busybox or alpine container with an infinite sleep) to mount these PVC/PV pairs prior taking a Velero backup. - Velero File System Backup expects volumes to be mounted under `/` (`hostPath` is configurable as mentioned in [Configure Node Agent DaemonSet spec](#configure-node-agent-daemonset-spec)). Some Kubernetes systems (i.e., [vCluster][11]) don't mount volumes under the `` sub-dir, Velero File System Backup is not working with them. -- File system restores of the same pod won't start until all the volumes of the pod get bound, even though some of the volumes have been bound and ready for restore. An a result, if a pod has multiple volumes, while only part of the volumes are restored by file system restore, these file system restores won't start until the other volumes are restored completely by other restore types (i.e., [CSI Snapshot Restore][12], [CSI Snapshot Data Movement][13]), the file system restores won't happen concurrently with those other types of restores. +- File system restores of the same pod won't start until all the volumes of the pod get bound, even though some of the volumes have been bound and ready for restore. An a result, if a pod has multiple volumes, while only part of the volumes are restored by file system restore, these file system restores won't start until the other volumes are restored completely by other restore types (i.e., [CSI Snapshot Restore][12], [CSI Snapshot Data Movement][13]), the file system restores won't happen concurrently with those other types of restores. +- On volumes where the underlying filesystem enforces mount-constant identity (Azure Files SMB/CIFS, Azure Blob via blobfuse, GCP Cloud Storage FUSE, and similar), FSB restore's `chown`/`chmod` can report success while changing nothing, silently losing file ownership (and on FUSE mounts, permission bits) with no error surfaced anywhere. See [File Ownership and Permission Preservation](#file-ownership-and-permission-preservation) below. + +## File Ownership and Permission Preservation + +[#file-ownership-and-permission-preservation](#file-ownership-and-permission-preservation) + +Some volume types enforce a **mount-constant identity**: file ownership and/or permission mode are determined +entirely by the mount configuration rather than being stored per-file on the underlying storage. On these +filesystems, when FSB restore runs `chown`/`chmod` as root, the system call **returns success while changing +nothing**: the restored files simply present whatever owner/mode the mount is configured to force. Because no +error is ever raised, this is a silent failure: the restore reports `Completed` with zero warnings, and nothing +in the node-agent or data mover pod logs indicates a problem. + +This is a distinct failure mode from cases where the storage backend actively rejects the ownership change +(for example, NFS server-side `root_squash`, which returns a real `EPERM`). That class of failure can, in +principle, be caught by inspecting the error path. The mount-constant-identity case cannot, because there is no +error to catch. + +**Affected volume types (verified or by design):** + +| Storage | Ownership storage | `chown` as root | `chmod` as root | +| ---------------------------------------------------------------- | ----------------------------------------------- | ------------------------------------------ | ------------------------------------------ | +| Azure Files SMB/CIFS, default or forced `uid=`/`gid=` mount options | Mount-constant | Silent no-op | Forced by `file_mode=`/`dir_mode=` | +| Azure Files SMB with `idsfromsid,modefromsid` mount options | Stored in NTFS security descriptors | Works, persists, survives remount | Works, persists | +| Azure Blob via blobfuse (`blobfuse2`) | Mount-constant | Silent no-op | Silent no-op | +| Azure Blob NFSv3 (Premium) | Real POSIX (server-side) | Works | Works | +| Azure Files NFS 4.1 (Premium) | Real POSIX (server-side) | Works | Works | +| Azure Files NFS 4.1 with `rootSquashType: RootSquash` | Real POSIX, but root is squashed | Real `EPERM` (see NFS ownership caveat below) | Works | +| GCP Cloud Storage FUSE (`gcsfuse.csi.storage.gke.io`) | Mount-time uid/gid/mode, not stored | Not supported - silent-loss class | Not supported - silent-loss class | +| AWS FSx for Windows (SMB, NTFS ACLs) | NTFS ACLs, don't map to POSIX ownership | Doesn't map | Doesn't map | +| AWS EFS Access Points with a `PosixUser` | Access Point overrides uid/gid for all operations | Neutralized server-side | N/A | + +Azure Disk (block storage) and plain Azure Files/EFS without the above configurations use real POSIX semantics +and are not affected. + +### Verified remediation for Azure Files SMB + +Add `idsfromsid,modefromsid` to the StorageClass `mountOptions`, and do **not** force `uid=`/`gid=`/`mode=` +alongside them. This stores real per-file ownership and mode in the share's NTFS security descriptors and gives +full fidelity across backup and restore. + +Caveats: + +- On a fresh share, the volume root receives a translated security descriptor on first mount (typically + `uid=0 gid= mode=1707`). Non-root workloads may need a one-time root init container to `chown`/`chmod` + the volume root before the main container starts; this operation itself works correctly on this mount. +- A restrictive owner/mode on the volume root can prevent Velero's FSB restore-wait init container from + accessing the volume if its identity doesn't match the workload's. If you hit a restore stuck at + `Init:0/1`, configure the restore helper's security context (`secCtxRunAsUser`, `secCtxRunAsGroup`, or `secCtx`) + to match your workload's UID/GID. See [Customize Restore Helper Container](#customize-restore-helper-container). + +As an alternative, Azure Files NFS 4.1 (Premium tier) or Azure Blob NFSv3 (Premium tier) also preserve ownership +and mode with full fidelity, **provided you avoid `rootSquashType: RootSquash`**. Root-squashed NFS mounts +reject root's `chown` with a real `EPERM`, which is a different (but related) failure. See the NFS ownership +note below. + +### No remediation exists for blobfuse or gcsfuse + +For Azure Blob via blobfuse and GCP Cloud Storage FUSE, there is currently no mount option or configuration +that preserves per-file ownership or mode. This is a limitation of the FUSE drivers themselves, not something +Velero or its restore path can work around. If your workload depends on stat-level ownership fidelity (for +example, databases like PostgreSQL or MySQL that refuse to start if the data directory's ownership doesn't +match the running user), avoid these volume types for that data. Use block storage, a real POSIX-backed +protocol (e.g. Azure Files NFS 4.1, Azure Blob NFSv3), or Azure Files SMB with `idsfromsid,modefromsid` instead. + +### Related: NFS root_squash ownership loss + +A related but mechanically distinct issue affects NFS mounts with server-side `root_squash` enabled: the +`chown` call receives a real `EPERM` from the server, but Velero's kopia integration currently sets +`IgnorePermissionErrors: true`, which silently discards that error. The end result looks the same to the user +(a `Completed` restore with lost ownership), but the underlying mechanism differs. Here an error genuinely +occurs, it is simply swallowed, whereas on mount-constant-identity filesystems no error is ever generated in +the first place. If you're troubleshooting ownership loss on NFS-backed volumes with root squashing enabled, +this is the more likely cause. + +### Diagnosing which case you're hitting + +Check the mount options inside the affected pod: + + `mount | grep -E 'cifs|fuse|nfs'` + +Look for `uid=`/`gid=` (CIFS) or a FUSE filesystem type. The typical symptom in all these cases is a restore +that reports `Completed` with no warnings, followed by an application failing immediately afterward with an +ownership-related error, for example: + + FATAL: data directory "/var/lib/postgresql/data/pgdata" has wrong ownership + HINT: The server must be started by the user that owns the data directory. + +This signature, a clean restore followed by an immediate ownership-related crash, is the indicator that +you're affected by one of the limitations described above rather than a genuine restore failure. + ## Customize Restore Helper Container diff --git a/site/content/docs/main/fine-grained-backup-filters.md b/site/content/docs/main/fine-grained-backup-filters.md new file mode 100644 index 000000000..d49cf6c93 --- /dev/null +++ b/site/content/docs/main/fine-grained-backup-filters.md @@ -0,0 +1,856 @@ +--- +title: "Fine-Grained Backup Filters" +layout: docs +--- + +This guide explains how to use Velero's **fine-grained backup filters**: per-namespace, per-kind rules with independent label selectors and resource name patterns. Configuration lives in the same **ResourcePolicy ConfigMap** you may already use for volume policies. + +For architecture and pipeline details, see the [design document](https://github.com/velero-io/velero/blob/main/design/backup-filter-enhancement/fine-grained-backup-filters-design.md). + +--- + +## Introduction + +Velero's global backup filters apply the same namespace list, resource types, and label selector to every namespace in a backup. That works for many clusters, but common scenarios need more control: + +- **Different namespaces, different strategies** — back up everything in a database namespace, but only Deployments and ConfigMaps in a frontend namespace. +- **Filter by resource name** — back up `app-config` and `app-secret` without also capturing `monitoring-config`. +- **Different labels per kind** — Deployments labeled `app=workload-1` and StatefulSets labeled `app=workload-2` in the same namespace. + +Fine-grained filters add two optional sections to the ResourcePolicy ConfigMap: + +| Section | Scope | Behavior | +|---------|-------|----------| +| `namespacedFilterPolicies` | Namespaces you match (exact name or glob) | **Exclusive allowlist** — only resource kinds listed in `resourceFilters` (or covered by a catch-all) are backed up from those namespaces | +| `clusterScopedFilterPolicy` | Cluster-scoped resources globally | **Refinement overlay** — listed kinds get per-kind label and name rules; unlisted cluster-scoped kinds still use global BackupSpec filters | + +**No new BackupSpec CRD fields** are required. Reference the policy from `Backup.spec.resourcePolicy` or `velero backup create --resource-policies-configmap`. + +**Backward compatible:** if you omit both new sections, backups behave exactly as they do today. + +--- + +## Prerequisites and wiring + +### What you need + +- Velero installed with backup filters support (see your Velero release notes). +- A ResourcePolicy ConfigMap in the Velero namespace (`velero` by default). +- Permission to create Backups (or Schedules) that reference the ConfigMap. + +### End-to-end pattern + +Every example below follows the same three steps: + +1. **Create or update** a ConfigMap with `data.policy` containing `version: v1` and your filter rules. +2. **Create a Backup** (or Schedule) that includes the target namespaces and references the ConfigMap. +3. **Verify** with `velero backup describe` and inspect backup contents or logs. + +### Minimal skeleton + +Use this once; later examples show only the `policy:` body. + +**ResourcePolicy ConfigMap:** + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-backup-filter-policy + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + - namespaces: + - my-namespace + resourceFilters: + - kinds: [ConfigMap] + labelSelector: + matchLabels: + app: my-app +``` + +**Backup:** + +```yaml +apiVersion: velero.io/v1 +kind: Backup +metadata: + name: my-backup + namespace: velero +spec: + includedNamespaces: + - my-namespace + resourcePolicy: + kind: configmap + name: my-backup-filter-policy + storageLocation: default +``` + +**CLI equivalent:** + +```bash +velero backup create my-backup \ + --include-namespaces my-namespace \ + --resource-policies-configmap my-backup-filter-policy +``` + +**Verify:** + +```bash +velero backup describe my-backup +velero backup describe my-backup -o json | jq '.namespacedFilterPolicies' +``` + +### Important: do not mix old-style BackupSpec resource filters + +When `namespacedFilterPolicies` or `clusterScopedFilterPolicy` is present in the ResourcePolicy, **do not** set these on the Backup: + +- `spec.includedResources` / `spec.excludedResources` +- `spec.includeClusterResources` + +Use `includeExcludePolicy` inside the ResourcePolicy ConfigMap for global resource-type include/exclude instead. Velero rejects backups that combine the new policy sections with old-style fields. + +Schedules follow the same rule: configure filters in the ResourcePolicy ConfigMap, not via deprecated resource filter fields on the Schedule template. + +--- + +## Examples + +Each example includes: **goal**, **policy YAML**, **backup notes**, **expected outcome**, and **how to verify**. + +--- + +### Example 0 — Baseline (no new filters) + +**Goal:** Confirm that namespaces without a `namespacedFilterPolicies` entry still use global BackupSpec filters. + +**Policy:** Omit `namespacedFilterPolicies` and `clusterScopedFilterPolicy` entirely (or use a ConfigMap with only `volumePolicies` / `includeExcludePolicy`). + +**Backup:** + +```yaml +spec: + includedNamespaces: + - ns-a + - ns-b + - production + # No resourcePolicy — global filters only +``` + +**Expected outcome:** All resources in included namespaces follow `includedNamespaces`, `labelSelector`, `includedResources`, and related global fields — same as before this feature. + +**Verify:** `velero backup describe` shows no namespace-scoped filter policies section. + +--- + +### Example 1 — Per-namespace kinds and labels + +**Goal:** In `ns-a`, back up only ConfigMaps, Secrets, Deployments, and Pods with `app=my-app`. In `ns-b`, use global filters (no policy entry for that namespace). + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [ConfigMap, Secret, Deployment, Pod] + labelSelector: + matchLabels: + app: my-app +``` + +**Backup:** + +```yaml +spec: + includedNamespaces: + - ns-a + - ns-b + resourcePolicy: + kind: configmap + name: per-namespace-resource-filter-policy # or your ConfigMap name +``` + +**Expected outcome:** + +- **ns-a:** Only listed kinds with label `app=my-app` (e.g. `app-config`, `app-secret`, `app-deployment`). Resources like `monitoring-config` (different labels) are excluded. +- **ns-b:** Everything allowed by global filters (no namespace policy match). + +**Verify:** `velero backup describe` lists resolved filters for `ns-a`. + +--- + +### Example 2 — Exact resource names + +**Goal:** Back up only two ConfigMaps by exact name, optionally requiring a label. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - target-namespace + resourceFilters: + - kinds: [ConfigMap] + names: [vm-1, vm-2] + labelSelector: + matchLabels: + resource-type: VirtualMachine +``` + +**Backup:** `includedNamespaces: [target-namespace]` plus `resourcePolicy` reference. + +**Expected outcome:** Only `vm-1` and `vm-2` ConfigMaps with `resource-type=VirtualMachine`. `vm-3` and other ConfigMaps are excluded. + +**Verify:** Backup archive contains exactly those two ConfigMaps in `target-namespace`. + +--- + +### Example 3 — Glob name patterns with exclusions + +**Goal:** Back up `app-*` ConfigMaps and Secrets in `production`, but exclude temporary and debug names. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [ConfigMap, Secret] + names: ["app-*"] + excludedNames: ["*-tmp-*", "*-debug-*", "*-tmp", "*-debug"] +``` + +**Expected outcome:** + +- **Included:** `app-config`, `app-cache-config`, `app-secret`, `app-db-secret` +- **Excluded:** `app-tmp-config`, `app-debug-config` (excluded by `excludedNames`), and `monitoring-tmp-secret` (excluded because it does not match the `names: ["app-*"]` allowlist) + +`excludedNames` takes precedence over `names` when both match. + +**Verify:** Inspect backup item list. + +--- + +### Example 4 — Per-kind label selectors + +**Goal:** Apply different label rules to different resource types in the same namespace. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - target-namespace + resourceFilters: + - kinds: [ConfigMap] + orLabelSelectors: + - matchLabels: + app: production-workload-1 + component: vm-group + - matchLabels: + app: production-workload-2 + component: vm-service +``` + +**Expected outcome:** ConfigMaps matching either label combination are backed up; other ConfigMaps in the namespace are not (for this kind). + +**Note:** Prefer `matchExpressions` with `In` for value-OR on a single key (see next example). Use `orLabelSelectors` when you need OR across **independent multi-key groups**. `labelSelector` and `orLabelSelectors` cannot appear in the same `resourceFilters` entry. + +--- + +### Example 4b — Set-based label selectors (`matchExpressions`) + +**Goal:** Back up Deployments and Pods that are in `prod` or `staging`, belong to `app=my-app`, and do **not** carry a skip label. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [Deployment, Pod] + labelSelector: + matchLabels: + app: my-app + matchExpressions: + - key: environment + operator: In + values: [prod, staging] + - key: do-not-backup + operator: DoesNotExist +``` + +**Supported operators:** `In`, `NotIn`, `Exists`, `DoesNotExist` (same as Kubernetes / Velero global `--selector`). + +**Other useful patterns:** + +```yaml +# Exclude environments +matchExpressions: + - key: environment + operator: NotIn + values: [dev, test] + +# Require a label key to be present (any value) +matchExpressions: + - key: tier + operator: Exists +``` + +**Expected outcome:** Only Deployments/Pods with `app=my-app`, `environment` in `{prod, staging}`, and without `do-not-backup` are backed up. + +--- + +### Example 5 — OR label selectors across kinds + +**Goal:** Back up ConfigMaps, Secrets, or Deployments that match any of several label conditions. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [ConfigMap, Secret] + orLabelSelectors: + - matchLabels: + app: my-app + - matchLabels: + app: monitoring + - kinds: [Deployment] + orLabelSelectors: + - matchLabels: + app: my-app + - matchLabels: + app: monitoring + - matchLabels: + component: backend +``` + +**Expected outcome:** Resources included if they match **any** selector in `orLabelSelectors` for their kind (AND within each selector, OR across the list). + +--- + +### Example 6 — Multiple criteria on one kind + +**Goal:** Combine exact names with OR label selectors for a single kind. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - target-namespace + resourceFilters: + - kinds: [ConfigMap] + names: [vm-1, vm-2] + orLabelSelectors: + - matchLabels: + resource-type: VirtualMachine + - matchLabels: + component: vm-group + - matchLabels: + component: vm-service +``` + +**Expected outcome:** Only `vm-1` and `vm-2` that also satisfy one of the label OR branches. + +--- + +### Example 7 — One policy entry, multiple namespaces + +**Goal:** Apply the same rules to `ns-a`, `ns-b`, and `production` in a single policy block. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + - ns-b + - production + resourceFilters: + - kinds: [ConfigMap] + - kinds: [Deployment] + labelSelector: + matchLabels: + tier: web +``` + +**Expected outcome:** + +- All ConfigMaps in those namespaces (no label filter on that entry). +- Deployments with `tier=web` only. + +--- + +### Example 8 — Namespace glob patterns and ordering + +**Goal:** Different backup breadth for `team-frontend-prod`, `team-frontend-dev`, and `team-backend-test` using glob patterns. + +**Policy (correct order — most specific first):** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - "team-frontend-*" + resourceFilters: + - kinds: [Deployment, Service, ConfigMap] + - namespaces: + - "team-*" + resourceFilters: + - kinds: [Deployment, Service] + - namespaces: + - team-frontend-prod # exact match + resourceFilters: + - kinds: [Deployment, Service, ConfigMap, Secret, PersistentVolumeClaim] +``` + +**Expected outcome:** + +| Namespace | Matched policy | Kinds backed up | +|-----------|----------------|-----------------| +| `team-frontend-prod` | First entry (exact) | 5 kinds | +| `team-frontend-dev` | `team-frontend-*` | 3 kinds | +| `team-backend-test` | `team-*` | 2 kinds | + +**Wrong order (avoid):** If `team-*` is listed **before** `team-frontend-*`, then `team-frontend-dev` matches the broader `team-*` rule first and only Deployments and Services are backed up — the more specific `team-frontend-*` rule is never reached. + +Velero evaluates namespaces by looking for an **exact match** first, and then evaluates glob patterns in **definition order** (first-match wins). Because `team-frontend-prod` is an exact match in this policy, its evaluation is unaffected by glob ordering. However, for namespaces relying on glob patterns like `team-frontend-dev`, the order of the glob patterns is critical. + +**Backup:** Include all relevant namespaces in `includedNamespaces` (they must still pass the global namespace filter). + +--- + +### Example 9 — Catch-all by label + +**Goal:** Back up any resource kind that has a given label, without listing every kind. Kind-specific entries override the catch-all. + +**Policy (recommended explicit form):** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: ["*"] # catch-all + labelSelector: + matchLabels: + app: common-app + - kinds: [ConfigMap, Secret] # override for these kinds + labelSelector: + matchLabels: + app: specialized-app +``` + +**Equivalent:** `kinds: []` (empty) also denotes a catch-all; `kinds: ["*"]` is preferred for readability. + +**Rules:** + +- At most **one** catch-all per namespace policy entry. +- Catch-all entries **cannot** use `names` or `excludedNames` — use kind-specific entries for name filtering. +- Catch-all does **not** inherit `BackupSpec.labelSelector`; set `labelSelector` or `orLabelSelectors` on the catch-all entry explicitly. + +**Expected outcome:** ConfigMaps and Secrets use `app=specialized-app`; all other kinds listed only via catch-all use `app=common-app`. + +--- + +### Example 10 — Catch-all with per-kind name overrides + +**Goal:** Pin critical Deployments and Secrets by exact name; back up everything else with a label convention. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [Deployment] + names: [api-server, worker] + - kinds: [Secret] + names: [db-credentials, tls-cert] + - kinds: ["*"] + labelSelector: + matchLabels: + backup: "true" +``` + +**Expected outcome:** + +- Deployments: only `api-server` and `worker` +- Secrets: only `db-credentials` and `tls-cert` +- Other kinds (ConfigMap, Service, …): resources with `backup=true` only + +**Verify:** `other-deployment` and `no-backup-label-config` should be absent; `backup-labeled-config` and `catch-all-labeled-service` should be present. + +--- + +### Example 11 — Override-only catch-all (no label on catch-all) + +**Goal:** Apply a strict name filter to one kind while including all other kinds without listing them or adding labels. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [Secret] + names: [app-secret] + - kinds: ["*"] # no labelSelector — all other kinds included +``` + +**Expected outcome:** + +- Secrets: only `app-secret` +- Other kinds in `ns-a`: all instances included (subject to global filters and allowlist semantics for listed vs unlisted kinds via catch-all) + +Use this when you need a narrow exception for one type and broad inclusion for the rest of the namespace. + +--- + +### Example 12 — Cluster-scoped refinement + +**Goal:** Refine which cluster-scoped resources are backed up by name and label, without replacing global cluster-scoped inclusion. + +**Policy:** + +```yaml +version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: [StorageClass] + names: ["my-app-*"] + - kinds: [ClusterRole, ClusterRoleBinding] + labelSelector: + matchLabels: + app: my-app +``` + +**Backup (required):** You must still include cluster-scoped kinds on the Backup: + +```yaml +spec: + includedNamespaces: + - ns-a + includedClusterScopedResources: + - storageclasses + - clusterroles + - clusterrolebindings + resourcePolicy: + kind: configmap + name: cluster-scoped-filter-policy +``` + +**Expected outcome (full overlay):** + +- StorageClasses matching `my-app-*` only +- ClusterRoles and ClusterRoleBindings with `app=my-app` only +- Namespace-scoped resources in `ns-a`: global filters (no `namespacedFilterPolicies` in this example) + +**Partial overlay:** If `includedClusterScopedResources` lists only `clusterroles` and `clusterrolebindings`, StorageClasses are **not** backed up even if listed in `clusterScopedFilterPolicy` — global inclusion is evaluated first. + +**Differences from namespace policies:** + +- **Not** an allowlist — unlisted cluster-scoped kinds fall back to global filters. +- **No catch-all** — `kinds: []` or `kinds: ["*"]` is invalid and fails validation. + +--- + +### Example 13 — Global `includeExcludePolicy` and namespace filters + +**Goal:** Set a global resource-type baseline, then refine per namespace. Understand that global **exclusions** cannot be overridden per namespace. + +**Policy:** + +```yaml +version: v1 +includeExcludePolicy: + includedNamespaceScopedResources: + - configmaps + - secrets + - deployments + - services +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [ConfigMap, Secret] + labelSelector: + matchLabels: + app: my-app + - namespaces: + - production + resourceFilters: + - kinds: [ConfigMap] + names: ["app-*"] +``` + +**Expected outcome:** + +- **ns-a:** ConfigMaps and Secrets with `app=my-app` (within global allowlist) +- **production:** ConfigMaps matching `app-*` pattern +- **Other included namespaces:** Only kinds allowed by `includeExcludePolicy` (no per-namespace override) + +**Global exclusion wins (important):** + +```yaml +includeExcludePolicy: + excludedNamespaceScopedResources: + - secrets +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [ConfigMap, Secret, Deployment] + labelSelector: + matchLabels: + app: my-app +``` + +**Result:** No Secrets in the backup — the namespace policy cannot re-include a globally excluded kind. Velero logs a warning at backup start if you list an excluded kind in `namespacedFilterPolicies`. + +**Backup tip:** Do not set `includedResources` on the Backup; use `includeExcludePolicy` in the ConfigMap instead. + +--- + +### Example 14 — Volume policies and namespace filters together + +**Goal:** Use volume snapshot/fs-backup rules and namespace filters in one ConfigMap. + +**Policy:** + +```yaml +version: v1 +volumePolicies: + - conditions: + capacity: "0,10Gi" + storageClass: + - standard + action: + type: fs-backup + - conditions: + capacity: "10Gi,100Gi" + action: + type: snapshot +namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [ConfigMap] + names: ["app-*"] + excludedNames: ["*-tmp-*", "*-debug-*", "*-tmp", "*-debug"] + - kinds: [Secret] + labelSelector: + matchLabels: + workload: application +``` + +**Expected outcome:** Volume actions apply to PVCs per `volumePolicies`; resource inclusion follows `namespacedFilterPolicies`. The sections are independent. + +--- + +### Example 15 — `velero.io/exclude-from-backup=true` always wins + +**Goal:** Ensure explicitly excluded resources never appear in the backup, even when they match namespace filters or catch-all rules. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [ConfigMap, Secret] + labelSelector: + matchLabels: + app: my-app + - kinds: ["*"] + labelSelector: + matchLabels: + app: my-app +``` + +**On resources to exclude**, set: + +```yaml +metadata: + labels: + velero.io/exclude-from-backup: "true" +``` + +**Expected outcome:** Resources with `app=my-app` **and** `velero.io/exclude-from-backup=true` are excluded. Same rule applies to cluster-scoped resources refined by `clusterScopedFilterPolicy`. + +--- + +## Concepts reference + +### `resourceFilters` fields + +| Field | Description | +|-------|-------------| +| `kinds` | Resource type names (e.g. `ConfigMap`, `deployments`). Empty or `["*"]` = catch-all (namespace policies only). | +| `labelSelector` | Kubernetes-style selector with `matchLabels` and/or `matchExpressions` (`In`, `NotIn`, `Exists`, `DoesNotExist`). All requirements are AND-ed. | +| `orLabelSelectors` | List of selectors; match if **any** entry matches (AND within each, OR across the list). Use for OR of multi-key groups; prefer `In` for value-OR on one key. Mutually exclusive with `labelSelector`. | +| `names` | Exact names or glob patterns to include. | +| `excludedNames` | Patterns to exclude; wins over `names` when both match. | + +Only kinds listed in `resourceFilters` (or covered by catch-all) are collected from namespaces matched by `namespacedFilterPolicies`. + +### Glob pattern syntax + +Name and namespace patterns use the same glob style as elsewhere in Velero (`gobwas/glob`): + +- Supported: `*`, `?`, `[abc]`, `[a-z]` +- Not supported: `**`, regex, `|`, `()`, `!`, `{}`, `,` + +Examples: `app-*`, `team-frontend-*`, `*-tmp`. + +### Precedence cheat sheet + +**Namespaces** + +1. `BackupSpec.excludedNamespaces` — excluded namespaces are never backed up; namespace policies cannot override this. +2. `namespacedFilterPolicies` — first matching pattern (exact match checked before globs in pattern order). +3. No match — use global BackupSpec + `includeExcludePolicy`. + +**Namespace-scoped resources (when a namespace policy matches)** + +1. Global `includeExcludePolicy` exclusions (e.g. `excludedNamespaceScopedResources`) apply first. +2. Only kinds in `resourceFilters` (or catch-all) are allowlisted for collection. +3. Per-kind `labelSelector` / `orLabelSelectors` for API list calls. +4. Per-kind `names` / `excludedNames` at backup write time. +5. Label `velero.io/exclude-from-backup=true` always excludes. + +**Cluster-scoped resources** + +1. Must be allowed by `includedClusterScopedResources` / global cluster settings. +2. If `clusterScopedFilterPolicy` lists the kind, apply its label and name rules. +3. If not listed in `clusterScopedFilterPolicy`, use global BackupSpec filters. +4. `velero.io/exclude-from-backup=true` always excludes. + +```mermaid +flowchart TD + nsGlobal[BackupSpec namespace include/exclude] + nsPolicy{namespacedFilterPolicies match?} + nsAllow[Allowlist kinds + per-kind filters] + nsGlobalFallback[Global BackupSpec + includeExcludePolicy] + + nsGlobal --> nsPolicy + nsPolicy -->|yes| nsAllow + nsPolicy -->|no| nsGlobalFallback + + csInclude[includedClusterScopedResources] + csPolicy{kind in clusterScopedFilterPolicy?} + csRefine[Per-kind label and name rules] + csGlobal[Global cluster filters] + + csInclude --> csPolicy + csPolicy -->|yes| csRefine + csPolicy -->|no| csGlobal +``` + +### Catch-all summary + +| Rule | Detail | +|------|--------| +| Syntax | `kinds: ["*"]` or `kinds: []` | +| Count | At most one catch-all per `namespacedFilterPolicies` entry | +| Names | `names` / `excludedNames` not allowed on catch-all | +| Override | Kind-specific entries take precedence over catch-all | +| Label inheritance | Does not use `BackupSpec.labelSelector` | +| Cluster-scoped | Catch-all **not** supported in `clusterScopedFilterPolicy` | + +--- + +## Troubleshooting and validation + +### Verify a backup + +```bash +velero backup describe BACKUP_NAME +velero backup logs BACKUP_NAME +velero backup describe BACKUP_NAME -o json | jq '.namespacedFilterPolicies' +velero backup describe BACKUP_NAME -o json | jq '.clusterScopedFilterPolicy' +``` + +Catch-all entries appear as ` (all other kinds)` in text output, or `"isCatchAll": true` in JSON. + +### Common misconfigurations + +| Symptom | Likely cause | Fix | +|---------|----------------|-----| +| Fewer resources than expected in `team-frontend-prod` | Broad namespace pattern listed before specific one | Reorder policies: most specific `namespaces` first | +| Namespace policy lists Secrets but none in backup | `includeExcludePolicy` excludes `secrets` globally | Remove global exclusion or accept no Secrets | +| `ClusterRole` in namespace policy has no effect | Cluster-scoped kind in `namespacedFilterPolicies` | Move rule to `clusterScopedFilterPolicy`; check logs for warning | +| Backup fails at creation with filter message | Old-style `includedResources` with new policies | Move resource types to `includeExcludePolicy` in ConfigMap | +| Catch-all does not use backup-wide label | By design | Set `labelSelector` on the catch-all entry | +| Cluster-scoped policy validation error on `kinds: ["*"]` | Catch-all not allowed for cluster policy | List each cluster-scoped kind explicitly | + +### Velero logs + +```bash +kubectl logs -n velero deployment/velero | grep -i "namespacedFilterPolicies\|clusterScopedFilterPolicy" +kubectl logs -n velero deployment/velero | grep "globally excluded by includeExcludePolicy" +kubectl logs -n velero deployment/velero | grep "cluster-scoped" +``` + +### Validation errors (policy ConfigMap) + +Velero validates the ResourcePolicy when a backup starts. Common errors: + +| Error (summary) | Cause | +|-----------------|--------| +| `at least one namespace must be specified` | Empty `namespaces: []` | +| `at least one resourceFilter must be specified` | Empty `resourceFilters: []` | +| `names or excludedNames cannot be specified for catch-all filters` | Name patterns on catch-all entry | +| `only one catch-all resource filter is allowed` | Multiple catch-alls in one policy entry | +| `kind "X" appears in both resourceFilters[...]` | Same kind in two entries | +| `labelSelector and orLabelSelectors cannot co-exist` | Both set in one entry | +| `invalid label selector` | Bad operator, values, or label key/value syntax | +| `duplicate namespace pattern` | Same namespace string in two policy entries | +| `invalid glob pattern` | Bad characters in namespace or name pattern | +| `clusterScopedFilterPolicy... kinds must be specified (catch-all is not supported)` | Empty or `["*"]` kinds in cluster policy | +| `include-resources, exclude-resources... cannot be used with namespace-scoped or cluster-scoped global filter policies` | Old-style BackupSpec filters with new policy | + +### Silent edge cases (no error) + +- Namespace pattern matches no existing namespace — policy loaded but never applied. +- Kind listed but no instances in namespace — empty result, backup still succeeds. +- `excludedNames` narrows `names` — e.g. `names: ["app-*"]` + `excludedNames: ["app-config"]` excludes `app-config` only. + +--- + +## Restore behavior + +Restore is unchanged: it restores whatever is in the backup archive. Resources excluded by fine-grained filters are simply absent. Use `Restore.spec.includedNamespaces` (and existing restore filters) to limit what you restore from a partial backup. + +Fine-grained resource filtering is also available on the restore path using `namespacedFilterPolicies` and `clusterScopedFilterPolicy`. For details on the restore-side policies, see the [Fine-grained restore filters design](https://github.com/velero-io/velero/blob/main/design/restore-filter-enhancement/fine-grained-restore-filters-design.md). + +--- + +## Related links + +- [Fine-grained backup filters design](https://github.com/velero-io/velero/blob/main/design/backup-filter-enhancement/fine-grained-backup-filters-design.md) diff --git a/site/content/docs/main/fine-grained-restore-filters.md b/site/content/docs/main/fine-grained-restore-filters.md new file mode 100644 index 000000000..0f7c52c26 --- /dev/null +++ b/site/content/docs/main/fine-grained-restore-filters.md @@ -0,0 +1,726 @@ +--- +title: "Fine-Grained Restore Filters" +layout: docs +--- + +This guide explains how to use Velero's **fine-grained restore filters**: per-namespace, per-kind rules with independent label selectors and resource name patterns. Configuration lives in a **ResourcePolicy ConfigMap**, using the exact same format introduced for fine-grained backup filters. + +For architecture and pipeline details, see the [design document](https://github.com/velero-io/velero/blob/main/design/restore-filter-enhancement/fine-grained-restore-filters-design.md). + +--- + +## Introduction + +Velero's traditional restore filters apply the same namespace list, resource types, and label selector to every namespace being restored. Common scenarios need more control: + +- **Selective restore from a full backup** — restore only specific application components from a namespace, leaving out monitoring or logging resources that were also backed up. +- **Cross-environment migration** — restore StatefulSets and PVCs in a database namespace, but only Deployments and Services in a frontend namespace. +- **Filter by resource name** — restore `app-config` and `app-secret` without restoring `monitoring-config` from the same namespace. +- **Restore-time override** — apply different label selectors during restore than were used during backup to handle environment differences. + +Fine-grained filters add two optional sections to the ResourcePolicy ConfigMap: + +| Section | Scope | Behavior | +|---------|-------|----------| +| `namespacedFilterPolicies` | Namespaces you match (exact name or glob) | **Exclusive allowlist** — only resource kinds listed in `resourceFilters` (or covered by a catch-all) are restored for those namespaces, provided they pass global filters. | +| `clusterScopedFilterPolicy` | Cluster-scoped resources globally | **Refinement overlay** — listed kinds get per-kind label and name rules; unlisted cluster-scoped kinds still use global RestoreSpec filters. | + +**Backward compatible:** Fine-grained restore filters are optional. If a restore does not reference a ResourcePolicy, Velero relies solely on standard RestoreSpec filters (includedNamespaces, includedResources, labelSelector, etc.). + +--- + +## Prerequisites and wiring + +### What you need + +- A ResourcePolicy ConfigMap in the Velero namespace (`velero` by default). +- Permission to create Restores that reference the ConfigMap. + +### End-to-end pattern + +Every example below follows the same three steps: + +1. **Create or update** a ConfigMap with `data.policy` containing `version: v1` and your filter rules. +2. **Create a Restore** that includes the target namespaces and references the ConfigMap. +3. **Verify** with `velero restore describe` and inspect the restored resources. + +### Minimal skeleton + +Use this once; later examples show only the `policy:` body. + +**ResourcePolicy ConfigMap:** + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-restore-filter-policy + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + - namespaces: + - my-namespace + resourceFilters: + - kinds: [ConfigMap] + labelSelector: + matchLabels: + app: my-app +``` + +**Restore:** + +```yaml +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: my-restore + namespace: velero +spec: + backupName: my-backup + includedNamespaces: + - my-namespace + resourcePolicy: + kind: configmap + name: my-restore-filter-policy +``` + +**CLI equivalent:** + +```bash +velero restore create my-restore \ + --from-backup my-backup \ + --include-namespaces my-namespace \ + --resource-policies-configmap my-restore-filter-policy +``` + +**Verify:** + +```bash +velero restore describe my-restore +``` + +### Important: Interaction with Global Filters + +The restore pipeline evaluates **global resource filters first**: +- `RestoreSpec.IncludedResources` and `RestoreSpec.ExcludedResources` act as a global gate. +- A resource kind **must** pass the global gate before per-namespace filters are evaluated. +- **A namespace policy cannot re-include a globally excluded kind.** If you globally exclude `secrets`, listing `Secret` in a namespace policy will have no effect. + +--- + +## Examples + +Each example includes: **goal**, **policy YAML**, **restore notes**, and **expected outcome**. + +--- + +### Example 0 — Baseline (no new filters) + +**Goal:** Confirm that namespaces without a `namespacedFilterPolicies` entry still use global RestoreSpec filters. + +**Policy:** Omit `namespacedFilterPolicies` and `clusterScopedFilterPolicy` entirely. + +**Restore:** + +```yaml +spec: + includedNamespaces: + - ns-a + - ns-b + # No resourcePolicy — global filters only +``` + +**Expected outcome:** All resources in included namespaces follow `includedNamespaces`, `labelSelector`, `includedResources`, and related global fields — same as before this feature. + +--- + +### Example 1 — Per-namespace kinds and labels + +**Goal:** In `ns-a`, restore only ConfigMaps, Secrets, Deployments, and Pods with `app=my-app`. In `ns-b`, use global filters (no policy entry for that namespace). + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [ConfigMap, Secret, Deployment, Pod] + labelSelector: + matchLabels: + app: my-app +``` + +**Restore:** + +```yaml +spec: + includedNamespaces: + - ns-a + - ns-b + resourcePolicy: + kind: configmap + name: per-namespace-resource-filter-policy +``` + +**Expected outcome:** + +- **ns-a:** Only listed kinds with label `app=my-app` (e.g. `app-config`, `app-secret`, `app-deployment`). Resources like `monitoring-config` (different labels) are excluded. +- **ns-b:** Everything allowed by global filters (no namespace policy match). + +--- + +### Example 2 — Exact resource names + +**Goal:** Restore only two ConfigMaps by exact name, optionally requiring a label. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - target-namespace + resourceFilters: + - kinds: [ConfigMap] + names: [vm-1, vm-2] + labelSelector: + matchLabels: + resource-type: VirtualMachine +``` + +**Expected outcome:** Only `vm-1` and `vm-2` ConfigMaps with `resource-type=VirtualMachine` are restored. `vm-3` and other ConfigMaps are skipped. + +--- + +### Example 3 — Glob name patterns with exclusions + +**Goal:** Restore `app-*` ConfigMaps and Secrets in `production`, but exclude temporary and debug names. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [ConfigMap, Secret] + names: ["app-*"] + excludedNames: ["*-tmp-*", "*-debug-*", "*-tmp", "*-debug"] +``` + +**Expected outcome:** + +- **Included:** `app-config`, `app-cache-config`, `app-secret` +- **Excluded:** `app-config-tmp`, `app-tmp-config`, `app-debug-config`, `monitoring-tmp-secret` + +`excludedNames` takes precedence over `names` when both match. + +--- + +### Example 4 — Per-kind label selectors + +**Goal:** Apply different label rules to different resource types in the same namespace. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - target-namespace + resourceFilters: + - kinds: [ConfigMap] + orLabelSelectors: + - matchLabels: + app: production-workload-1 + component: vm-group + - matchLabels: + app: production-workload-2 + component: vm-service +``` + +**Expected outcome:** ConfigMaps matching either label combination are restored; other ConfigMaps in the namespace are not. + +**Note:** Prefer `matchExpressions` with `In` for value-OR on a single key (see next example). Use `orLabelSelectors` when you need OR across **independent multi-key groups**. `labelSelector` and `orLabelSelectors` cannot appear in the same `resourceFilters` entry. + +--- + +### Example 4b — Set-based label selectors (`matchExpressions`) + +**Goal:** Restore Deployments and Pods that are in `prod` or `staging`, belong to `app=my-app`, and do **not** carry a skip label. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [Deployment, Pod] + labelSelector: + matchLabels: + app: my-app + matchExpressions: + - key: environment + operator: In + values: [prod, staging] + - key: do-not-restore + operator: DoesNotExist +``` + +**Supported operators:** `In`, `NotIn`, `Exists`, `DoesNotExist` (same as Kubernetes / Velero global `--selector`). + +**Other useful patterns:** + +```yaml +# Exclude environments +matchExpressions: + - key: environment + operator: NotIn + values: [dev, test] + +# Require a label key to be present (any value) +matchExpressions: + - key: tier + operator: Exists +``` + +**Expected outcome:** Only Deployments/Pods with `app=my-app`, `environment` in `{prod, staging}`, and without `do-not-restore` are restored. + +--- + +### Example 5 — OR label selectors across kinds + +**Goal:** Restore ConfigMaps, Secrets, or Deployments that match any of several label conditions. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [ConfigMap, Secret] + orLabelSelectors: + - matchLabels: + app: my-app + - matchLabels: + app: monitoring + - kinds: [Deployment] + orLabelSelectors: + - matchLabels: + app: my-app + - matchLabels: + app: monitoring + - matchLabels: + component: backend +``` + +**Expected outcome:** Resources included if they match **any** selector in `orLabelSelectors` for their kind (AND within each selector, OR across the list). + +--- + +### Example 6 — Multiple criteria on one kind + +**Goal:** Combine exact names with OR label selectors for a single kind. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - target-namespace + resourceFilters: + - kinds: [ConfigMap] + names: [vm-1, vm-2] + orLabelSelectors: + - matchLabels: + resource-type: VirtualMachine + - matchLabels: + component: vm-group + - matchLabels: + component: vm-service +``` + +**Expected outcome:** Only `vm-1` and `vm-2` that also satisfy one of the label OR branches. + +--- + +### Example 7 — One policy entry, multiple namespaces + +**Goal:** Apply the same rules to `ns-a`, `ns-b`, and `production` in a single policy block. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + - ns-b + - production + resourceFilters: + - kinds: [ConfigMap] + - kinds: [Deployment] + labelSelector: + matchLabels: + tier: web +``` + +**Expected outcome:** + +- All ConfigMaps in those namespaces (no label filter on that entry). +- Deployments with `tier=web` only. + +--- + +### Example 8 — Namespace glob patterns and ordering + +**Goal:** Different restore breadth for `team-frontend-prod`, `team-frontend-dev`, and `team-backend-test` using glob patterns. + +**Note on Precedence:** Exact namespace matches always take precedence regardless of where they are listed. However, if multiple glob patterns could match a namespace, they are evaluated in the order they appear. Always list specific globs before broad globs. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + # Globs must be ordered specific-to-broad + - namespaces: + - "team-frontend-*" # specific pattern match + resourceFilters: + - kinds: [Deployment, Service, ConfigMap] + - namespaces: + - "team-*" # broad pattern + resourceFilters: + - kinds: [Deployment, Service] + + # Exact matches always win, even if placed at the bottom + - namespaces: + - team-frontend-prod # exact match + resourceFilters: + - kinds: [Deployment, Service, ConfigMap, Secret, PersistentVolumeClaim] +``` + +**Expected outcome:** + +| Namespace | Matched policy | Kinds restored | +|-----------|----------------|-----------------| +| `team-frontend-prod` | `team-frontend-prod` (Exact match priority) | 5 kinds | +| `team-frontend-dev` | `team-frontend-*` (First matching glob) | 3 kinds | +| `team-backend-test` | `team-*` (First matching glob) | 2 kinds | + +Velero uses **first-match** semantics: the first policy entry whose namespace pattern matches wins. + +--- + +### Example 9 — Catch-all by label + +**Goal:** Restore any resource kind that has a given label, without listing every kind. Kind-specific entries override the catch-all. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: ["*"] # catch-all + labelSelector: + matchLabels: + app: common-app + - kinds: [ConfigMap, Secret] # override for these kinds + labelSelector: + matchLabels: + app: specialized-app +``` + +**Rules:** + +- At most **one** catch-all per namespace policy entry. +- Catch-all entries **cannot** use `names` or `excludedNames`. +- Catch-all does **not** inherit `RestoreSpec.LabelSelector`. + +**Expected outcome:** ConfigMaps and Secrets use `app=specialized-app`; all other kinds listed only via catch-all use `app=common-app`. + +--- + +### Example 10 — Catch-all with per-kind name overrides + +**Goal:** Pin critical Deployments and Secrets by exact name; restore everything else with a label convention. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [Deployment] + names: [api-server, worker] + - kinds: [Secret] + names: [db-credentials, tls-cert] + - kinds: ["*"] + labelSelector: + matchLabels: + restore: "true" +``` + +**Expected outcome:** + +- Deployments: only `api-server` and `worker` +- Secrets: only `db-credentials` and `tls-cert` +- Other kinds (ConfigMap, Service, …): resources with `restore=true` only + +--- + +### Example 11 — Override-only catch-all (no label on catch-all) + +**Goal:** Apply a strict name filter to one kind while restoring all other kinds without listing them or adding labels. + +**Policy:** + +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [Secret] + names: [app-secret] + - kinds: ["*"] # no labelSelector — all other kinds included +``` + +**Expected outcome:** + +- Secrets: only `app-secret` +- Other kinds in `ns-a`: all instances restored (subject to global filters) + +--- + +### Example 12 — Cluster-scoped refinement + +**Goal:** Refine which cluster-scoped resources are restored by name and label, without replacing global cluster-scoped inclusion. + +**Policy:** + +```yaml +version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: [StorageClass] + names: ["my-app-*"] + - kinds: [ClusterRole, ClusterRoleBinding] + labelSelector: + matchLabels: + app: my-app +``` + +**Restore (required):** You must still include cluster-scoped kinds on the Restore: + +```yaml +spec: + includeClusterResources: true + resourcePolicy: + kind: configmap + name: cluster-scoped-filter-policy +``` + +**Expected outcome:** + +- StorageClasses matching `my-app-*` only +- ClusterRoles and ClusterRoleBindings with `app=my-app` only +- Other cluster-scoped resources: restored according to global filters. + +**Differences from namespace policies:** + +- **Not** an allowlist — unlisted cluster-scoped kinds fall back to global filters. +- **No catch-all** — `kinds: []` or `kinds: ["*"]` is invalid and fails validation. + +--- + +### Example 13 — Global `ExcludedResources` and namespace filters + +**Goal:** Understand that global **exclusions** cannot be overridden per namespace. + +**Restore:** +```yaml +spec: + excludedResources: + - secrets +``` + +**Policy:** +```yaml +version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [ConfigMap, Secret, Deployment] + labelSelector: + matchLabels: + app: my-app +``` + +**Result:** No Secrets are restored — the namespace policy cannot re-include a globally excluded kind. Velero logs a warning at restore start if you list an excluded kind in `namespacedFilterPolicies`. + +--- + +### Example 14 — Separate ConfigMaps for Backup and Restore + +**Goal:** Understand why you cannot use a single ConfigMap for both backup and restore operations if it contains backup-specific policies. + +**Policy:** + +```yaml +version: v1 +volumePolicies: + - conditions: + capacity: "0,10Gi" + action: + type: fs-backup +namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [ConfigMap, Secret] + names: ["app-*"] +``` + +**Expected outcome:** The restore operation will **fail validation**. The Velero restore pipeline strictly rejects any ResourcePolicy ConfigMap containing `volumePolicies` or `includeExcludePolicy`. To avoid this, the restore-side ConfigMap should contain only the restore-supported sections (`namespacedFilterPolicies` and/or `clusterScopedFilterPolicy`). + +--- + +### Example 15 — `velero.io/exclude-from-backup=true` always wins + +**Goal:** Ensure explicitly excluded resources never appear in the restore. + +If a resource was backed up (perhaps before the label was added, or manually modified in the archive) but has `velero.io/exclude-from-backup: "true"`, the restore pipeline honors it. Any item carrying this label is skipped regardless of whether it matches global or per-namespace restore filters. + +--- + +## Concepts reference + +### `resourceFilters` fields + +| Field | Description | +|-------|-------------| +| `kinds` | Resource type names (e.g. `ConfigMap`, `deployments`). Empty or `["*"]` = catch-all (namespace policies only). | +| `labelSelector` | Kubernetes-style selector with `matchLabels` and/or `matchExpressions` (`In`, `NotIn`, `Exists`, `DoesNotExist`). All requirements are AND-ed. | +| `orLabelSelectors` | List of selectors; match if **any** entry matches (AND within each, OR across the list). Use for OR of multi-key groups; prefer `In` for value-OR on one key. Mutually exclusive with `labelSelector`. | +| `names` | Exact names or glob patterns to include. | +| `excludedNames` | Patterns to exclude; wins over `names` when both match. | + +### Glob pattern syntax + +Name and namespace patterns use the same glob style as elsewhere in Velero (`gobwas/glob`): + +- Supported: `*`, `?`, `[abc]`, `[a-z]` +- Not supported: `**`, regex, `|`, `()`, `!`, `{}`, `,` + +Examples: `app-*`, `team-frontend-*`, `*-tmp`. + +### Precedence cheat sheet + +**Namespaces** + +1. `RestoreSpec.ExcludedNamespaces` — excluded namespaces are never restored. +2. `namespacedFilterPolicies` — first matching pattern (exact match checked before globs in pattern order). +3. No match — use global RestoreSpec filters. + +**Namespace-scoped resources (when a namespace policy matches)** + +1. Global `RestoreSpec.IncludedResources` / `ExcludedResources` apply first. +2. Only kinds in `resourceFilters` (or catch-all) are allowlisted for restoration. +3. Per-kind `labelSelector` / `orLabelSelectors` replace global selectors. +4. Per-kind `names` / `excludedNames` filter by resource name. +5. Label `velero.io/exclude-from-backup=true` always excludes. +6. **Plugin Additional Items** bypass fine-grained filters to ensure dependencies (like PVs) are restored. + +**Cluster-scoped resources** + +1. Must be allowed by global cluster settings (`includeClusterResources`). +2. If `clusterScopedFilterPolicy` lists the kind, apply its label and name rules. +3. If not listed in `clusterScopedFilterPolicy`, use global RestoreSpec filters. +4. `velero.io/exclude-from-backup=true` always excludes. + +### Catch-all summary + +| Rule | Detail | +|------|--------| +| Syntax | `kinds: ["*"]` or `kinds: []` | +| Count | At most one catch-all per `namespacedFilterPolicies` entry | +| Names | `names` / `excludedNames` not allowed on catch-all | +| Override | Kind-specific entries take precedence over catch-all | +| Label inheritance | Does not use `RestoreSpec.LabelSelector` | +| Cluster-scoped | Catch-all **not** supported in `clusterScopedFilterPolicy` | + +--- + +## Troubleshooting and validation + +### Verify a restore + +```bash +velero restore describe RESTORE_NAME +velero restore logs RESTORE_NAME +``` + +The output of `velero restore describe` will show the `Resource Policy` field if a ConfigMap was used. + +### Common misconfigurations + +| Symptom | Likely cause | Fix | +|---------|----------------|-----| +| Fewer resources than expected in `team-frontend-prod` | Broad namespace pattern listed before specific one | Reorder policies: most specific `namespaces` first | +| Namespace policy lists Secrets but none restored | `RestoreSpec.ExcludedResources` excludes `secrets` globally | Remove global exclusion or accept no Secrets | +| `ClusterRole` in namespace policy has no effect | Cluster-scoped kind in `namespacedFilterPolicies` | Move rule to `clusterScopedFilterPolicy`; check logs for warning | +| Catch-all does not use restore-wide label | By design | Set `labelSelector` on the catch-all entry | +| Cluster-scoped policy validation error on `kinds: ["*"]` | Catch-all not allowed for cluster policy | List each cluster-scoped kind explicitly | + +### Velero logs + +```bash +kubectl logs -n velero deployment/velero | grep -i "namespacedFilterPolicies\|clusterScopedFilterPolicy" +kubectl logs -n velero deployment/velero | grep "globally excluded by RestoreSpec.ExcludedResources" +``` + +### Validation errors (policy ConfigMap) + +Velero validates the ResourcePolicy when a restore starts. Common errors: + +| Error (summary) | Cause | +|-----------------|--------| +| `at least one namespace must be specified` | Empty `namespaces: []` | +| `at least one resourceFilter must be specified` | Empty `resourceFilters: []` | +| `names or excludedNames cannot be specified for catch-all filters` | Name patterns on catch-all entry | +| `only one catch-all resource filter is allowed` | Multiple catch-alls in one policy entry | +| `kind "X" appears in both resourceFilters[...]` | Same kind in two entries | +| `labelSelector and orLabelSelectors cannot co-exist` | Both set in one entry | +| `invalid label selector` | Bad operator, values, or label key/value syntax | +| `duplicate namespace pattern` | Same namespace string in two policy entries | +| `invalid glob pattern` | Bad characters in namespace or name pattern | +| `clusterScopedFilterPolicy... kinds must be specified (catch-all is not supported)` | Empty or `["*"]` kinds in cluster policy | + +### Silent edge cases (no error) + +- Namespace pattern matches no existing namespace in the backup — policy loaded but never applied. +- Kind listed but no instances in namespace — empty result, restore still succeeds. +- `excludedNames` narrows `names` — e.g. `names: ["app-*"]` + `excludedNames: ["app-config"]` excludes `app-config` only. + +--- + +## Related links + +- [Fine-grained restore filters design](https://github.com/velero-io/velero/blob/main/design/restore-filter-enhancement/fine-grained-restore-filters-design.md) diff --git a/site/content/docs/main/namespace.md b/site/content/docs/main/namespace.md index 68561e720..70d84b09a 100644 --- a/site/content/docs/main/namespace.md +++ b/site/content/docs/main/namespace.md @@ -17,6 +17,20 @@ To have namespace consistency, specify the namespace for all Velero operational velero client config set namespace= ``` +If Velero was installed in the namespace of your current kubeconfig context, you can have operational commands automatically use that namespace, without having to type it out or update it every time you switch contexts: + +```bash +velero client config set namespace-mode=auto +``` + +With `namespace-mode=auto` set, Velero resolves the namespace from the current kubeconfig context (or the context specified with `--kubecontext`) on every command invocation, instead of using the static `namespace` value. If the namespace can't be resolved from the kubeconfig context (for example, the context has no namespace set, or the kubeconfig can't be loaded), Velero falls back to the static `namespace` value, or the `velero` default if that isn't set either. + +To disable `namespace-mode=auto` and go back to using the static `namespace` value, clear it by setting it to an empty value: + +```bash +velero client config set namespace-mode= +``` + Alternatively, you may use the global `--namespace` flag with any operational command to tell Velero where to run. [0]: basic-install.md#install-the-cli diff --git a/site/content/docs/main/plugin-release-instructions.md b/site/content/docs/main/plugin-release-instructions.md index 46494cac9..02ca6940d 100644 --- a/site/content/docs/main/plugin-release-instructions.md +++ b/site/content/docs/main/plugin-release-instructions.md @@ -19,11 +19,11 @@ Plugins the Velero core team is responsible include all those listed in [the Vel 1. Once the PR is merged, checkout the upstream `main` branch. Your local upstream might be named `upstream` or `origin`, so use this command: `git checkout /main`. 1. Tag the git version - `git tag v`. 1. Push the git tag - `git push --tags ` to trigger the image build. -2. Wait for the container images to build. You may check the progress of the GH action that triggers the image build at `https://github.com/vmware-tanzu//actions` +2. Wait for the container images to build. You may check the progress of the GH action that triggers the image build at `https://github.com/velero-io//actions` 3. Verify that an image with the new tag is available at `https://hub.docker.com/repository/docker/velero//`. 4. Run the Velero [e2e tests][2] using the new image. Until it is made configurable, you will have to edit the [plugin version][1] in the test. ### Release -1. If all e2e tests pass, go to the GitHub release page of the plugin (`https://github.com/vmware-tanzu//releases`) and manually create a release for the new tag. +1. If all e2e tests pass, go to the GitHub release page of the plugin (`https://github.com/velero-io//releases`) and manually create a release for the new tag. 1. Copy and paste the content of the new changelog file into the release description field. [1]: https://github.com/velero-io/velero/blob/c8dfd648bbe85db0184ea53296de4220895497e6/test/e2e/velero_utils.go#L27 diff --git a/site/content/docs/main/resource-filtering.md b/site/content/docs/main/resource-filtering.md index 88584b362..9f57a7f4e 100644 --- a/site/content/docs/main/resource-filtering.md +++ b/site/content/docs/main/resource-filtering.md @@ -5,8 +5,8 @@ layout: docs *Filter objects by namespace, type, labels or resource policies.* -This page describes how to filter resource for backup and restore. -User could use the include and exclude flags with the `velero backup` and `velero restore` commands. And user could also use resource policies to handle backup. +This page describes how to filter resources for backup and restore. +Users can use include and exclude flags with the `velero backup` and `velero restore` commands. Users can also use resource policies for fine-grained resource filtering during backup and restore, as well as volume handling during backup. By default, Velero includes all objects in a backup or restore when no filtering options are used. ## Includes @@ -229,104 +229,139 @@ Kubernetes namespace resources to exclude from the backup, formatted as resource ``` ## Resource policies -Velero provides resource policies to filter resources to do backup, which may contain `includeExcludePolicy` and `volumePolicies`. -### Creating resource policies +Velero provides resource policies (defined in a ConfigMap and referenced via `--resource-policies-configmap` or `spec.resourcePolicy`) to define fine-grained resource filters and volume handling rules. -Below is the two-step of using resource policies in backup: -1. Creating resource policies configmap +Resource policies support both **Backup** and **Restore** operations, though certain policy sections are specific to backup workflows. - Users need to create one configmap in Velero install namespace from a YAML file that defined resource policies. The creating command would be like the below: +### Supported policy sections by operation + +| Policy Section | Description | Supported Operations | Learn More | +| --- | --- | --- | --- | +| `namespacedFilterPolicies` | Fine-grained per-namespace and per-kind filters with label selectors and resource name patterns. | **Backup** & **Restore** | [Fine-Grained Backup Filters](fine-grained-backup-filters.md) / [Fine-Grained Restore Filters](fine-grained-restore-filters.md) | +| `clusterScopedFilterPolicy` | Fine-grained cluster-scoped filter overlays with per-kind label selectors and resource name patterns. | **Backup** & **Restore** | [Fine-Grained Backup Filters](fine-grained-backup-filters.md) / [Fine-Grained Restore Filters](fine-grained-restore-filters.md) | +| `volumePolicies` | Rules to control volume data backup methods (`skip`, `snapshot`, `fs-backup`) based on conditions. | **Backup** only | See [VolumePolicy](#volumepolicy-backup-only) | +| `includeExcludePolicy` | Reusable scoped resource include/exclude filters. | **Backup** only | See [IncludeExcludePolicy](#includeexcludepolicy-backup-only) | + +### Creating and referencing resource policies + +Using resource policies is a two-step process: + +1. **Create the resource policies ConfigMap** + + Create a ConfigMap in the Velero installation namespace (typically `velero`) containing your YAML policy definition: ```bash kubectl create cm --from-file -n velero ``` -2. Creating a backup reference to the defined resource policies - Users create a backup with the flag `--resource-policies-configmap`, which will reference the current backup to the defined resource policies. The creating command would be like the below: - ```bash - velero backup create --resource-policies-configmap - ``` - This flag could also be combined with the other include and exclude filters above +2. **Reference the resource policies ConfigMap in a Backup or Restore** + + * **For Backup:** Reference the ConfigMap via CLI flag or in the Backup CR spec: + ```bash + velero backup create --resource-policies-configmap + ``` + Or in `Backup.spec`: + ```yaml + spec: + resourcePolicy: + kind: ConfigMap + name: + ``` + + * **For Restore:** Reference the ConfigMap via CLI flag or in the Restore CR spec: + ```bash + velero restore create --from-backup --resource-policies-configmap + ``` + Or in `Restore.spec`: + ```yaml + spec: + resourcePolicy: + kind: ConfigMap + name: + ``` + + These flags and fields can also be combined with standard include and exclude options. ### YAML template -The policies YAML config file would look like this: -- Yaml template: - ```yaml - # currently only supports v1 version - version: v1 - # The filters in includeExcludePolicy work the same as the scoped resources filters in the Spec of a Backup - # NOTE: similar to scoped filters in Backup Spec, the includeExcludePolicy does not work with --include-resources, --exclude-resources and --include-cluster-resources filters in Backup. - includeExcludePolicy: - includedClusterScopedResources: - - "crd" - - "pv" - excludedClusterScopedResources: [] - includedNamespaceScopedResources: - - "pod" - - "service" - - "deployment" - - "pvc" - excludedNamespaceScopedResources: - - "configmap" - - "secret" - volumePolicies: - # each policy consists of a list of conditions and an action - # we could have lots of policies, but if the resource matched the first policy, the latter will be ignored - # each key in the object is one condition, and one policy will apply to resources that meet ALL conditions - # NOTE: capacity or storageClass is suited for [Persistent Volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes), and pod [Volume](https://kubernetes.io/docs/concepts/storage/volumes) not support it. - - conditions: - # capacity condition matches the volumes whose capacity falls into the range - capacity: "10,100Gi" - # pv matches specific csi driver - csi: - driver: ebs.csi.aws.com - # pv matches one of the storage class list - storageClass: - - gp2 - - standard - # pvc matches specific phase(s) - pvcPhase: - - Pending - # pvc matches specific volume mode - pvcVolumeMode: Block - # pvc matches specific access mode(s) - pvcAccessModes: - - ReadWriteOnce - action: - type: skip - - conditions: - capacity: "0,100Gi" - # nfs volume source with specific server and path (nfs could be empty or only config server or path) - nfs: - server: 192.168.200.90 - path: /mnt/data - action: - type: skip - - conditions: - nfs: - server: 192.168.200.90 - action: - type: fs-backup - - conditions: - # nfs could be empty which matches any nfs volume source - nfs: {} - action: - type: skip - - conditions: - # csi could be empty which matches any csi volume source - csi: {} - action: - type: snapshot - - conditions: - volumeTypes: - - emptyDir - - downwardAPI - - configmap - - cinder - action: - type: skip - ``` -### IncludeExcludePolicy + +The policies YAML config file showing all supported sections: + +```yaml +# Currently supports v1 version +version: v1 + +# Fine-grained namespace-scoped filters (Supported for both Backup and Restore) +namespacedFilterPolicies: + - namespace: "app-ns-*" + resourceFilters: + - kind: "deployment" + labelSelector: + matchLabels: + app: frontend + includedResourceNames: + - "web-*" + - kind: "secret" + excludedResourceNames: + - "sensitive-secret" + +# Fine-grained cluster-scoped filter overlay (Supported for both Backup and Restore) +clusterScopedFilterPolicy: + resourceFilters: + - kind: "storageclass" + labelSelector: + matchLabels: + tier: gold + +# Volume handling policies (Supported for Backup ONLY) +volumePolicies: + - conditions: + capacity: "10,100Gi" + csi: + driver: ebs.csi.aws.com + storageClass: + - gp2 + - standard + pvcPhase: + - Pending + pvcVolumeMode: Block + pvcAccessModes: + - ReadWriteOnce + action: + type: skip + - conditions: + nfs: {} + action: + type: fs-backup + +# Legacy scoped resource include/exclude filters (Supported for Backup ONLY) +# NOTE: Cannot be combined with --include-resources, --exclude-resources, or --include-cluster-resources in Backup. +includeExcludePolicy: + includedClusterScopedResources: + - "crd" + - "pv" + excludedClusterScopedResources: [] + includedNamespaceScopedResources: + - "pod" + - "service" + - "deployment" + - "pvc" + excludedNamespaceScopedResources: + - "configmap" + - "secret" +``` + +### Fine-grained backup and restore filters + +`namespacedFilterPolicies` and `clusterScopedFilterPolicy` allow defining per-namespace and per-kind rules with independent label selectors and resource name patterns. + +* **During Backup:** Controls which resources are backed up from matching namespaces or kinds. +* **During Restore:** Controls which resources are restored from a backup archive without modifying the backup itself. + +For comprehensive guides, syntax details, and detailed examples, see: +* [Fine-Grained Backup Filters](fine-grained-backup-filters.md) +* [Fine-Grained Restore Filters](fine-grained-restore-filters.md) + +### IncludeExcludePolicy (Backup only) The `includeExcludePolicy` is used to filter resources based on the namespace-scoped and cluster-scoped resources. User can use it to define a group of filters and reuse them across different backups. @@ -365,7 +400,7 @@ velero backup create --resource-policies-configmap my-policy --inc The backup will include all resources in namespace `my-workload-ns`, including `configmap` and `event`, and all CRDs and `apiservices` in the cluster. -### VolumePolicy +### VolumePolicy (Backup only) VolumePolicy is a data structure to control how velero handle the volumes matching certain conditions. #### Supported VolumePolicy actions @@ -617,6 +652,7 @@ a volume policy but for a particular volume included in the backup there are no in such a scenario the legacy approach will be used for backing up the particular volume. Considering everything, the recommendation would be to use only one of the approaches to backup volumes - volume policy approach or the opt-in/opt-out legacy approach, and not mix them for clarity. - Snapshot action can either be a native snapshot or a csi snapshot or csi snapshot datamover, as is the case with the current flow where velero itself makes the decision based on the backup CR's existing options. +- The `snapshot` action supports an optional `snapshotClass` parameter that specifies which VolumeSnapshotClass to use for CSI snapshots. This is useful when multiple storage arrays share the same CSI driver but require different VolumeSnapshotClasses. When specified, this takes priority over backup annotations and VolumeSnapshotClass labels, but is overridden by PVC-level annotations. See the [CSI documentation](csi.md) for the full VolumeSnapshotClass selection priority order. - The `snapshot` action via Volume Policy has higher priority if there is a `snapshot` action matching for a particular volume, this volume would be backed up via snapshot irrespective of the value of `backup.Spec.SnapshotVolumes`. - If for a particular volume there is no `snapshot` matching action then the volume will be backed up via snapshot given that `backup.Spec.SnapshotVolumes` is not explicitly set to false. - Let's see some examples on how to use the volume policy feature for `fs-backup` and `snapshot` action purposes: @@ -705,6 +741,29 @@ volumePolicies: - `fs-backup` on `Volume 1` because `Volume 1` satisfies the criteria for `fs-backup` action. - Also, for Volume 2 as no matching action was found so legacy approach will be used as a fallback option for this volume (`fs-backup` operation will be done as `defaultVolumesToFSBackup: true` is specified by the user). +***Example 6: User has two storage arrays using the same CSI driver and needs different VolumeSnapshotClasses for each*** +1. User specifies the volume policy as follows: +```yaml +version: v1 +volumePolicies: +- conditions: + storageClass: + - array-1-sc + action: + type: snapshot + parameters: + snapshotClass: vsc-array-1 +- conditions: + storageClass: + - array-2-sc + action: + type: snapshot + parameters: + snapshotClass: vsc-array-2 +``` +2. User creates a backup using this volume policy +3. The outcome would be that velero would use `vsc-array-1` VolumeSnapshotClass for volumes on storage class `array-1-sc` and `vsc-array-2` VolumeSnapshotClass for volumes on storage class `array-2-sc`, even though both storage classes use the same CSI driver. + ### Global backup volume policies Resource policies (volume policies) are normally opt-in per backup via `--resource-policies-configmap`. An administrator can instead configure a cluster-wide baseline that applies to **every** backup by starting the Velero server with the `--global-backup-volume-policies-configmap` flag, pointing at a ConfigMap in the Velero install namespace: diff --git a/site/content/docs/main/restore-resource-modifiers.md b/site/content/docs/main/restore-resource-modifiers.md index 0c1f2f217..39248ad30 100644 --- a/site/content/docs/main/restore-resource-modifiers.md +++ b/site/content/docs/main/restore-resource-modifiers.md @@ -184,4 +184,47 @@ resourceModifierRules: ### Wildcard Support for GroupResource The user can specify a wildcard for groupResource in the conditions' struct. This will allow the user to apply the patches for all the resources of a particular group or all resources in all groups. For example, `*.apps` will apply to all the resources in the `apps` group, `*` will apply to all the resources in core group, `*.*` will apply to all the resources in all groups. -- If both `*.groupName` and `namespaces` are specified, the patches will be applied to all the namespaced resources in this group in the specified namespaces and all the cluster resources in this group. \ No newline at end of file +- If both `*.groupName` and `namespaces` are specified, the patches will be applied to all the namespaced resources in this group in the specified namespaces and all the cluster resources in this group. + +## Default Resource Modifiers + +Velero supports a server-level default resource modifier that applies automatically to all restores without requiring per-restore configuration. +This is useful for common transformations like stripping stale CNI annotations that can break workloads after restore. + +### Configuration + +1. Create a ConfigMap in the Velero namespace with your default resource modifier rules: + +```bash +kubectl apply -f examples/default-resource-modifier-cni.yaml +``` + +2. Configure the Velero server to use it, either during install: + +```bash +velero install --default-resource-modifier-configmap=default-restore-resource-modifiers ... +``` + +Or by editing an existing deployment: + +```bash +kubectl -n velero edit deploy velero +# Add to the server args: --default-resource-modifier-configmap=default-restore-resource-modifiers +``` + +### Precedence + +When a per-restore modifier is specified via `--resource-modifier-configmap`, it takes exclusive precedence and the default is not applied. + +### Opt-out + +To skip the default modifier for a specific restore without specifying a per-restore modifier: + +```bash +velero restore create --from-backup my-backup --skip-default-resource-modifier +``` + +### Error Handling + +If the default ConfigMap is missing or contains invalid data, Velero logs a warning and proceeds with the restore. +Per-restore modifier errors remain fatal and cause the restore to fail validation. \ No newline at end of file diff --git a/site/content/docs/main/self-signed-certificates.md b/site/content/docs/main/self-signed-certificates.md index 41eb8b247..87576dae2 100644 --- a/site/content/docs/main/self-signed-certificates.md +++ b/site/content/docs/main/self-signed-certificates.md @@ -150,7 +150,7 @@ Velero provides a way for you to skip TLS verification on the object store when * velero backup download * velero backup logs * velero restore describe -* velero restore log +* velero restore logs If true, the object store's TLS certificate will not be checked for validity before Velero or backup repository connects to the object storage. You can permanently skip TLS verification for an object store by setting `Spec.Config.InsecureSkipTLSVerify` to true in the [BackupStorageLocation](api-types/backupstoragelocation.md) CRD. diff --git a/site/content/docs/main/support-process.md b/site/content/docs/main/support-process.md index d142329f8..5c1363e7a 100644 --- a/site/content/docs/main/support-process.md +++ b/site/content/docs/main/support-process.md @@ -40,4 +40,4 @@ Generally speaking, new GitHub issues will fall into one of several categories. - If the issue ends up being a feature request or a bug, update the title and follow the appropriate process for it - If the reporter becomes unresponsive after multiple pings, close out the issue due to inactivity and comment that the user can always reach out again as needed -[0]: https://github.com/vmware-tanzu?q=velero&type=&language= +[0]: https://github.com/velero-io?q=velero&type=&language= diff --git a/site/content/docs/main/supported-configmaps/node-agent-configmap.md b/site/content/docs/main/supported-configmaps/node-agent-configmap.md index fc90d4436..f8f8f2c5c 100644 --- a/site/content/docs/main/supported-configmaps/node-agent-configmap.md +++ b/site/content/docs/main/supported-configmaps/node-agent-configmap.md @@ -297,6 +297,9 @@ For detailed information, see [BackupPVC Configuration for Data Movement Backup] - **`storageClass`**: Alternative storage class for backup PVCs (defaults to source PVC's storage class) - **`readOnly`**: This is a boolean value. If set to `true` then `ReadOnlyMany` will be the only value set to the backupPVC's access modes. Otherwise `ReadWriteOnce` value will be used. - **`spcNoRelabeling`**: This is a boolean value. If set to true, then `pod.Spec.SecurityContext.SELinuxOptions.Type` will be set to `spc_t`. From the SELinux point of view, this will be considered a `Super Privileged Container` which means that selinux enforcement will be disabled and volume relabeling will not occur. This field is ignored if `readOnly` is `false`. +- **`secretNames`**: List of secret names to copy from the source PVC's namespace to the Velero namespace before creating the backupPVC (deleted after the DataUpload completes). Needed for CSI drivers that require namespace-scoped secrets to provision the volume, e.g. ODF/ceph-csi encrypted volumes (`ceph-csi-kms-token`). +- **`configMapNames`**: List of configmap names to copy from the source PVC's namespace to the Velero namespace before creating the backupPVC (deleted after the DataUpload completes). Needed for CSI drivers that require namespace-scoped configmaps to provision the volume, e.g. a tenant ceph-csi KMS config (`ceph-csi-kms-config`). +- **`readWriteOncePod`**: This is a boolean value. If set to `true`, then `ReadWriteOncePod` will be the only value set to the backupPVC's access modes, so the kubelet labels the volume at mount time instead of relabeling every file. Requires a CSI driver with `seLinuxMount: true` and a storage class that supports `ReadWriteOncePod` PVCs from a snapshot. This field is ignored if `readOnly` is `true`. **Use Cases:** - Use read-only volumes for faster snapshot-to-volume conversion @@ -307,6 +310,7 @@ For detailed information, see [BackupPVC Configuration for Data Movement Backup] **Important Notes:** - Ensure specified storage classes exist and support required access modes - In SELinux environments, always set `spcNoRelabeling: true` when using `readOnly: true` +- In SELinux environments where the storage does not support `ReadOnlyMany`, use `readWriteOncePod: true` instead; it is ignored when `readOnly: true` is also set - Failures result in DataUpload CR staying in `Accepted` phase until timeout (30m default) #### Storage Class Mapping @@ -360,6 +364,8 @@ For detailed information, see [RestorePVC Configuration for Data Movement Restor #### Configuration Options - **`ignoreDelayBinding`**: Ignore `WaitForFirstConsumer` binding mode constraints +- **`secretNames`**: List of secret names to copy from the target (restore) namespace to the Velero namespace before creating the restorePVC (deleted after the DataDownload completes). Needed for CSI drivers that require namespace-scoped secrets to provision the volume, e.g. ODF/ceph-csi encrypted volumes (`ceph-csi-kms-token`). +- **`configMapNames`**: List of configmap names to copy from the target (restore) namespace to the Velero namespace before creating the restorePVC (deleted after the DataDownload completes). Needed for CSI drivers that require namespace-scoped configmaps to provision the volume, e.g. a tenant ceph-csi KMS config (`ceph-csi-kms-config`). **Use Cases:** - Improve restore parallelism by not waiting for pod scheduling diff --git a/site/content/docs/main/troubleshooting.md b/site/content/docs/main/troubleshooting.md index dc692771c..df5d71753 100644 --- a/site/content/docs/main/troubleshooting.md +++ b/site/content/docs/main/troubleshooting.md @@ -77,6 +77,19 @@ Here are some things to verify if you receive `SignatureDoesNotMatch` errors: * Make sure your S3-compatible layer is using [signature version 4][5] (such as Ceph RADOS v12.2.7) * For Ceph, try using a native Ceph account for credentials instead of external providers such as OpenStack Keystone +### `velero backup logs` or `velero describe` fails with `no such host` + +Downloading artifacts uses a pre-signed URL built from the `s3Url` in your `BackupStorageLocation`. If that address is only resolvable inside the cluster, such as a Kubernetes Service name, the Velero client cannot fetch the artifact even though the backup or restore itself succeeded: + +``` +Warnings: +``` + +The backup or restore is unaffected. Only the download of its log or results file fails. + +To fix this, give the location a `publicUrl` that your client can reach. See [Expose Minio outside your cluster][26] for the Minio case; the same applies to any object store addressed by an in-cluster name. + ## Velero (or a pod it was backing up) restarted during a backup and the backup is stuck InProgress Velero cannot resume backups that were interrupted. Backups stuck in the `InProgress` phase can be deleted with `kubectl delete backup -n `. @@ -250,3 +263,4 @@ Please refer to [Issue 9007](https://github.com/velero-io/velero/issues/9007) fo [11]: /plugins [12]: https://kubernetes.io/docs/concepts/configuration/secret/#editing-a-secret [25]: https://kubernetes.slack.com/messages/velero +[26]: contributions/minio.md diff --git a/site/content/docs/v1.18/data-movement-backup-pvc-configuration.md b/site/content/docs/v1.18/data-movement-backup-pvc-configuration.md index 9d48b0a5f..956d03997 100644 --- a/site/content/docs/v1.18/data-movement-backup-pvc-configuration.md +++ b/site/content/docs/v1.18/data-movement-backup-pvc-configuration.md @@ -40,6 +40,16 @@ The users can specify the ConfigMap name during velero installation by CLI: - `annotations`: permits to set annotations on the backupPVC itself. typically useful for some CSI provider which cannot mount a VolumeSnapshot without a custom annotation. +- `secretNames`: a list of secret names to copy from the source PVC's namespace to the Velero namespace before the backupPVC is + created, and delete after the DataUpload completes. This is needed for CSI drivers that require namespace-scoped secrets to + provision the volume, for example ODF/ceph-csi encrypted volumes that fetch a KMS token secret (`ceph-csi-kms-token`) from the + PVC's namespace. Without this, the backupPVC created in the Velero namespace fails to provision because the secret only exists + in the source namespace. + +- `configMapNames`: a list of configmap names to copy from the source PVC's namespace to the Velero namespace before the backupPVC + is created, and delete after the DataUpload completes. This is needed for CSI drivers that require namespace-scoped configmaps to + provision the volume, for example a tenant-specific ceph-csi KMS connection override configmap (`ceph-csi-kms-config`). + A sample of `backupPVC` config as part of the ConfigMap would look like: ```json { @@ -60,11 +70,21 @@ A sample of `backupPVC` config as part of the ConfigMap would look like: "storage-class-4": { "readOnly": true, "spcNoRelabeling": true + }, + "ocs-storagecluster-ceph-rbd-encrypted": { + "secretNames": ["ceph-csi-kms-token"], + "configMapNames": ["ceph-csi-kms-config"] } } } ``` +**Note on encrypted volumes:** the copied secrets/configmaps are labeled `velero.io/backup-pvc-secret=` and +deleted when the DataUpload completes (or on failure). If concurrent DataUploads from different namespaces need a secret with the same +name but different content in the Velero namespace, they conflict. For ceph-csi, +this can be avoided by configuring a unique +[`tenantTokenName` per tenant](https://github.com/ceph/ceph-csi/blob/devel/docs/design/proposals/encryption-with-vault-tokens.md#example-of-the-kms-configuration-file-for-vault-tokens). + **Note:** - Users should make sure that the storage class specified in `backupPVC` config should exist in the cluster and can be used by the `backupPVC`, otherwise the corresponding DataUpload CR will stay in `Accepted` phase until timeout (data movement prepare timeout value is 30m by default). diff --git a/site/content/docs/v1.18/data-movement-restore-pvc-configuration.md b/site/content/docs/v1.18/data-movement-restore-pvc-configuration.md index 1cb8fa14d..1728d346e 100644 --- a/site/content/docs/v1.18/data-movement-restore-pvc-configuration.md +++ b/site/content/docs/v1.18/data-movement-restore-pvc-configuration.md @@ -12,6 +12,10 @@ Velero introduces a new section in the node agent configuration ConfigMap (the n - `ignoreDelayBinding`: If this flag is set, the data movement restore will ignore the delay binding requirements from `WaitForFirstConsumer` mode, create the restore pod and provision the volume associated to an arbitrary node. When multiple volume restores happen in parallel, the restore pods will be spread evenly to all the nodes. +- `secretNames`: a list of secret names to copy from the target (restore) namespace to the Velero namespace before the restorePVC is created, and delete after the DataDownload completes. This is needed for CSI drivers that require namespace-scoped secrets to provision the volume, for example ODF/ceph-csi encrypted volumes that fetch a KMS token secret (`ceph-csi-kms-token`) from the PVC's namespace. Without this, the restorePVC created in the Velero namespace fails to provision because the secret only exists in the target namespace. + +- `configMapNames`: a list of configmap names to copy from the target (restore) namespace to the Velero namespace before the restorePVC is created, and delete after the DataDownload completes. This is needed for CSI drivers that require namespace-scoped configmaps to provision the volume, for example a tenant-specific ceph-csi KMS connection override configmap (`ceph-csi-kms-config`). + The users can specify the ConfigMap name during velero installation by CLI: `velero install --node-agent-configmap=` @@ -20,11 +24,15 @@ A sample of `restorePVC` config as part of the ConfigMap would look like: ```json { "restorePVC": { - "ignoreDelayBinding": true + "ignoreDelayBinding": true, + "secretNames": ["ceph-csi-kms-token"], + "configMapNames": ["ceph-csi-kms-config"] } } ``` +**Note on encrypted volumes:** unlike `backupPVC` (which is keyed per source storage class), `restorePVC` is a single config that applies to all restore PVCs. The copied secrets/configmaps are labeled `velero.io/backup-pvc-secret=` and deleted when the DataDownload completes (or on failure). + **Note:** - If `ignoreDelayBinding` is set, the restored volume is provisioned in the storage areas associated to an arbitrary node, if the restored pod cannot be scheduled to that node, e.g., because of topology constraints, the data mover restore still completes, but the workload is not usable since the restored pod cannot mount the restored volume - At present, node selection is not supported for data mover restore, so the restored volume may be attached to any node in the cluster; once node selection is supported and enabled, the restored volume will be attached to one of the selected nodes only. In this way, node selection and `ignoreDelayBinding` can work together even though the environment is with topology constraints diff --git a/site/content/docs/v1.18/supported-configmaps/node-agent-configmap.md b/site/content/docs/v1.18/supported-configmaps/node-agent-configmap.md index 0062b0391..4ffea44cb 100644 --- a/site/content/docs/v1.18/supported-configmaps/node-agent-configmap.md +++ b/site/content/docs/v1.18/supported-configmaps/node-agent-configmap.md @@ -295,6 +295,8 @@ For detailed information, see [BackupPVC Configuration for Data Movement Backup] - **`storageClass`**: Alternative storage class for backup PVCs (defaults to source PVC's storage class) - **`readOnly`**: This is a boolean value. If set to `true` then `ReadOnlyMany` will be the only value set to the backupPVC's access modes. Otherwise `ReadWriteOnce` value will be used. - **`spcNoRelabeling`**: This is a boolean value. If set to true, then `pod.Spec.SecurityContext.SELinuxOptions.Type` will be set to `spc_t`. From the SELinux point of view, this will be considered a `Super Privileged Container` which means that selinux enforcement will be disabled and volume relabeling will not occur. This field is ignored if `readOnly` is `false`. +- **`secretNames`**: List of secret names to copy from the source PVC's namespace to the Velero namespace before creating the backupPVC (deleted after the DataUpload completes). Needed for CSI drivers that require namespace-scoped secrets to provision the volume, e.g. ODF/ceph-csi encrypted volumes (`ceph-csi-kms-token`). +- **`configMapNames`**: List of configmap names to copy from the source PVC's namespace to the Velero namespace before creating the backupPVC (deleted after the DataUpload completes). Needed for CSI drivers that require namespace-scoped configmaps to provision the volume, e.g. a tenant ceph-csi KMS config (`ceph-csi-kms-config`). **Use Cases:** - Use read-only volumes for faster snapshot-to-volume conversion @@ -358,6 +360,8 @@ For detailed information, see [RestorePVC Configuration for Data Movement Restor #### Configuration Options - **`ignoreDelayBinding`**: Ignore `WaitForFirstConsumer` binding mode constraints +- **`secretNames`**: List of secret names to copy from the target (restore) namespace to the Velero namespace before creating the restorePVC (deleted after the DataDownload completes). Needed for CSI drivers that require namespace-scoped secrets to provision the volume, e.g. ODF/ceph-csi encrypted volumes (`ceph-csi-kms-token`). +- **`configMapNames`**: List of configmap names to copy from the target (restore) namespace to the Velero namespace before creating the restorePVC (deleted after the DataDownload completes). Needed for CSI drivers that require namespace-scoped configmaps to provision the volume, e.g. a tenant ceph-csi KMS config (`ceph-csi-kms-config`). **Use Cases:** - Improve restore parallelism by not waiting for pod scheduling diff --git a/site/content/posts/2025-09-15-Velero-1.17.md b/site/content/posts/2025-09-15-Velero-1.17.md new file mode 100644 index 000000000..88394f3d6 --- /dev/null +++ b/site/content/posts/2025-09-15-Velero-1.17.md @@ -0,0 +1,99 @@ +--- +title: "Velero 1.17: Volume Group Snapshots, Modernized fs-backup, and Windows Support" +excerpt: Velero 1.17 introduces VolumeGroupSnapshot support for crash-consistent multi-volume backups, a modernized fs-backup architecture, Windows cluster support, and significant scalability improvements for data movers. +author_name: Shubham Pampattiwar +slug: Velero-1.17 +categories: ['velero','release'] +image: /img/posts/post-1.17.jpg +tags: ['Velero Team', 'Shubham Pampattiwar', 'Velero Release'] +--- + +We are pleased to announce the release of [Velero v1.17](https://github.com/velero-io/velero/releases/tag/v1.17.0). This is a feature-rich release that delivers volume group snapshot support, a modernized fs-backup architecture, Windows workload backup/restore, and major scalability improvements for data movers. + +### Full list of changes can be found [here](https://github.com/velero-io/velero/releases/tag/v1.17.0) + +## Release Highlights + +### Volume Group Snapshot Support + +Velero 1.17 supports [volume group snapshots](https://kubernetes.io/blog/2024/12/18/kubernetes-1-32-volume-group-snapshot-beta/), a beta feature in Kubernetes, for both CSI snapshot backup and CSI snapshot data movement. This allows snapshots to be taken from multiple volumes at the same point-in-time to achieve write order consistency, which is important for achieving better data consistency when multiple correlated volumes are backed up together. + +See the [documentation](https://velero.io/docs/v1.17/volume-group-snapshots/) for details. + +### Modernized fs-backup + +The fs-backup subsystem has been rebuilt on the micro-service architecture, bringing several benefits: + +- **Feature parity**: Load concurrency control, cancel, and resume on restart are now available for fs-backup. +- **Improved robustness**: Running backups and restores survive node-agent restarts. Resource allocation is more granular, so the failure of one backup/restore does not impact others. +- **Steady resource usage**: Node-agent pods no longer request large amounts of memory and hold it for extended periods. + +See the [design document](https://github.com/vmware-tanzu/velero/tree/v1.17.0/design/Implemented/vgdp-micro-service-for-fs-backup/vgdp-micro-service-for-fs-backup.md) for details. + +### Windows Cluster Support for fs-backup + +Velero fs-backup now supports backing up and restoring Windows workloads. By leveraging the new micro-service architecture, data mover pods can run on Windows nodes and handle Windows volumes. Together with CSI snapshot data movement for Windows delivered in v1.16, Velero now supports Windows workload backup/restore across all scenarios. + +### Priority Class Support + +[Kubernetes priority classes](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass) are now supported across all Velero modules. Users can configure priority classes separately for Velero server, node-agent, data mover pods, and backup repository maintenance jobs. + +See the [design document](https://github.com/vmware-tanzu/velero/tree/v1.17.0/design/Implemented/priority-class-name-support_design.md) for details. + +### Include/Exclude Policy for Resource Policy + +Velero resource policy now supports `includeExcludePolicy` alongside the existing `volumePolicy`. This allows users to set include/exclude filters for resources in a resource policy configmap, making these filters reusable across multiple backups. + +## Scalability and Resiliency Improvements + +### Reduced Data Mover Pod Congestion + +A new `PrepareQueueLength` setting in node-agent configuration limits how many data mover pods and volumes are created ahead of available data path quota. This prevents excessive cluster resource consumption, particularly helpful in large-scale environments. This applies to both fs-backup and CSI snapshot data movement. + +See the [design document](https://github.com/vmware-tanzu/velero/tree/v1.17.0/design/Implemented/node-agent-load-soothing.md) for details. + +### Enhanced Node-Agent Restart Handling + +Data movements in all phases now survive node-agent restarts and resume automatically. Orphaned data movements from scenarios like cluster node absence are canceled appropriately after restart. + +### Restore Node-Selection for CSI Snapshot Data Movement + +CSI snapshot data movement restore now has the same node-selection capability as backup. Users can specify which nodes can or cannot run data mover pods for both backup and restore, with per-storage-class configuration for environments where a storage class is not usable by all cluster nodes. + +## Breaking Changes + +### Deprecation of Restic + +Per the [Velero deprecation policy](https://github.com/vmware-tanzu/velero/tree/v1.17.0/GOVERNANCE.md#deprecation-policy), backup under the Restic path is removed in v1.17. `--uploader-type=restic` is no longer a valid installation configuration. Restores from previous Restic-path backups remain supported until v1.19. + +### Repository Maintenance Job Configuration + +Repository maintenance job configurations have been moved from Velero server parameters to a repository maintenance job configmap. The following server parameters are removed: `--keep-latest-maintenance-jobs`, `--maintenance-job-cpu-request`, `--maintenance-job-mem-request`, `--maintenance-job-cpu-limit`, `--maintenance-job-mem-limit`. + +## Community Contributions + +Thank you to everyone who contributed to this release: + +- [@Lyndon-Li](https://github.com/Lyndon-Li) -- modernized fs-backup, Windows support, data mover scalability, node-agent restart handling +- [@blackpiglet](https://github.com/blackpiglet) -- configmap validation, maintenance job improvements, VolumeSnapshot cleanup +- [@shubham-pampattiwar](https://github.com/shubham-pampattiwar) -- VolumeGroupSnapshot support, VGS documentation, maintenance job configmap, VGS PVC plugin +- [@sseago](https://github.com/sseago) -- hook tracking improvements +- [@kaovilai](https://github.com/kaovilai) -- priority class restore ordering, ResticIdentifier fix +- [@reasonerjt](https://github.com/reasonerjt) -- include/exclude resource policy, BSL availability metrics +- [@ywk253100](https://github.com/ywk253100) -- server version check improvements +- [@priyansh17](https://github.com/priyansh17) -- context-based logging, Azure credential cleanup +- [@amastbau](https://github.com/amastbau) -- label selector restore fix +- [@longxiucai](https://github.com/longxiucai) -- parameterized kubelet mount path +- [@farodin91](https://github.com/farodin91) -- bug fixes +- [@flx5](https://github.com/flx5) -- bug fixes +- [@hu-keyu](https://github.com/hu-keyu) -- bug fixes +- [@pandurangkhandeparker](https://github.com/pandurangkhandeparker) -- bug fixes +- [@vishal-chdhry](https://github.com/vishal-chdhry) -- bug fixes + +## Join the Community + +- **Slack**: [#velero-users](https://kubernetes.slack.com/messages/velero) and [#velero-dev](https://kubernetes.slack.com/messages/velero-dev) on Kubernetes Slack +- **GitHub**: [github.com/velero-io/velero](https://github.com/velero-io/velero) +- **Community Meetings**: Bi-weekly, alternating US/Europe and US/Asia time zones. See the [community page](https://velero.io/community/) for details. +- **LinkedIn**: [Project Velero](https://www.linkedin.com/company/project-velero) +- **Twitter/X**: [@projectvelero](https://twitter.com/projectvelero) diff --git a/site/content/posts/2026-03-06-Velero-1.18.md b/site/content/posts/2026-03-06-Velero-1.18.md new file mode 100644 index 000000000..6a27372a8 --- /dev/null +++ b/site/content/posts/2026-03-06-Velero-1.18.md @@ -0,0 +1,93 @@ +--- +title: "Velero 1.18: Concurrent Backups, Cache Volumes, and More" +excerpt: Velero 1.18 introduces concurrent backup processing, cache volume support for data movers, incremental backup size reporting, and several scalability and performance improvements. +author_name: Shubham Pampattiwar +slug: Velero-1.18 +categories: ['velero','release'] +image: /img/posts/post-1.18.jpg +tags: ['Velero Team', 'Shubham Pampattiwar', 'Velero Release'] +--- + +We are pleased to announce the release of [Velero v1.18](https://github.com/velero-io/velero/releases/tag/v1.18.0). This release brings significant improvements in concurrency, performance, and observability, with contributions from engineers across multiple organizations. + +### Full list of changes can be found [here](https://github.com/velero-io/velero/releases/tag/v1.18.0) + +## Release Highlights + +### Concurrent Backup Processing + +Velero can now process multiple backups concurrently. This is a major usability improvement for multi-tenant environments -- backups submitted by different users or teams run simultaneously without interfering with each other. + +Previously, backups were serialized, meaning a long-running backup would block all other pending backups. With concurrent processing, backup throughput scales with available resources. + +See the [design document](https://github.com/vmware-tanzu/velero/blob/main/design/Implemented/concurrent-backup-processing.md) for details. + +### Cache Volume Support for Data Movers + +Velero 1.18 allows users to configure cache volumes for data mover pods during restore operations for both CSI snapshot data movement and fs-backup. This solves several real-world problems: + +- Data mover pods failing when a pod's ephemeral disk is limited +- Multiple data mover pods failing to run concurrently on a single node due to disk constraints +- Combined with backup repository cache limit configuration, appropriately sized cache volumes improve restore throughput + +See the [design document](https://github.com/vmware-tanzu/velero/blob/main/design/Implemented/backup-repo-cache-volume.md) for details. + +### Incremental Backup Size Reporting + +Users can now observe the incremental size of data mover backups for CSI snapshot data movement and fs-backup. This provides visibility into data reduction from incremental backups, helping teams understand and optimize their backup storage usage. + +### Wildcard Namespace Filtering + +Velero now supports Glob regular expressions for namespace filters during backup and restore. This allows users to filter namespaces in batch -- for example, backing up all namespaces matching `team-*` or excluding `test-*` namespaces. + +### VolumePolicy Enhancements + +VolumePolicy receives two improvements in this release: + +- **PVC Phase support**: Users can now filter volumes by PVC phase, enabling actions like skipping PVCs in Pending or Lost status from backups to avoid failures caused by unbound volumes. +- **VolumeGroupSnapshot integration**: Volume policies now apply to VolumeGroupSnapshot PVC filtering, building on the VolumeGroupSnapshot support introduced in v1.17. + +## Scalability and Resiliency + +### Prevent Velero Server OOM for Large Backup Repositories + +Some backup repository operations are now executed outside the Velero server process, preventing OOM kills when working with large repositories. + +### VolumePolicy Performance + +VolumePolicy evaluation has been optimized for environments with large numbers of pods and PVCs, resulting in significantly improved performance through a PVC-to-Pod cache that avoids redundant lookups. + +### Events for Data Mover Pod Diagnostics + +Events are now recorded in data mover pod diagnostics, giving users more information for troubleshooting when data mover pods fail. + +## Breaking Changes + +### Deprecation of PVC Selected Node Feature + +Per the [Velero deprecation policy](https://github.com/vmware-tanzu/velero/blob/main/GOVERNANCE.md#deprecation-policy), the PVC selected node feature is deprecated in v1.18. Velero now handles PVC selected-node annotations automatically, so no user action is required. + +## Community Contributions + +This release includes contributions from across the Velero community. Thank you to everyone who contributed: + +- [@sseago](https://github.com/sseago) -- concurrent backup processing, incremental size reporting +- [@Lyndon-Li](https://github.com/Lyndon-Li) -- cache volume support, data mover diagnostics +- [@blackpiglet](https://github.com/blackpiglet) -- maintenance job improvements, restore ordering fixes +- [@shubham-pampattiwar](https://github.com/shubham-pampattiwar) -- VolumePolicy performance, VolumeGroupSnapshot filtering, Prometheus metrics +- [@kaovilai](https://github.com/kaovilai) -- BSL secret-based CA certificate support +- [@mpryc](https://github.com/mpryc) -- plugin init container DNS fix +- [@mjnagel](https://github.com/mjnagel) -- install command `--apply` flag +- [@Joeavaikath](https://github.com/Joeavaikath) -- backup label cleanup +- [@0xLeo258](https://github.com/0xLeo258) -- VolumeSnapshotter cache concurrency control +- [@clementnuss](https://github.com/clementnuss) -- bug fixes +- [@priyansh17](https://github.com/priyansh17) -- backend improvements +- [@T4iFooN-IX](https://github.com/T4iFooN-IX) -- documentation fixes + +## Join the Community + +- **Slack**: [#velero-users](https://kubernetes.slack.com/messages/velero) and [#velero-dev](https://kubernetes.slack.com/messages/velero-dev) on Kubernetes Slack +- **GitHub**: [github.com/velero-io/velero](https://github.com/velero-io/velero) +- **Community Meetings**: Bi-weekly, alternating US/Europe and US/Asia time zones. See the [community page](https://velero.io/community/) for details. +- **LinkedIn**: [Project Velero](https://www.linkedin.com/company/project-velero) +- **Twitter/X**: [@projectvelero](https://twitter.com/projectvelero) diff --git a/site/content/posts/2026-07-30-Velero-Joins-CNCF-Sandbox.md b/site/content/posts/2026-07-30-Velero-Joins-CNCF-Sandbox.md new file mode 100644 index 000000000..cf5d0f61d --- /dev/null +++ b/site/content/posts/2026-07-30-Velero-Joins-CNCF-Sandbox.md @@ -0,0 +1,69 @@ +--- +title: "Velero Joins the CNCF Sandbox" +excerpt: Velero has been accepted into the Cloud Native Computing Foundation as a Sandbox project, bringing Kubernetes-native backup and disaster recovery under vendor-neutral, community-driven governance. +author_name: Shubham Pampattiwar +slug: Velero-Joins-CNCF-Sandbox +categories: ['velero','announcements'] +image: /img/cncf-color.svg +tags: ['Velero Team', 'Shubham Pampattiwar', 'CNCF'] +--- + +![CNCF Logo](/img/cncf-color.svg) + +We are excited to announce that Velero has been accepted into the [Cloud Native Computing Foundation (CNCF)](https://www.cncf.io/) as a Sandbox project. This marks a significant milestone for the project, placing Velero under vendor-neutral, community-driven governance alongside other foundational cloud native tools. + +The CNCF Technical Oversight Committee (TOC) accepted the [Sandbox application](https://github.com/cncf/sandbox/issues/457), and the transition was formally announced at KubeCon + CloudNativeCon Europe 2026 in Amsterdam. + +## What This Means + +Joining the CNCF Sandbox means Velero is now governed by the same open, vendor-neutral principles that guide projects like Kubernetes, Prometheus, and Envoy. In practice, this means: + +- **Vendor-neutral governance**: No single company controls the project roadmap. Decisions are made through consensus-based processes with supermajority voting. +- **Community-driven development**: The project's direction is shaped by its maintainers and contributors, who represent multiple organizations. +- **Long-term sustainability**: CNCF provides a neutral home that ensures the project's continuity regardless of changes in any single company's priorities. + +For existing Velero users, nothing changes in how you use the tool. Velero continues to operate at the Kubernetes API layer, providing backup, restore, disaster recovery, and migration capabilities for your clusters and applications. + +## Our Journey + +Velero's journey began at Heptio, the Kubernetes company founded by Joe Beda and Craig McLuckie, where it was originally known as Ark. After VMware acquired Heptio in 2019, the project continued to grow under VMware's stewardship. Following Broadcom's acquisition of VMware, the decision was made to contribute Velero to the CNCF, ensuring the project's future under community governance. + +Throughout these transitions, one thing has remained constant: a growing and engaged open source community. Today, Velero has over 10,000 GitHub stars, 1,500+ forks, 500M+ Docker Hub pulls, and is used by organizations across industries for Kubernetes data protection. + +We are grateful to Broadcom for contributing Velero to the CNCF and to everyone who has contributed to the project over the years. + +## Current Maintainers + +Velero is maintained by engineers from multiple organizations, reflecting the project's vendor-neutral nature: + +| Maintainer | GitHub | Affiliation | +|---|---|---| +| Daniel Jiang | [@reasonerjt](https://github.com/reasonerjt) | Broadcom | +| Wenkai Yin | [@ywk253100](https://github.com/ywk253100) | Broadcom | +| Xun Jiang | [@blackpiglet](https://github.com/blackpiglet) | Broadcom | +| Yonghui Li | [@Lyndon-Li](https://github.com/Lyndon-Li) | Broadcom | +| Scott Seago | [@sseago](https://github.com/sseago) | Red Hat (OpenShift) | +| Shubham Pampattiwar | [@shubham-pampattiwar](https://github.com/shubham-pampattiwar) | Red Hat (OpenShift) | +| Tiger Kaovilai | [@kaovilai](https://github.com/kaovilai) | Red Hat (OpenShift) | +| Anshul Ahuja | [@anshulahuja98](https://github.com/anshulahuja98) | Microsoft (Azure) | + +## What's Next + +Joining the CNCF Sandbox is the beginning of a new chapter for Velero. Here is what we are focused on: + +- **Growing the community**: We want more contributors, more adopters, and more voices shaping the project's direction. Whether you are a user, operator, or developer, there is a place for you in the Velero community. +- **Strengthening the project**: We are continuing to improve Velero's core capabilities around backup performance, data protection, and ecosystem integration. +- **Path to Incubation**: Our goal is to demonstrate the community health, adoption, and maturity needed to advance to CNCF Incubation status. + +## Get Involved + +We welcome contributions of all kinds -- code, documentation, bug reports, feature requests, and feedback. + +- **Slack**: Join [#velero-users](https://kubernetes.slack.com/messages/velero) and [#velero-dev](https://kubernetes.slack.com/messages/velero-dev) on Kubernetes Slack +- **GitHub**: [github.com/velero-io/velero](https://github.com/velero-io/velero) +- **Community Meetings**: We hold bi-weekly community meetings alternating between US/Europe and US/Asia-friendly time zones. See the [community page](https://velero.io/community/) for details. +- **LinkedIn**: Follow us at [Project Velero](https://www.linkedin.com/company/project-velero) +- **Twitter/X**: [@projectvelero](https://twitter.com/projectvelero) +- **Contributing**: Check out our [contribution guide](https://velero.io/docs/main/start-contributing/) to get started. + +We are excited about this new chapter and look forward to building the future of Kubernetes data protection together with the community. diff --git a/site/content/resources/_index.md b/site/content/resources/_index.md index 5f05a811a..2a3e6217a 100644 --- a/site/content/resources/_index.md +++ b/site/content/resources/_index.md @@ -3,7 +3,35 @@ title: Resources description: Velero Resources id: resources --- -Here you will find external resources about Velero, such as videos, podcasts, and community articles. +Here you will find external resources about Velero, including conference talks, videos, podcasts, and community articles. + +## Conference Talks + +### KubeCon + CloudNativeCon + +* **KubeCon EU 2026 (Amsterdam)** - [Snapshots Gone Wild: Taming Multi-PVC Chaos with VolumeGroupSnapshot](https://kccnceu2026.sched.com/event/2CW53/snapshots-gone-wild-taming-multi-pvc-chaos-with-volumegroupsnapshot-shubham-pampattiwar-scott-seago-red-hat) - Shubham Pampattiwar & Scott Seago, Red Hat + + {{< youtube pLmRkRO6O6E >}} + +* **KubeCon India 2026 (Mumbai)** - [Sponsored Demo: Cloud Native AI: Model Management with Harbor & Velero](https://kccncind2026.sched.com/event/2OdTx/sponsored-demo-cloud-native-ai-model-management-with-harbor-velero-dhruv-tyagi-broadcom) - Dhruv Tyagi, Broadcom + +* **KubeCon China 2024 (Hong Kong)** - [The Challenges of Kubernetes Data Protection - Real Examples and Solutions with Velero](https://kccncossaidevchn2024.sched.com/event/1eYb8/the-challenges-of-kubernetes-data-protection-real-examples-and-solutions-with-velero-kuberneteszha-velerozha-kang-reji-wenkai-yin-broadcom-bruce-zou-shanghai-jibu-tech) - Wenkai Yin, Broadcom & Bruce Zou, Shanghai Jibu Tech + +* **KubeCon EU 2023 (Amsterdam)** - [Disaster Recovery: Bringing Back Production from Scratch in Under 1 Hour Using KOps, ArgoCD and Velero](https://kccnceu2023.sched.com/event/1Hye8/disaster-recovery-bringing-back-production-from-scratch-in-under-1-hour-using-kops-argocd-and-velero-andre-jay-marcelo-tanner-ada-support) - Andre Jay Marcelo-Tanner, Ada Support + + {{< youtube oPQW99NiV_0 >}} + +### Open Source Summit + +* **Open Source Summit NA 2022 (Austin)** - [Velero - The Cloud Native Backup for Kubernetes](https://ossna2022.sched.com/event/11Nu9) - Orlin Vasilev, VMware & Scott Seago, Red Hat + + {{< youtube DKMW69OSI7c >}} + +### DevConf + +* **DevConf.IN 2025** - From Chaos to Control: Mastering Kubernetes Backups and Restore with Velero - Aziza Karol & Prasad Joshi + + {{< youtube Bo4lSle0J7k >}} ## All community meetings diff --git a/site/data/docs/main-toc.yml b/site/data/docs/main-toc.yml index 271705a1b..82f4bbd8f 100644 --- a/site/data/docs/main-toc.yml +++ b/site/data/docs/main-toc.yml @@ -33,6 +33,10 @@ toc: url: /enable-api-group-versions-feature - page: Resource filtering url: /resource-filtering + - page: Fine-Grained Backup Filters + url: /fine-grained-backup-filters + - page: Fine-grained restore filters + url: /fine-grained-restore-filters - page: Namespace glob patterns url: /namespace-glob-patterns - page: Backup reference diff --git a/site/static/img/cncf-color.svg b/site/static/img/cncf-color.svg new file mode 100644 index 000000000..6ed428836 --- /dev/null +++ b/site/static/img/cncf-color.svg @@ -0,0 +1,88 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/site/static/img/posts/post-1.17.jpg b/site/static/img/posts/post-1.17.jpg new file mode 100644 index 000000000..93f4b9ad5 Binary files /dev/null and b/site/static/img/posts/post-1.17.jpg differ diff --git a/site/static/img/posts/post-1.18.jpg b/site/static/img/posts/post-1.18.jpg new file mode 100644 index 000000000..0ae4f7ffd Binary files /dev/null and b/site/static/img/posts/post-1.18.jpg differ diff --git a/test/Makefile b/test/Makefile index ae58e2c95..4f051ae00 100644 --- a/test/Makefile +++ b/test/Makefile @@ -48,6 +48,7 @@ GOBIN := $(REPO_ROOT)/.go/bin TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR) GINKGO := $(GOBIN)/ginkgo +GINKGO_VERSION := $(shell go list -m -f '{{.Version}}' github.com/onsi/ginkgo/v2 2>/dev/null) KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize @@ -186,7 +187,7 @@ ginkgo: ${GOBIN}/ginkgo # This target does not run if ginkgo is already in $GOBIN ${GOBIN}/ginkgo: - GOBIN=${GOBIN} go install github.com/onsi/ginkgo/v2/ginkgo@v2.22.0 + GOBIN=${GOBIN} go install github.com/onsi/ginkgo/v2/ginkgo@${GINKGO_VERSION} .PHONY: run-e2e run-e2e: ginkgo diff --git a/test/e2e/README.md b/test/e2e/README.md index a621ee70c..2120e8b55 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -28,7 +28,7 @@ These are the current set of limitations with the E2E tests. 1. Flag `-install-velero` is for purpose of having tests on an existed Velero instance, but by default `-install-velero` is set to true, because it's mandatory for some of cases to testing on specific version of Velero, such as upgrade and migration tests. In upgrade tests, we must install a specific old version and then upgrade it to the target version, multiple installations is involved here, also migration tests have the same situation with upgrade tests, therefore if you're going to test against an existed Velero instance, make sure to skip upgrade and migration tests from a single E2E test execution. 1. To improve E2E test execution efficiency, E2E tests will skip re-installation between test cases except for those which need a fresh Velero installation like upgrade , migration and some other test cases. When starting a E2E test execution which setting flag `-install-velero` with the default value(true), there will be a Velero installation at the beginning, then test cases will be run in random order, and test cases behavior is as below: 1. If the scheduled test case is upgrade (or other cases needs a fresh Velero installation), then upgrade test will uninstall the current Velero instance at the beginning and uninstall the tested Velero instance in the end to avoid unexpected installation parameters for the following test cases. - 1. If the scheduled test case is the normal one, it will check the existence of Velero instance, if no one there then start a new standard instaillation, otherwise proceeding test steps. + 1. If the scheduled test case is the normal one, it will check the existence of Velero instance, if no one there then start a new standard installation, otherwise proceeding test steps. ## 3. Configuration for E2E tests @@ -113,10 +113,10 @@ Below is a mapping between `make` variables to E2E configuration flags. 1. `MIGRATE_FROM_VELERO_VERSION `: `-migrate-from-velero-version`. Optional. 1. `ADDITIONAL_BSL_PLUGINS `: `-additional-bsl-plugins`. Optional. 1. `ADDITIONAL_OBJECT_STORE_PROVIDER`: `-additional-bsl-object-store-provider`. Optional. -1. `ADDITIONAL_CREDS_FILE`: `-additional-bsl-bucket`. Optional. -1. `ADDITIONAL_BSL_BUCKET`: `-additional-bsl-prefix`. Optional. -1. `ADDITIONAL_BSL_PREFIX`: `-additional-bsl-config`. Optional. -1. `ADDITIONAL_BSL_CONFIG`: `-additional-bsl-credentials-file`. Optional. +1. `ADDITIONAL_CREDS_FILE`: `-additional-bsl-credentials-file`. Optional. +1. `ADDITIONAL_BSL_BUCKET`: `-additional-bsl-bucket`. Optional. +1. `ADDITIONAL_BSL_PREFIX`: `-additional-bsl-prefix`. Optional. +1. `ADDITIONAL_BSL_CONFIG`: `-additional-bsl-config`. Optional. 1. `FEATURES`: `-features`. Optional. 1. `REGISTRY_CREDENTIAL_FILE`: `-registry-credential-file`. Optional. 1. `KIBISHII_DIRECTORY`: `-kibishii-directory`. Optional. @@ -127,14 +127,14 @@ Below is a mapping between `make` variables to E2E configuration flags. 1. `SNAPSHOT_MOVE_DATA`: `-snapshot-move-data`. Optional. 1. `DATA_MOVER_plugin`: `-data-mover-plugin`. Optional. 1. `STANDBY_CLUSTER_CLOUD_PROVIDER`: `-standby-cluster-cloud-provider`. Optional. -1. `STANDBY_CLUSTER_PLUGINS`: `-dstandby-cluster-plugins`. Optional. +1. `STANDBY_CLUSTER_PLUGINS`: `-standby-cluster-plugins`. Optional. 1. `STANDBY_CLUSTER_OBJECT_STORE_PROVIDER`: `-standby-cluster-object-store-provider`. Optional. 1. `INSTALL_VELERO `: `-install-velero`. Optional. 1. `DEBUG_VELERO_POD_RESTART`: `-debug-velero-pod-restart`. Optional. 1. `FAIL_FAST`: `--fail-fast`. Optional. 1. `HAS_VSPHERE_PLUGIN`: `--has-vsphere-plugin`. Optional. 1. `WORKER_OS`: `--worker-os`. Optional. -1. `IMAGE_REGISTRY_PROXY`: `--image-registry-proxy.` Optional. +1. `IMAGE_REGISTRY_PROXY`: `--image-registry-proxy` Optional. ### Examples @@ -270,7 +270,7 @@ OBJECT_STORE_PROVIDER=aws \ CREDS_FILE= \ BSL_CONFIG=region= \ BSL_BUCKET= \ -BSL_PREFIX= \ +BSL_PREFIX= \ VSL_CONFIG=region= \ SNAPSHOT_MOVE_DATA=true \ STANDBY_CLUSTER_CLOUD_PROVIDER=aws \ @@ -369,7 +369,7 @@ there're some tests need to be run in a single execution or pipeline with specif Following pipelines should cover all E2E tests along with proper filters: 1. **CSI pipeline:** As we can see lots of labels in E2E test code, there're many snapshot-labeled test scripts. To cover CSI scenario, a pipeline with CSI enabled should be a good choice, otherwise, we will double all the snapshot cases for CSI scenario, it's very time-wasting. By providing `FEATURES=EnableCSI` and `PLUGINS=`, a CSI pipeline is ready for testing. -1. **Data mover pipeline:** Data mover scenario is the same scenario with migaration test except the restriction of migaration between different providers, so it better to separated it out from other pipelines. Please refer the example in previous. +1. **Data mover pipeline:** Data mover scenario is the same scenario with migration test except the restriction of migration between different providers, so it better to separated it out from other pipelines. Please refer the example in previous. 1. **File system backup pipeline:** Set `UPLOADER_TYPE` to `kopia` for all file system backup test cases; 1. **Long time pipeline:** Long time cases should be group into one pipeline, currently these test cases with labels `Scale`, `Schedule` or `TTL` can be group into a pipeline, and make sure to skip them off in any other pipelines. @@ -381,7 +381,7 @@ Following pipelines should cover all E2E tests along with proper filters: When adding a test, aim to instantiate an API client only once at the beginning of the test. There is a constructor `newTestClient` that facilitates the configuration and instantiation of clients. Also, please use the `kubebuilder` runtime controller client for any new test, as we will phase out usage of `client-go` API clients. ## 8. TestCase frame related -TestCase frame provide a serials of interface to concatenate one complete e2e test. it's makes the testing be concise and explicit. +TestCase frame provide a series of interfaces to concatenate one complete e2e test. it makes the testing be concise and explicit. ### VeleroBackupRestoreTest interface VeleroBackupRestoreTest interface provided a standard workflow of backup and restore, which makes the whole testing process clearer and code reusability. diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index a8bbbce4c..57121c378 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -392,7 +392,7 @@ var _ = Describe( APIGroupVersionsTest, ) var _ = Describe( - "CRD of apiextentions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", + "CRD of apiextensions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", Label("APIGroup", "APIExtensions", "SKIP_KIND"), APIExtensionsVersionsTest, ) diff --git a/test/e2e/nodeagentconfig/node-agent-config.go b/test/e2e/nodeagentconfig/node-agent-config.go index 3eb508234..cab65dfb5 100644 --- a/test/e2e/nodeagentconfig/node-agent-config.go +++ b/test/e2e/nodeagentconfig/node-agent-config.go @@ -62,7 +62,7 @@ var LoadAffinities func() = TestFunc(&NodeAgentConfigTestCase{ { NodeSelector: metav1.LabelSelector{ MatchLabels: map[string]string{ - "kubernetes.io/arch": "amd64", + corev1api.LabelArchStable: "amd64", }, }, StorageClass: test.StorageClassName2, diff --git a/test/testdata/volume-snapshot-class/kind.yaml b/test/testdata/volume-snapshot-class/kind.yaml new file mode 100644 index 000000000..35aa60ad4 --- /dev/null +++ b/test/testdata/volume-snapshot-class/kind.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: snapshot.storage.k8s.io/v1 +deletionPolicy: Delete +driver: hostpath.csi.k8s.io +kind: VolumeSnapshotClass +metadata: + labels: + velero.io/csi-volumesnapshot-class: "true" + name: e2e-volume-snapshot-class diff --git a/test/util/k8s/deployment.go b/test/util/k8s/deployment.go index 42e2d6ac7..01209aeb7 100644 --- a/test/util/k8s/deployment.go +++ b/test/util/k8s/deployment.go @@ -102,7 +102,7 @@ func NewDeployment( { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Values: []string{common.WorkerOSWindows}, Operator: corev1api.NodeSelectorOpIn, }, diff --git a/test/util/k8s/pod.go b/test/util/k8s/pod.go index 718beab98..ce8580eaa 100644 --- a/test/util/k8s/pod.go +++ b/test/util/k8s/pod.go @@ -84,7 +84,7 @@ func CreatePod( { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "kubernetes.io/os", + Key: corev1api.LabelOSStable, Values: []string{common.WorkerOSWindows}, Operator: corev1api.NodeSelectorOpIn, }, diff --git a/test/util/k8s/pvc.go b/test/util/k8s/pvc.go index 7d9610141..36de6fa4c 100644 --- a/test/util/k8s/pvc.go +++ b/test/util/k8s/pvc.go @@ -68,19 +68,19 @@ func (p *PVCBuilder) WithResourceStorage(q resource.Quantity) *PVCBuilder { } func CreatePVC(client TestClient, ns, name, sc string, ann map[string]string) (*corev1api.PersistentVolumeClaim, error) { - pvcBulder := NewPVC(ns, name) + pvcBuilder := NewPVC(ns, name) if ann != nil { - pvcBulder.WithAnnotation(ann) + pvcBuilder.WithAnnotation(ann) } if sc != "" { - pvcBulder.WithStorageClass(sc) + pvcBuilder.WithStorageClass(sc) } - return client.ClientGo.CoreV1().PersistentVolumeClaims(ns).Create(context.TODO(), pvcBulder.Result(), metav1.CreateOptions{}) + return client.ClientGo.CoreV1().PersistentVolumeClaims(ns).Create(context.TODO(), pvcBuilder.Result(), metav1.CreateOptions{}) } -func CreatePvc(client TestClient, pvcBulder *PVCBuilder) error { - _, err := client.ClientGo.CoreV1().PersistentVolumeClaims(pvcBulder.Namespace).Create(context.TODO(), pvcBulder.Result(), metav1.CreateOptions{}) +func CreatePvc(client TestClient, pvcBuilder *PVCBuilder) error { + _, err := client.ClientGo.CoreV1().PersistentVolumeClaims(pvcBuilder.Namespace).Create(context.TODO(), pvcBuilder.Result(), metav1.CreateOptions{}) return err }