diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..bcc7f34dd --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# maintainers are the overall code owners +* @velero-io/Maintainer \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 45332806b..682c01231 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -15,6 +15,20 @@ updates: schedule: interval: "weekly" labels: + - "Dependencies" + - "go" + - "kind/changelog-not-required" + ignore: + - dependency-name: "*" + update-types: ["version-update:semver-major", "version-update:semver-minor", "version-update:semver-patch"] + # Dependencies listed in pkg/apis/go.mod + - package-ecosystem: "gomod" + directory: "/pkg/apis" # Location of package manifests + schedule: + interval: "weekly" + labels: + - "Dependencies" + - "go" - "kind/changelog-not-required" ignore: - dependency-name: "*" diff --git a/.github/workflows/auto_assign_prs.yml b/.github/workflows/auto_assign_prs.yml index 9b915533c..8966b235e 100644 --- a/.github/workflows/auto_assign_prs.yml +++ b/.github/workflows/auto_assign_prs.yml @@ -14,6 +14,7 @@ permissions: jobs: # Automatically assigns reviewers and owner add-reviews: + if: github.repository == 'velero-io/velero' runs-on: ubuntu-latest steps: - name: Set the author of a PR as the assignee diff --git a/.github/workflows/auto_label_prs.yml b/.github/workflows/auto_label_prs.yml index 042cc7e95..21540d8cb 100644 --- a/.github/workflows/auto_label_prs.yml +++ b/.github/workflows/auto_label_prs.yml @@ -15,6 +15,7 @@ permissions: jobs: # Automatically labels PRs based on file globs in the change. triage: + if: github.repository == 'velero-io/velero' runs-on: ubuntu-latest steps: - uses: actions/labeler@v5 diff --git a/.github/workflows/auto_request_review.yml b/.github/workflows/auto_request_review.yml index 47844bc6c..096e4bdbc 100644 --- a/.github/workflows/auto_request_review.yml +++ b/.github/workflows/auto_request_review.yml @@ -11,6 +11,7 @@ permissions: jobs: auto-request-review: + if: github.repository == 'velero-io/velero' name: Auto Request Review runs-on: ubuntu-latest steps: diff --git a/.github/workflows/e2e-test-kind.yaml b/.github/workflows/e2e-test-kind.yaml index 7dcd51408..fc77cb4d3 100644 --- a/.github/workflows/e2e-test-kind.yaml +++ b/.github/workflows/e2e-test-kind.yaml @@ -1,4 +1,6 @@ name: "Run the E2E test on kind" +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true on: push: pull_request: @@ -57,8 +59,10 @@ jobs: # Check and build MinIO image once for all e2e tests - name: Check Bitnami MinIO Dockerfile version id: minio-version + env: + GH_TOKEN: ${{ github.token }} run: | - DOCKERFILE_SHA=$(curl -s https://api.github.com/repos/bitnami/containers/commits?path=bitnami/minio/2025/debian-12/Dockerfile\&per_page=1 | jq -r '.[0].sha') + DOCKERFILE_SHA=$(curl -s -H "Authorization: Bearer $GH_TOKEN" https://api.github.com/repos/bitnami/containers/commits?path=bitnami/minio/2026/debian-12/Dockerfile\&per_page=1 | jq -r '.[0].sha') echo "dockerfile_sha=${DOCKERFILE_SHA}" >> $GITHUB_OUTPUT - name: Cache MinIO Image uses: actions/cache@v4 @@ -133,11 +137,11 @@ jobs: - name: Install MinIO run: | docker run -d --rm -p 9000:9000 -e "MINIO_ROOT_USER=minio" -e "MINIO_ROOT_PASSWORD=minio123" -e "MINIO_DEFAULT_BUCKETS=bucket,additional-bucket" bitnami/minio:local - - uses: engineerd/setup-kind@v0.6.2 + - uses: helm/kind-action@v1 with: - skipClusterLogsExport: true + cluster_name: "kind" version: "v0.32.0" - image: "kindest/node:v${{ matrix.k8s }}" + node_image: "kindest/node:v${{ matrix.k8s }}" - name: Fetch built CLI id: cli-cache uses: actions/cache@v4 @@ -185,7 +189,7 @@ jobs: timeout-minutes: 30 - name: Upload debug bundle if: ${{ failure() }} - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@v7 with: name: DebugBundle-k8s-${{ matrix.k8s }}-job-${{ strategy.job-index }} path: /home/runner/work/velero/velero/test/e2e/debug-bundle* diff --git a/.github/workflows/pr-changelog-check.yml b/.github/workflows/pr-changelog-check.yml index 0f296853a..f9fb14f37 100644 --- a/.github/workflows/pr-changelog-check.yml +++ b/.github/workflows/pr-changelog-check.yml @@ -7,6 +7,7 @@ on: jobs: build: + if: github.repository == 'velero-io/velero' name: Run Changelog Check runs-on: ubuntu-latest steps: diff --git a/.github/workflows/pr-ci-check.yml b/.github/workflows/pr-ci-check.yml index aea136329..ba55e6ab0 100644 --- a/.github/workflows/pr-ci-check.yml +++ b/.github/workflows/pr-ci-check.yml @@ -24,7 +24,7 @@ jobs: - name: Make ci run: make ci - name: Upload test coverage - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@v6 with: token: ${{ secrets.CODECOV_TOKEN }} files: coverage.out diff --git a/.github/workflows/pr-codespell.yml b/.github/workflows/pr-codespell.yml index 65d2a1885..b65ae7ae5 100644 --- a/.github/workflows/pr-codespell.yml +++ b/.github/workflows/pr-codespell.yml @@ -3,6 +3,7 @@ on: [pull_request] jobs: codespell: + if: github.repository == 'velero-io/velero' name: Run Codespell runs-on: ubuntu-latest steps: diff --git a/.github/workflows/pr-containers.yml b/.github/workflows/pr-containers.yml index c2fea1386..910192171 100644 --- a/.github/workflows/pr-containers.yml +++ b/.github/workflows/pr-containers.yml @@ -19,7 +19,7 @@ jobs: - name: Set up QEMU id: qemu - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 with: platforms: all diff --git a/.github/workflows/pr-filepath-check.yml b/.github/workflows/pr-filepath-check.yml index 260a09dc4..9b8ca593d 100644 --- a/.github/workflows/pr-filepath-check.yml +++ b/.github/workflows/pr-filepath-check.yml @@ -3,6 +3,7 @@ on: [pull_request] jobs: filepath-check: + if: github.repository == 'velero-io/velero' name: Check for invalid characters in file paths runs-on: ubuntu-latest steps: diff --git a/.github/workflows/prow-action.yml b/.github/workflows/prow-action.yml index e247590fe..871f69f8f 100644 --- a/.github/workflows/prow-action.yml +++ b/.github/workflows/prow-action.yml @@ -11,6 +11,7 @@ permissions: jobs: execute: + if: github.repository == 'velero-io/velero' runs-on: ubuntu-latest steps: - uses: jpmcb/prow-github-actions@v1.1.3 diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 10b191630..b45af38d9 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -29,7 +29,7 @@ jobs: - name: Set up QEMU id: qemu - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 with: platforms: all - name: Set up Docker Buildx @@ -45,7 +45,7 @@ jobs: - name: Test run: make test - name: Upload test coverage - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@v6 with: token: ${{ secrets.CODECOV_TOKEN }} files: coverage.out diff --git a/.github/workflows/rebase.yml b/.github/workflows/rebase.yml index 07c86b534..064bef70a 100644 --- a/.github/workflows/rebase.yml +++ b/.github/workflows/rebase.yml @@ -5,7 +5,7 @@ name: Automatic Rebase jobs: rebase: name: Rebase - if: github.event.issue.pull_request != '' && contains(github.event.comment.body, '/rebase') + if: github.repository == 'velero-io/velero' && github.event.issue.pull_request != '' && contains(github.event.comment.body, '/rebase') runs-on: ubuntu-latest steps: - name: Checkout the latest code diff --git a/.github/workflows/stale-issues.yml b/.github/workflows/stale-issues.yml index 0dffc96c4..99a74872b 100644 --- a/.github/workflows/stale-issues.yml +++ b/.github/workflows/stale-issues.yml @@ -5,6 +5,7 @@ on: jobs: stale: + if: github.repository == 'velero-io/velero' runs-on: ubuntu-latest steps: - uses: actions/stale@v10.1.1 diff --git a/Makefile b/Makefile index 80db72e3a..515abf88d 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,10 @@ BIN ?= velero # This repo's root import path (under GOPATH). PKG := github.com/vmware-tanzu/velero +# Container tool for local development targets (shell, lint, build-image, etc.) +# Override with CONTAINER_TOOL=podman to use podman instead of docker. +CONTAINER_TOOL ?= docker + # Where to push the docker image. REGISTRY ?= velero # In order to push images to an insecure registry, follow the two steps: @@ -63,7 +67,7 @@ else endif BUILDER_IMAGE := $(REGISTRY)/build-image:$(BUILDER_IMAGE_TAG) -BUILDER_IMAGE_CACHED := $(shell docker images -q ${BUILDER_IMAGE} 2>/dev/null ) +BUILDER_IMAGE_CACHED := $(shell $(CONTAINER_TOOL) images -q ${BUILDER_IMAGE} 2>/dev/null ) HUGO_IMAGE := ghcr.io/gohugoio/hugo @@ -103,6 +107,11 @@ define BUILDX_ERROR buildx not enabled, refusing to run this recipe see: https://velero.io/docs/main/build-from-source/#making-images-and-updating-velero for more info endef + +define DOCKER_ONLY_ERROR +this target requires docker buildx/manifest and is not supported with CONTAINER_TOOL=$(CONTAINER_TOOL). +use docker for multi-arch image targets, or build single-arch images with podman directly. +endef # comma cannot be escaped and can only be used in Make function arguments by putting into variable comma=, @@ -198,7 +207,7 @@ shell: build-dirs build-env @# because the Kubernetes code-generator tools require the project to @# exist in a directory hierarchy ending like this (but *NOT* necessarily @# under $GOPATH). - @docker run \ + @$(CONTAINER_TOOL) run \ -e GOFLAGS \ -e GOPROXY \ -i $(TTY) \ @@ -217,6 +226,9 @@ shell: build-dirs build-env /bin/sh $(CMD) container: +ifneq ($(CONTAINER_TOOL),docker) + $(error $(DOCKER_ONLY_ERROR)) +endif ifneq ($(BUILDX_ENABLED), true) $(error $(BUILDX_ERROR)) endif @@ -246,6 +258,9 @@ container-linux-%: @BUILDX_ARCH=$* $(MAKE) container-linux container-linux: +ifneq ($(CONTAINER_TOOL),docker) + $(error $(DOCKER_ONLY_ERROR)) +endif @echo "building container: $(IMAGE):$(VERSION)-linux-$(BUILDX_ARCH)" @docker buildx build --pull \ @@ -269,6 +284,9 @@ container-windows-%: @BUILDX_OSVERSION=$(firstword $(subst -, ,$*)) BUILDX_ARCH=$(lastword $(subst -, ,$*)) $(MAKE) container-windows container-windows: +ifneq ($(CONTAINER_TOOL),docker) + $(error $(DOCKER_ONLY_ERROR)) +endif @echo "building container: $(IMAGE):$(VERSION)-windows-$(BUILDX_OSVERSION)-$(BUILDX_ARCH)" @docker buildx build --pull \ @@ -290,6 +308,9 @@ container-windows: @echo "built container: $(IMAGE):$(VERSION)-windows-$(BUILDX_OSVERSION)-$(BUILDX_ARCH)" push-manifest: +ifneq ($(CONTAINER_TOOL),docker) + $(error $(DOCKER_ONLY_ERROR)) +endif @echo "building manifest: $(IMAGE_TAG) for $(foreach osarch, $(ALL_OS_ARCH), $(IMAGE_TAG)-${osarch})" @docker manifest create --amend --insecure=$(INSECURE_REGISTRY) $(IMAGE_TAG) $(foreach osarch, $(ALL_OS_ARCH), $(IMAGE_TAG)-${osarch}) @@ -363,24 +384,30 @@ else ifneq ($(BUILDER_IMAGE_CACHED),) @echo "Using Cached Image: $(BUILDER_IMAGE)" else @echo "Trying to pull build-image: $(BUILDER_IMAGE)" - docker pull -q $(BUILDER_IMAGE) || $(MAKE) build-image + $(CONTAINER_TOOL) pull -q $(BUILDER_IMAGE) || $(MAKE) build-image endif build-image: @# When we build a new image we just untag the old one. @# This makes sure we don't leave the orphaned image behind. - $(eval old_id=$(shell docker image inspect --format '{{ .ID }}' ${BUILDER_IMAGE} 2>/dev/null)) + $(eval old_id=$(shell $(CONTAINER_TOOL) image inspect --format '{{ .ID }}' ${BUILDER_IMAGE} 2>/dev/null)) ifeq ($(BUILDX_ENABLED), true) - @cd hack/build-image && docker buildx build --build-arg=GOPROXY=$(GOPROXY) --output=type=docker --pull -t $(BUILDER_IMAGE) -f $(BUILDER_IMAGE_DOCKERFILE_REALPATH) . -else - @cd hack/build-image && docker build --build-arg=GOPROXY=$(GOPROXY) --pull -t $(BUILDER_IMAGE) -f $(BUILDER_IMAGE_DOCKERFILE_REALPATH) . +ifneq ($(CONTAINER_TOOL),docker) + $(error $(DOCKER_ONLY_ERROR)) endif - $(eval new_id=$(shell docker image inspect --format '{{ .ID }}' ${BUILDER_IMAGE} 2>/dev/null)) + @cd hack/build-image && $(CONTAINER_TOOL) buildx build --build-arg=GOPROXY=$(GOPROXY) --output=type=docker --pull -t $(BUILDER_IMAGE) -f $(BUILDER_IMAGE_DOCKERFILE_REALPATH) . +else + @cd hack/build-image && $(CONTAINER_TOOL) build --build-arg=GOPROXY=$(GOPROXY) --pull -t $(BUILDER_IMAGE) -f $(BUILDER_IMAGE_DOCKERFILE_REALPATH) . +endif + $(eval new_id=$(shell $(CONTAINER_TOOL) image inspect --format '{{ .ID }}' ${BUILDER_IMAGE} 2>/dev/null)) @if [ "$(old_id)" != "" ] && [ "$(old_id)" != "$(new_id)" ]; then \ - docker rmi -f $$id || true; \ + $(CONTAINER_TOOL) rmi -f $$id || true; \ fi push-build-image: +ifneq ($(CONTAINER_TOOL),docker) + $(error $(DOCKER_ONLY_ERROR)) +endif @# this target will push the build-image it assumes you already have docker @# credentials needed to accomplish this. @# Pushing will be skipped if a custom Dockerfile was used to build the image. @@ -392,17 +419,17 @@ else endif build-image-hugo: - cd site && docker build --pull -t $(HUGO_IMAGE) . + cd site && $(CONTAINER_TOOL) build --pull -t $(HUGO_IMAGE) . clean: # if we have a cached image then use it to run go clean --modcache # this test checks if we there is an image id in the BUILDER_IMAGE_CACHED variable. ifneq ($(strip $(BUILDER_IMAGE_CACHED)),) $(MAKE) shell CMD="-c 'go clean --modcache'" - docker rmi -f $(BUILDER_IMAGE) || true + $(CONTAINER_TOOL) rmi -f $(BUILDER_IMAGE) || true endif rm -rf .go _output - docker rmi $(HUGO_IMAGE) + $(CONTAINER_TOOL) rmi $(HUGO_IMAGE) .PHONY: modules @@ -447,7 +474,7 @@ release: ./hack/release-tools/goreleaser.sh'" serve-docs: build-image-hugo - docker run \ + $(CONTAINER_TOOL) run \ --rm \ -v "$$(pwd)/site:/project" \ -it -p 1313:1313 \ diff --git a/changelogs/unreleased/10004-chlins b/changelogs/unreleased/10004-chlins new file mode 100644 index 000000000..142a83705 --- /dev/null +++ b/changelogs/unreleased/10004-chlins @@ -0,0 +1 @@ +Support selecting the data mover type (velero-fs or velero-block) through the volume policy snapshot action's dataMover parameter diff --git a/changelogs/unreleased/9792-subhramit b/changelogs/unreleased/9792-subhramit new file mode 100644 index 000000000..8868ed627 --- /dev/null +++ b/changelogs/unreleased/9792-subhramit @@ -0,0 +1 @@ +Restores from backups not in a completed or partially failed phase are now rejected. diff --git a/changelogs/unreleased/9847-adam-jian-zhang b/changelogs/unreleased/9847-adam-jian-zhang new file mode 100644 index 000000000..82de39ca0 --- /dev/null +++ b/changelogs/unreleased/9847-adam-jian-zhang @@ -0,0 +1 @@ +Fix issue #9813, add validations for ClusterScopedFilterPolicy diff --git a/changelogs/unreleased/9863-blackpiglet b/changelogs/unreleased/9863-blackpiglet new file mode 100644 index 000000000..49bae8d36 --- /dev/null +++ b/changelogs/unreleased/9863-blackpiglet @@ -0,0 +1 @@ +Support change-id and volume-id in backup workflow. \ No newline at end of file diff --git a/changelogs/unreleased/9880-adam-jian-zhang b/changelogs/unreleased/9880-adam-jian-zhang new file mode 100644 index 000000000..010def90c --- /dev/null +++ b/changelogs/unreleased/9880-adam-jian-zhang @@ -0,0 +1 @@ +Fix issue #9815, implement core logic of backup with ClusterScopedFilterPolicy and NamespacedFilterPolicies diff --git a/changelogs/unreleased/9881-adam-jian-zhang b/changelogs/unreleased/9881-adam-jian-zhang new file mode 100644 index 000000000..8c37c6062 --- /dev/null +++ b/changelogs/unreleased/9881-adam-jian-zhang @@ -0,0 +1 @@ +Fix issue #9816, add cli support for backup with ClusterScopedFilterPolicy and NamespacedFilterPolicies diff --git a/changelogs/unreleased/9887-Lyndon-Li b/changelogs/unreleased/9887-Lyndon-Li new file mode 100644 index 000000000..a44418c9c --- /dev/null +++ b/changelogs/unreleased/9887-Lyndon-Li @@ -0,0 +1 @@ +Add WriteAt implementation for Incremental aware object writer for block data mover \ No newline at end of file diff --git a/changelogs/unreleased/9892-Lyndon-Li b/changelogs/unreleased/9892-Lyndon-Li new file mode 100644 index 000000000..6caac1b18 --- /dev/null +++ b/changelogs/unreleased/9892-Lyndon-Li @@ -0,0 +1 @@ +Enhance RebindVolume method for generic restore exposer to support block data mover \ No newline at end of file diff --git a/changelogs/unreleased/9896-shubham-pampattiwar b/changelogs/unreleased/9896-shubham-pampattiwar new file mode 100644 index 000000000..034507c1f --- /dev/null +++ b/changelogs/unreleased/9896-shubham-pampattiwar @@ -0,0 +1 @@ +Skip VGS cleanup when backup did not use VolumeGroupSnapshots diff --git a/changelogs/unreleased/9899-Lyndon-Li b/changelogs/unreleased/9899-Lyndon-Li new file mode 100644 index 000000000..fdcee0a29 --- /dev/null +++ b/changelogs/unreleased/9899-Lyndon-Li @@ -0,0 +1 @@ +Refactor object writer interface to align with incremental aware object writer \ No newline at end of file diff --git a/changelogs/unreleased/9903-Joeavaikath b/changelogs/unreleased/9903-Joeavaikath new file mode 100644 index 000000000..d586acfaa --- /dev/null +++ b/changelogs/unreleased/9903-Joeavaikath @@ -0,0 +1 @@ +Fix restore finalization overwriting dynamically provisioned PV labels with stale backup values diff --git a/changelogs/unreleased/9904-Joeavaikath b/changelogs/unreleased/9904-Joeavaikath new file mode 100644 index 000000000..9ef1cfae5 --- /dev/null +++ b/changelogs/unreleased/9904-Joeavaikath @@ -0,0 +1 @@ +Add CONTAINER_TOOL variable to Makefile for podman support diff --git a/changelogs/unreleased/9906-chlins b/changelogs/unreleased/9906-chlins new file mode 100644 index 000000000..78df0d109 --- /dev/null +++ b/changelogs/unreleased/9906-chlins @@ -0,0 +1 @@ +feat(resourcepolicies): support PVC volume mode and access mode matching diff --git a/changelogs/unreleased/9908-adam-jian-zhang b/changelogs/unreleased/9908-adam-jian-zhang new file mode 100644 index 000000000..2ba573609 --- /dev/null +++ b/changelogs/unreleased/9908-adam-jian-zhang @@ -0,0 +1 @@ +Fix issue #9907, add cache for the GetNamespaceFilter call diff --git a/changelogs/unreleased/9913-Lyndon-Li b/changelogs/unreleased/9913-Lyndon-Li new file mode 100644 index 000000000..363e482c1 --- /dev/null +++ b/changelogs/unreleased/9913-Lyndon-Li @@ -0,0 +1 @@ +Refactor generic restore exposer to clone the PV during rebind, so as to support block data mover \ No newline at end of file diff --git a/changelogs/unreleased/9921-shubham-pampattiwar b/changelogs/unreleased/9921-shubham-pampattiwar new file mode 100644 index 000000000..6475cfe58 --- /dev/null +++ b/changelogs/unreleased/9921-shubham-pampattiwar @@ -0,0 +1 @@ +Design: Server default restore resource modifier diff --git a/changelogs/unreleased/9928-chlins b/changelogs/unreleased/9928-chlins new file mode 100644 index 000000000..29ee82e6c --- /dev/null +++ b/changelogs/unreleased/9928-chlins @@ -0,0 +1 @@ +Add `--global-backup-volume-policies-configmap` server flag to configure cluster-wide global backup volume policies that are merged into every backup diff --git a/changelogs/unreleased/9930-Lyndon-Li b/changelogs/unreleased/9930-Lyndon-Li new file mode 100644 index 000000000..e8b21c2b8 --- /dev/null +++ b/changelogs/unreleased/9930-Lyndon-Li @@ -0,0 +1 @@ +Support fsType in DUCR and DDCR for data mover \ No newline at end of file diff --git a/changelogs/unreleased/9932-Lyndon-Li b/changelogs/unreleased/9932-Lyndon-Li new file mode 100644 index 000000000..a09169cf2 --- /dev/null +++ b/changelogs/unreleased/9932-Lyndon-Li @@ -0,0 +1 @@ +Wait restorePV detached before binding the cloned PV to avoid confusing the CSI driver \ No newline at end of file diff --git a/changelogs/unreleased/9933-Lyndon-Li b/changelogs/unreleased/9933-Lyndon-Li new file mode 100644 index 000000000..1d8582fe9 --- /dev/null +++ b/changelogs/unreleased/9933-Lyndon-Li @@ -0,0 +1 @@ +Recall the old rebind volume way for the case that volumeMode is not changed; and use the new way for volumeMode changed case \ No newline at end of file diff --git a/changelogs/unreleased/9939-adam-jian-zhang b/changelogs/unreleased/9939-adam-jian-zhang new file mode 100644 index 000000000..187bf7397 --- /dev/null +++ b/changelogs/unreleased/9939-adam-jian-zhang @@ -0,0 +1 @@ +Fix issue #9935, add resource policy on restore CRD diff --git a/changelogs/unreleased/9941-Lyndon-Li b/changelogs/unreleased/9941-Lyndon-Li new file mode 100644 index 000000000..fd29540a3 --- /dev/null +++ b/changelogs/unreleased/9941-Lyndon-Li @@ -0,0 +1 @@ +Decide restorePVC volumeMode by data mover type for block data mover \ No newline at end of file diff --git a/changelogs/unreleased/9943-lubronzhan b/changelogs/unreleased/9943-lubronzhan new file mode 100644 index 000000000..3c63bffdd --- /dev/null +++ b/changelogs/unreleased/9943-lubronzhan @@ -0,0 +1 @@ +Extract pkg/apis into its own Go module with a local replace directive in the root go.mod \ No newline at end of file diff --git a/changelogs/unreleased/9944-lubronzhan b/changelogs/unreleased/9944-lubronzhan new file mode 100644 index 000000000..25fb1f376 --- /dev/null +++ b/changelogs/unreleased/9944-lubronzhan @@ -0,0 +1 @@ +Fix flaky TestWaitExecHandleHooks test for 2-container hook ordering by synchronizing pod state changes with hook execution using channels \ No newline at end of file diff --git a/changelogs/unreleased/9945-Lyndon-Li b/changelogs/unreleased/9945-Lyndon-Li new file mode 100644 index 000000000..bdb4d8d5e --- /dev/null +++ b/changelogs/unreleased/9945-Lyndon-Li @@ -0,0 +1 @@ +Add snapshot operations for block uploader \ No newline at end of file diff --git a/changelogs/unreleased/9946-adam-jian-zhang b/changelogs/unreleased/9946-adam-jian-zhang new file mode 100644 index 000000000..7d7a9db76 --- /dev/null +++ b/changelogs/unreleased/9946-adam-jian-zhang @@ -0,0 +1 @@ +Fix issue #9936, restore filters via resource policy implementation diff --git a/changelogs/unreleased/9954-blackpiglet b/changelogs/unreleased/9954-blackpiglet new file mode 100644 index 000000000..d3215eb58 --- /dev/null +++ b/changelogs/unreleased/9954-blackpiglet @@ -0,0 +1 @@ +Add BackupType in backup.spec \ No newline at end of file diff --git a/changelogs/unreleased/9955-sseago b/changelogs/unreleased/9955-sseago new file mode 100644 index 000000000..6bed2f243 --- /dev/null +++ b/changelogs/unreleased/9955-sseago @@ -0,0 +1 @@ +Optimize VSC handle readiness polling for VSS backups diff --git a/changelogs/unreleased/9966-adam-jian-zhang b/changelogs/unreleased/9966-adam-jian-zhang new file mode 100644 index 000000000..c95540c80 --- /dev/null +++ b/changelogs/unreleased/9966-adam-jian-zhang @@ -0,0 +1 @@ +Fix issue #9937, add CLI support for restore filters via resource policy diff --git a/changelogs/unreleased/9968-ywk253100 b/changelogs/unreleased/9968-ywk253100 new file mode 100644 index 000000000..e152a5f49 --- /dev/null +++ b/changelogs/unreleased/9968-ywk253100 @@ -0,0 +1 @@ +Use forward slash as the path separator to make sure it works on both Linux and Windows nodes \ No newline at end of file diff --git a/changelogs/unreleased/9978-shubham-pampattiwar b/changelogs/unreleased/9978-shubham-pampattiwar new file mode 100644 index 000000000..856fe087f --- /dev/null +++ b/changelogs/unreleased/9978-shubham-pampattiwar @@ -0,0 +1 @@ +Add image volume type support to volume policies diff --git a/changelogs/unreleased/9979-Lyndon-Li b/changelogs/unreleased/9979-Lyndon-Li new file mode 100644 index 000000000..78134da35 --- /dev/null +++ b/changelogs/unreleased/9979-Lyndon-Li @@ -0,0 +1 @@ +Add the backup implementation for block data mover \ No newline at end of file diff --git a/changelogs/unreleased/9982-shubham-pampattiwar b/changelogs/unreleased/9982-shubham-pampattiwar new file mode 100644 index 000000000..aeb80e8da --- /dev/null +++ b/changelogs/unreleased/9982-shubham-pampattiwar @@ -0,0 +1 @@ +Validate user-provided labels and annotations in maintenance job diff --git a/changelogs/unreleased/9989-adam-jian-zhang b/changelogs/unreleased/9989-adam-jian-zhang new file mode 100644 index 000000000..ab0954c20 --- /dev/null +++ b/changelogs/unreleased/9989-adam-jian-zhang @@ -0,0 +1 @@ +Fix globalExcludes lookup, it should be lookup against lower case diff --git a/changelogs/unreleased/9993-chlins b/changelogs/unreleased/9993-chlins new file mode 100644 index 000000000..79c5236bb --- /dev/null +++ b/changelogs/unreleased/9993-chlins @@ -0,0 +1 @@ +Surface DeleteItemAction plugin errors from InvokeDeleteActions so backup deletion fails and retries instead of silently orphaning data mover snapshots and other private artifacts diff --git a/changelogs/unreleased/9994-Lyndon-Li b/changelogs/unreleased/9994-Lyndon-Li new file mode 100644 index 000000000..81ea8a01e --- /dev/null +++ b/changelogs/unreleased/9994-Lyndon-Li @@ -0,0 +1 @@ +Add block device operations for block uploader backup \ No newline at end of file diff --git a/changelogs/unreleased/9995-shubham-pampattiwar b/changelogs/unreleased/9995-shubham-pampattiwar new file mode 100644 index 000000000..691ab8d1a --- /dev/null +++ b/changelogs/unreleased/9995-shubham-pampattiwar @@ -0,0 +1 @@ +Fix PodVolumeBackup metadata loss on fs-backup timeout, which caused all fs-backup volumes to become unrestorable diff --git a/config/crd/v1/bases/velero.io_backups.yaml b/config/crd/v1/bases/velero.io_backups.yaml index 794c342c8..3b98f2ad4 100644 --- a/config/crd/v1/bases/velero.io_backups.yaml +++ b/config/crd/v1/bases/velero.io_backups.yaml @@ -41,6 +41,13 @@ spec: spec: description: BackupSpec defines the specification for a Velero backup. properties: + backupType: + description: BackupType specifies how volume data is backed up, with + possible values including Full and Incremental. + enum: + - Full + - Incremental + type: string csiSnapshotTimeout: description: |- CSISnapshotTimeout specifies the time used to wait for CSI VolumeSnapshot status turns to diff --git a/config/crd/v1/bases/velero.io_restores.yaml b/config/crd/v1/bases/velero.io_restores.yaml index 1b92ea4fc..c41fe88de 100644 --- a/config/crd/v1/bases/velero.io_restores.yaml +++ b/config/crd/v1/bases/velero.io_restores.yaml @@ -404,6 +404,33 @@ spec: - name type: object x-kubernetes-map-type: atomic + resourcePolicy: + description: |- + ResourcePolicy specifies the reference to a ConfigMap containing resource + filter policies for this restore. The ConfigMap can contain a + namespacedFilterPolicies section that specifies per-namespace resource type + filters, label selectors, and resource name patterns, and a + clusterScopedFilterPolicy section for per-kind filtering of cluster-scoped + resources. The ConfigMap format is the same as for BackupSpec.ResourcePolicy. + nullable: true + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic restorePVs: description: |- RestorePVs specifies whether to restore all included diff --git a/config/crd/v1/bases/velero.io_schedules.yaml b/config/crd/v1/bases/velero.io_schedules.yaml index 7719a4b13..4b13ecec7 100644 --- a/config/crd/v1/bases/velero.io_schedules.yaml +++ b/config/crd/v1/bases/velero.io_schedules.yaml @@ -80,6 +80,13 @@ spec: Template is the definition of the Backup to be run on the provided schedule properties: + backupType: + description: BackupType specifies how volume data is backed up, + with possible values including Full and Incremental. + enum: + - Full + - Incremental + type: string csiSnapshotTimeout: description: |- CSISnapshotTimeout specifies the time used to wait for CSI VolumeSnapshot status turns to diff --git a/config/crd/v1/crds/crds.go b/config/crd/v1/crds/crds.go index 032a1ac84..44d2b378c 100644 --- a/config/crd/v1/crds/crds.go +++ b/config/crd/v1/crds/crds.go @@ -30,14 +30,14 @@ import ( var rawCRDs = [][]byte{ []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xccW͎\xdb6\x10\xbe\xfb)\x06鵲\x1b\x14-\n\xdf\x12\xa7\x01\x82&\xc1\xc2\x0e\xf6NSc\x99Y\x8adɡS\xf7\xe7\u074b!%[\x96dk\xbd\x87\xa2\xbc\x89\x9c\xffo\xe6#U\x14\xc5L8\xf5\x88>(k\x96 \x9c\xc2?\b\r\x7f\x85\xf9\xd3/a\xae\xec\xe2\xf0z\xf6\xa4L\xb9\x84U\fd\xeb5\x06\x1b\xbd\xc4w\xb8SF\x91\xb2fV#\x89R\x90X\xce\x00\x841\x96\x04o\a\xfe\x04\x90\u0590\xb7Z\xa3/*4\xf3\xa7\xb8\xc5mT\xbaD\x9f\x8c\xb7\xae\x0f?\xcc_\xff<\xffi\x06`D\x8dK\xd8\n\xf9\x14\x9dGg\x83\"\xeb\x15\x86\xf9\x015z;Wv\x16\x1cJ\xb6^y\x1b\xdd\x12\xce\aY\xbb\xf1\x9c\xa3~\x9b\f\xad[C\xc7t\xa4U\xa0\xdfF\x8f?\xaa@I\xc4\xe9\xe8\x85\x1e\v$\x1d\ae\xaa\xa8\x85\x1f\b\xb0\x83 \xad\xc3%|\xe6X\x9c\x90X\xce\x00\x9aLSl\x05\x88\xb2L\xb5\x13\xfa\xc1+C\xe8WVǺ\xadY\x01_\x835\x0f\x82\xf6K\x98\xb7՝K\x8f\xa9\xb0_T\x8d\x81D\xed\x92l[\xb07\x156\xdftd\xe7\xa5 \x1c\x1a\xe3\xca\xcdϱ~9:\xbc\xb0r.\x04tβ\xc5@^\x99jv\x16>\xbcΥ\x90{\xacŲ\x91\xb5\x0e͛\x87\x0f\x8f?n.\xb6\x01\x9c\xb7\x0e=\xa9\x16\x9e\xbc:\xed\xd7\xd9\x05(1H\xaf\x1c\xa5\xe6\xf8\xbb\xb88\x03`\aY\vJ\xeeC\f@{lk\x8ce\x13\x13\xd8\x1d\xd0^\x05\xf0\xe8<\x064\xb93y[\x18\xb0ۯ(i\xde3\xbdA\xcff \xecm\xd4%\xb7\xef\x01=\x81Gi+\xa3\xfe<\xd9\x0e@69Ղ0\x10$\x14\x8d\xd0p\x10:\xe2\xf7 Lٳ\\\x8b#xd\x9f\x10M\xc7^R\b\xfd8>Y\x8f\xa0\xcc\xce.aO\xe4\xc2r\xb1\xa8\x14\xb5C)m]G\xa3\xe8\xb8H\U000e5d91\xac\x0f\x8b\x12\x0f\xa8\x17AU\x85\xf0r\xaf\b%E\x8f\v\xe1T\x91\x121i0\xe7u\xf9\x9do\xc68\\\xb8\x1d\x00\x9dW\x9a\xa4;\xe0\xe1\xd1\x02\x15@4\xa6r\x8ag\x14x\x8bK\xb7\xfeu\xf3\x05\xdaH2R\x19\x94\xb3\xe8\xa0.->\\Mev\xe8\xb3\xde\xce\xdb:\xd9DS:\xab\f\xa5\x0f\xa9\x15\x1a\x82\x10\xb7\xb5\"n\x83\xdf#\x06b\xe8\xfafW\x89\xb8`\x8b\x10\x1d\x8fN\xd9\x17\xf8``%j\xd4+\x11\xf0?ƊQ\t\x05\x83\xf0,\xb4\xbat\xdc\x17\xce\xe5\xed\x1c\xb4Tz\x05\xda>=n\x1cJF\x96\x8b˪j\xa7d\x9e\xa9\x9d\xf5 \x06\xf2\x97\x95\x1a\xa7\x00^\x99D7d\xbd\xa8\xf0\xa3\xcd6\xfbBSm\xc7\xeb혡6b\xa6\xad\xcc\t8.8b\x90\xf6\x82:d@B\x99\x13\xa7\x8c&y\x03\x99\x84\x8e`\xa60\xc2H|\x9f\xfa\xd1\xc8\xe3D\xa2\x9fFT8\xa5\xbd\xfd\x06vGh\xbaF\x9bXG2\xd9\"\xf8h\xee\n\xf6\x9c\xe3ʚ\x9d\xaa\x86\x81v/\xb2k\xe0N8\xe9e\xbb\xee\xf9\xe4L\xb9\xb9α\x14m\xe71 ;UE\x7f\r\xbc\x9dB]\x0e(\x04\xc0D\xad\xc5V\xe3\x12\xc8G\xbcR\x91\xc1\xac\\V\x84\xef\xc7\t\xe0\xd6\x17\u00a0L\xc9\xd3\xd2\\V\xec\xa4mFn\x7f4%\xf8\xcbgJw\xa1\x89\xf5\xd0]\x01O\xd6)1\xb2\xef1\x90\x92#\a\xaf^\xdd\xd7\x01l\xe6C\xc9t\xb4S\xe8'2~Ǽ\xcd9\x0e\x1b\xf0\x86\x93\x03?~\xf0\xf4\\z\xc9\xdc?^\x9a\xe8N|\xdeH3\x9bi\xa6S\xe6v\xa4ÈIg\xcb&\xb2F/\xf5\xe1\x1d\xf3ó\xaa<\xf6\xae\xceb\x9c\xecz2c4\xd1\x13\xe9U\xedYlO\x82b\xb8\x87\xef\x93B[M\x19\xbdO\xf7i\xde\xe5gԋ\x19_\x8b@\x1db\xe3G\xed\x04\xee\x1f\x87\x1am`l\f\x887\x18\xdan\xf1Fp\rQJ\xc4rx\xc5\x03\xe3[\vʏ\xe7\x82\xed\xbd\x8c9Ɖ\x1fC\x10\xd5T\x92\x9f\xb2T~=5* \xb66\xd2\x15\x04h?\x96\xe3mT&\"u{\x11\xa6\xe2|`\x99\xb1\xbe\xe8]\xb0\xb7B\xb8Fi\x9f\xf1\xdb\xc8\xee\x1aE9\xa4\xc5\x02>[\x1a?\xba\xc9j\x12M\xb7\x99&\x89\xbc'ϙ_`И\x1c\xf4\xdf0kEX\x8f^\x91\xd7g%/ik\xa7\x91\xf0\xf4\xff7.\xd6\v}\xd5\xd7:\x81\x96\x0f\xf8y\x94&\xe7j/\xb5%\x9bJ,\xaf\xe9\x11\xcakb\x90\xf2\xba\xf9j\x80[C5R\x89{G\xebj)2\xdc\xcf+\xc7d\x06\x1eC\xd4\xf4\xac\x04\xd6I\xb4\xc5/+\x9e\xdb\xefy\xf1\x8c\xcf\\^\x05lZj\xbc*\xf1^(}\xf5x2\xd9@\xc2\xd3}\xfd\xbb\xb9P9\xfd{\xf0n\xb7o\xff\x97\xfdy\xe3\x1d\xd9\x1e\n\xef\xc5q\xfa\xea\x1el\x06\xfe\r.;\xc1\x85\xfc\x9c\xe8\xee\xc4\xed\xe9/\x7f\t\x7f\xfd3\xfb7\x00\x00\xff\xff\x96֥5\xef\x13\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=]s\xdb8\x92\xef\xf9\x15(\xdd\xc3\xecnI\xf6\xa6\ue8ee\xfc\x96q\x92\x1d\xd5\xcc$\xde\xd8\xe3}\x86Ȗ\x841\bp\x00P\xb6\xf6\xee\xfe\xfb\x15\x1a\x00?DP\x04eٓ\xdd\r_\x12\x8b`\x03\xfd\xddh4\x80\xc5b\xf1\x86\x96\xec\x1e\x94fR\\\x11Z2x2 \xec_\xfa\xe2\xe1\xbf\xf5\x05\x93\x97\xbb\xb7o\x1e\x98ȯ\xc8u\xa5\x8d,\xbe\x80\x96\x95\xca\xe0=\xac\x99`\x86I\xf1\xa6\x00Csj\xe8\xd5\x1bB\xa8\x10\xd2P\xfb\xb3\xb6\x7f\x12\x92Ia\x94\xe4\x1c\xd4b\x03\xe2\xe2\xa1Z\xc1\xaab<\a\x85\xc0C\u05fb?_\xbc\xfd\xaf\x8b\xff|C\x88\xa0\x05\\\x91\x15\xcd\x1e\xaaR_쀃\x92\x17L\xbe\xd1%d\x16\xe4Fɪ\xbc\"\xcd\v\xf7\x89\xef\xce\r\xf5{\xfc\x1a\x7f\xe0L\x9b\x1f[?\xfeĴ\xc1\x17%\xaf\x14\xe5uO\xf8\x9bfbSq\xaa¯o\bљ,\xe1\x8a|\xb2]\x944\x83\xfc\r!~\xd4\xd8\xe5\xc2\x0fx\xf7\xd6AȶPP7\x16Bd\t\xe2\xdd\xcd\xf2\xfe\xdfo;?\x13\x92\x83\xce\x14+\r\xe2\xfe\xbf\x8b\xfaw\xe2GI\x98&\x94\xdc#\x8eDy\x92\x13\xb3\xa5\x86((\x15h\x10F\x13\xb3\x05\x92\xd1\xd2T\n\x88\\\x93\x1f\xab\x15(\x01\x06t\v^\xc6+m@\x11m\xa8\x01B\r\xa1\xa4\x94L\x18\xc2\x041\xac\x00\xf2\x87w7K\"W\xbfBf4\xa1\"'Tk\x991j ';ɫ\x02ܷ\x7f\xbc\xa8\xa1\x96J\x96\xa0\f\vDwOK\x92Z\xbf\x1e\xc3\xd5>\x96<\xee+\x92[\x91\x02\x87\x96'1䞢\x16?\xb3e\xbaA\x1f\x85\xcc\xfeL\x85\x1f\xfe\xc5\x01\xe8[P\x16\f\xd1[Y\xf1\xdcJ\xe2\x0e\x94%`&7\x82\xfd\xbd\x86\xad\x89\x91\xd8)\xa7\x06\xb4\xa5\x8c\x01%(';\xca+\x98[\xa2\x1c@.\xe8\x9e(\xb0}\x92J\xb4\xe0\xe1\a\xfap\x1c?K\x05\x84\x89\xb5\xbc\"[cJ}uy\xb9a&\xe8W&\x8b\xa2\x12\xcc\xec/QUت2R\xe9\xcb\x1cv\xc0/5\xdb,\xa8ʶ\xcc@f\xd9|IK\xb6@D\x04\xea\xd8E\x91\xff[\x10\x0f\xdd\xe9\xd6\xec\xad\xd8j\xa3\x98ش^\xa0~L`\x8fU\x1d'\x8c\x0e\x94C\xb1\xe1\x82\xfdɒ\xeeˇۻ\xb6\xa02\xed\x99Ғ\xd7!\xfeXj2\xb1\x06\xe5\xbe[+Y L\x10\xb9\x13U\x94s\xce@\x18\xa2\xabU\xc1\x8c\x15\x83\xdf*\xd0V\a\xe4!\xd8k\xb4Ad\x05\xa4*s+Ƈ\r\x96\x82\\\xd3\x02\xf85\xd5\xf0ʼ\xb2\\\xd1\v˄$n\xb5-\xebacG\xde\u058b` \aX\xeb\f\xcbm\tYG\xd1\xecWl\xcd2\xa7Nk\xa9\x1a\xbb\xe3l`\x97BqշO\xa6٭\xa0\xa5\xdeJs\xc7\n\x90\x959l1&kȼ\xdb\xe5\x01\x940B?^\xb4Y\x95\x86\xdc*\xed#e\x06\xc7|}\xbb$\xf7h\xac\xc2\xd7h\xb4*ML\xa5\x84\x95\x92H__\x80\xe6\xfb;\xf9\x8b\x06\x92W(ܙ\x02\xa4Ü\xac`m%A\x81\xfd\u07be\x02\xa5,m4\x0e@V=cc\x9f\xbb-X\xdaҊ\x1b\xaf'L\x93\xb7\x7f&\x05\x13\x95\xe9\x89\xda בR\xd4\xd0B\xee@\x9dB\xc4\xf7\xd4П\xed\xc7\a\xb4\xb3@\tB\xb5\xc4[y:\xae\xf6\xf82\xc6m\xf7,\xd7-\x88L\x93ٌHEf\xce\x03\xcf\xe6\xee\xeb\x8aq\xb3`\xa2\xdd\xc7#\xe3<\xf42\ryGC\xc7P}'?j'\xbc'\xd1b\x00V\x8b4\x8f[0[P\xa4\x94\xb5\xc7[3\x0eD﵁\xc2\x13&x\x11\x8fO\xa4'\xd4\x1d\xce=\bm\xe9\xea\x11\xe9#/*\xce\xe9\x8a\xc3\x151\xaa\x82\x01ڬ\xa4\xe4@\xc5\bq\xbe\x806,;\ai\x1c\xa4\ba\x94\x7fѡ\x00:M\xfa\x00\x84F@{\x9aY\xef\xccy\x8b\xb0]\xaaD\xc7T*Ȭվ\xf2ހ\x01G\x0f$$\xe1Rl@\xb9\xdem\xa4\x12\x04L\x81\x15\xb8\x9cXC\xab\x80[oB֕\xb5\xc1\x17\xc4j\xf7\xa0\f0\xa1\rЈp>\x83?\xf0\x94\xf1*\x87\xfc\xda\x05^\xb76~\xccC\xd4ܳ\x9a)|\xfap\x14\xa2\xf7Μe\x18\x04\xfaxo\x81qkLL\x1b'\xbd/\xc1\x85Ζ\x95~؍\xf7=j\x0f4\x18\xfb\xd1\xecO\xb39r\xb8\xdbk\xb7\x0fM\xa8\x82\x9a,\xc9v\x13\x8a\xd2\xec\xfb\xad\x99\x81\"Bţ\xf6$\x91\x9fT)\xba\x1f\xe0f\x1d\xff\x9f\x91\x9fC0\x0f8*B\xb3W\xe6\xe9a\xbf\xff\xcc\\=\x0f\x1f5\xcev)\x13\x96\x7fv\xe2\xd9a\x9fv\xf37K6!M\x04\x1e\x13\x0e\x1eN͎p\xebw\"\xd6Yd~H\xc8k\xd9\xf2\xc2\xfb\x0fI\xa9\xad\x94\x0fc\xd4\xf9\xc1\xb6i&E$ì\nY\xc1\x96\xee\x98T\x1e\xf5\xc6\xd5\xc2\x13d\x95\x89j=5$g\xeb5(\v\xa7\xdcR\r\xdaM\x93\x87\t2\x1c\xbe\x93\x96\x19\x89\xbe<\xc0\xa3a\xa4e\x13b>4t\x1bG\x1cz\xc9\xf0\u0601\xda\xf0\x1a\x9dq\xcev,\xaf(G\xbfLE\xe6\xf0\xa1\xf5\xb8bV\xe6\b\x93{c\x8eJ\xa6{\\@\x10\x90\xb2L\xea̔\xa4\x00\x1b\xf3\x16vN\xd0o:\x8c\xf9\x8a\xdaXE\x0eaO\x90Y\xaa\xe2\xa0}W9\x86\x91\x8d͘7L\xc1D\x04\xe1t\x05\x9ch\xe0\x90\x19\xa9\xe2\x14\x19\xe3\xb3{R\x8c\xe0\x00!#\x96\xaf;\xd3h\x108\x02\x92\xe0\x14n˲\xad\v\xf5\xac\x10!\x1c\x92K\xb0\x01\x9f!\xb4,y\xc4]4\xcfQ\xe6\xfbN\x8e\xe9z\xf3\x8ch\xfd!\xbc\x98\xfe7O\x82\xcdl\x9e(i\x1b\xfd\xeaR\xb6\x16\x87\xf8\x9c\xb6y\xfe9\t\x1b,\xff\tB{D\xfb\tf\x85\x92ezPn-U\x19\xe8\v\x1bNa\xa43'̄_\xc74\xa1\x13s\xf5\x92e\x1d\"|ݼ\x99.\xf4\x89\xacIщ\x17bL\xdd\xc5? _\xd0e\xdcz\x8f\x91̓\x9f\xda_\xcd\t[\xd7D\xcf\xe7d\u0378\x01u@\xfd\x93L}\xe0\xcc9\x88\x91\xe2\xf5\b\xa6\xefM\xb6\xfd\xf0dC0ݬT%\xd2\xe5\xf0c\x17Ȇh\xbf\xeb\x9eG\xe0\x12Lc3\x05\x05\xa6\xc7q\xc6\xd4\xfe\x05C\xabw\x9f\xde\xc7\xe7W\xed'A\xf2z\x88\x8c(\x9d{\xde\x1d`\xd4\x1e\x9f\x0f\xe1\xc3\x1b\x8c\x81\xea\t\x90[\n\x99\x13J\x1e`\xefB\x17*\x88\xe5\x0f\r\x8d\x13\xbaW\x80k2(g\x0f\xb0G0\xf1E\x96\xfe\x93*\r\xeey\x80}J\xb3\x03\x1a\xda11\xed\x17\x8f,\x9d\xec\x0fH\b̭\xa7\x8a\x81{\xbc*D\x964\xe2O\xa2-\tO\xa0\xfd\th&\x89J\xbb\x8f\xf6*%J\xc0w\xda\xf1\xd2j̖\x95hV1\xe3 \xd7\xc9\fu\xcf=\xe5,\xaf;r:\xb2\x14s\xf2I\x1a\xfbχ'\xa6\xfdB\xe6{\t\xfa\x934\xf8ˋP\xd4\r\xfc%\xe9\xe9z@E\x13\xce\xca[\x82\xb5\x97\xe2\x9cO\xb3\xd2VӞi\xb2\x14v\xba\xe2H\x92\xd8\x15\xae\xba\xba\xee\\GE\xa5q\x15MH\xb1pi\x9bXO\x9e\xdeRu\xc8\xfd\xecN}\x87w\xd6Y\xb87n\xed\x97\xd3\f\xf2\xb0\\\x83\x8b\x92\xd4\xc0\x86e\x89\xfd\x15\xa06@Jk\xc2\xd3$\"Ѱzl\xa6\x89O\x9a\xf7n?O\x8b\x87z\x8d\x7fa]\xce\xc2C0\xb2H\xa0\x81\xb7\xdd\xf98>\v\xab\xb3\t\xad\x82$\x8c6\x1dX\xb3\x1cn\x9aB\x94g\x90\x03\xbd8\x868\xa3ܥy\x8eu.\x94\xdfL\xf0(\x13da\xaaih\x8dݹ\xe0\x82\xe2R\xcb\xffXO\x8b\xda\xf4\x7f\xa4\xa4L\xe9\v\xf2\x0eKZ8t\xde\xf9\xa4Y\vLB\x97X\x92b\xe5gG\xb9\xf5\xfdր\v\x02\xdcE\x02r\u074b\x8b\xe6\xe4q+\xb5s\xdb\xf5\"\xce\xec\x01\xf6n\xc5p\xb4˶\x91\x99-\xc5\xcc\xc5\x10=\x83Q\a\x1cR\xf0=\x99\xe1\xbb\xd9sB\xa9DIMl\xd6\x11т\x96i\x12\x8a%E\xa9\x81\xba\x9d\xb0\x86 \xc4~X\x97\xca\xd8 \xfb\x18\xb6I\"ZJ\x1dY\xc8\x1f\x18ʈ\xf0\xdeHm\\\xbe\xac\x133G\x13j2$\xd1\b]\xbb\xfa%\xa9B\xb1\x895\xcac\xa9\xdf\xf6s\xb7\x05\r~\xbd\xc2'\xe6\x1cP;\xb3\x9b5\xfa\xed\xac\xfḓ\x97`'4È\x05\xbf-\x95\xcc@Gײ\x9b'\xc1_D\xaa2ڸ\xd79G\xeafI\xae$\xe3x\n4<\xe9!\xaf%\xc4\xc4\xf9\u0087\xa7VB\xd4\xea\xbe\xfd{LƦ\x8e\x8b`\xc9`Q\xd0\xc32\xa5\xa4!^\xbb/\x836x@n\xf2\xa16\x15Z\x82T_^\v\xe0\xd7\x10(\x14L,\xb1\x03\xf2\xf6\x05\x02\voCc\xc5&\xb1\xe7\xb4P\xf6:t\xd2p\xa7\xfe\xc1\xa9r)q\xa9@A\x87y\xfd\xac:ơB\x9aVBbB\xb8Y\xca\xfc;M\xd6Li\xd3\x1e\x82\x1e(S\x89\x82\x998\xf1\x12\x1f\x94:i\xde\xf5\xd9}\xd9Jwm\xe5c(\xcfr\x84I\xc4\x1cח\x80\xb05a\x86\x80\xc8d%0\x81c\xf5\x18\xbbp\xc4u\x16\x96\xa5*I\x9a\xf6\xdb\aDU\xa4\x11`\x81\x92\xc2\xc4\xd1LO\xbb\xf9G\xca\xf8K\xb0\xcd\fU\xb1Ş\xd3t\"\x94\xb8\xb5\v\xf2\n\xfaĊ\xaa \xb4\xb0\x1f\xc0\xb0\x8c\x0f\xa1\xdd+\xc5\xc8E\xc5\r+9.\xa4\xeeX\x1eM6\x98-\xec\xeb\x034~\x95\xb8\xf5ԟ\x04\xf3\xf9K-\xb5\x17\a\x91>\xd5\xe4\x118'4\xa6W=\xcc3w\x12S&\x17`\xfd\x91\xd5N\x7f0\x88?\xbei\xee\xc4\x1dwעW+b)&*\x86O\x91\x19t\x1c)\xf6\xa6\x17\xc1\xba8\x1c\x7f\xfb\xad\x02\xb5'x\x8eM\x1d\xe74\x9b\xc0\xbcbj;\x11\v\xa6\u009b\xad\xa1\xfcy/\xe8oT\x99\xbc\x13\xce\xeb\x1e\x8e\a\xbf\xb16\xa2\x99\xd4X\xc3g\xe7+\xd1>\x06>\x17\xb2\xfe:\xf2\xd9X\x80\x9c\xba[\xeae\xa78\xd3'9\xa3QEz\xe4\xf7;\xed\x82:e\xf7SZ\x01\xc0\xe8n\xa7\x97\x9a\xf2\x8cMz\x92㼴\xddL\xd3\x16\v_p\xf7\xd2K\xecZJ\xa4T\xca.\xa5itz\x85]I\xaf\xba\x1b\xe9\xb5v!%\xef>J*qI^\x05N-Q9q;\xcd\xf8\x1a\xef\xf1\xddD\t\xbb\x88\x12V\x7fǑ<\x01\xbd\x84]B\xd3v\a%\xf0,U\x15_q\x17\xd0+\xee\xfey\xed]?#\x925\xf2z\xda\ue793\x97,\xa4\xcaA\x1d]\xf6I\x95£\xf2\x972\xb7\xe9\x0e\xe4`\xbd#\x9c\xfag[u\xe2et\x0f\xfe\xa0Q\x97G\xb3\xa2\xe4{;C!\xb3\xf6\a\xa7I@T\xdaBo7\x92\xb3,\x12\xbbE\xcffr\x8d{\x87e\xe0\x89QY\xbbd\xa0\xb4\r\xe3\xa1\x1b\x86y\xdd#0גs\xf98q\xeeOK\xf6\x17<\xb9\xfb\x19١w7K\x84\x11\xc4\x03\x8f\x02\xaf\x8b\xb3jlV`\xddr\x83\xe7\x90\xee/\xd7\x1d\x88\xdd:\xc7\xf6Ḑ\xbbs\x90CX\xe0Mg&\xadu\xb9Y\xbaq\f\xf5be\x86\x8a=\x91XQc\xb6L勒*\xb3w\x85\x1a\xf3\xce\x18\x82/=\x96\xdd\x19\xf4\x1e\xfd\xb3\x9d\xa3\xe4\rG:\xe3\n\xe5\xbe\xec.\xfa\x1e\xd2\xee\x94q\f\xef^\x1cݷx\xc6q\f\x87%\v\xa4T\xe4\xe7h\xe5\xd7ٲfڟL\xfc\xb3\xdc\xc1\xfbh\xf6\xacC\x9eۃ\xe6\x91\xf2\xac\x00\xd1\x1d\xba;X\xa5\xba\x02<\x90\xb7\xff\xea\x19\xf5V\xa1k\x7f\xa6\xea)\x89\xb2\xdb.\x88\b~\xe1\x84\xd9\xd0Y\xcc>\xe1\x01\xf0{rs\x8fs\xb4ڴy\x15\xf5s\xb4\x90*\v\x8b\xc1\x118\xfe\x83\xef\xcf_\x9a\xa6\x8dTt\x03?Iw\xc6\xf6\x18ۻ\xad;g\xaf\xfb\xa8'ԏ\x06\xa5\x89\x1d\xc0\xebO\xfb>\x00\xd6\xd4|\xf7\x0e5\xb6\xa3\x9cxL\xb31\xfc\x14\xbe\xdf\xdd\xfd\xe4\xb02\xac\x80\x8b\xf7\x95+w\xb06Q\x83%q\xc0\xd6AZ\xd9\xffn\xe5#\x1e\xfe\x1b\xcfc\x86;\x13\x1ad\x14`\xb19\x96 NB\xa9*\xb9\xa49\xa8k)\xd6l3\x82\xdd/\x9d\xc6\an6\xc3\x1f=r\xb5\x8f\n\xf0\xcf\\\x83`c\x1e\u0381\x7fd\x1c\xb4\x1bV\x82\x01\xbe\xe9\x7fU\xdb\xe3\xaaX\xb9\x18nm_\xd6\x1d\f\xf88\x87\x16\xa6\xa2KP6\x8arI\xebJ\aY\x1dF\xbc\xe1\b\x13\x066П\x05\x1e\xb1\xc0\xeeTit\x9f\xc1\x9c\xe0\\\xe6\xc7X~\xab\x83\xfc\xfd\xf0\x97\a\x9cl\xa5\xbcb'\xee\xb9 \xe4\xe6\xfeZ\x93J\xe4\x98.\xbe\xff\xcb\xed$\xa9\xdbuN\xae\x0f\xda:fT\xef\xe3_\xb5\x82㖽pѱ\\G\x10\x18\x82Ӻ\a\xe4\x91\x19\x7fp\xd7yOZ\x1d\x9a\xf2\f\xddp\x80G\xfa\x8f\xdfq\xe0N\xfe\xf77\xa3xu\xac\x14\x1e\x93\xeao\x05\xc0cEO\xba\xe6`U\x17l\xd5\xc5_\xfa\x9d1P\x94&\x16k\x8c\x9b\xc3\xef\x8f\x01\xac\xe34i(oi%\r\rb\x91\xb6ދ\xecXa\x99\xb7FG\xb8yL\x1fc\x04\xb8\xf6\xfb!\xceF\x80\x1a\xe0\x10\x01t\x95e\xa0\xf5\xba\xe2|_o\xc7\xf8J\xa8\xf1\x912~>R8h\x83\x82`\xd1;\ni\x14a_\xee\r\"\x0f\x9a\x1e\xb6*M#\x85炯\x86Ԇ\x16']\xd8p\xdd\a\x83W\xf6\xa8\xbcUTI\xeb\xb1Sݰ?\xe6\\\x1ap\xeeK\x9cdYh\x90\x13\u0601 \xd6;;\x12\x87;\xa7&B\xf1;\\\x9d\x87\v\xfe.\xa4B\xa2\x17\x13\x11\x9f\xed\xd0x\x01\xcew\xba\x86\x89\xb5\xa2x\x9fI\x9f\b\xfd\xe0\xd7e+\xael\xf4\x0f\v\v\u2d285j\x9b3ͺ~\xe1yF\xee\xfav9\x04\xee\x14\x13\u05ff\xee\xe5\x99j\xdcG\xf7Y&\xad\x8f\xee$\x83\x16\x81X\xcb\xf8\xf9qGU?\xedPw\xfc\xd2\x05\x1cY\xd8CG9\xf7\x1b\x1d\vКn\xc2i\xee\x8fv\xea\xb1\x01\x01.=\xe7\x16O\"@\x9b]qݳ̝\xca\xd0\xccT\xd4w\x10\n|[\xad\xbeӄ\xcb\x18T\xbcЅ\x85\x9b\xc2\u009cl\"\xa1\x9eJ\xa6R\xe6p\x1fꆖ6\x18\t#w\x9a\xbb݀\xb3\r\xb3s\x1d˹\rU+\xba\x81E&9\a\xb4\xd6\xfdq\xbd\xa4\xae\xfb\xbd\x87_\x80\xeaQ\xd4>\xb6\xdb\xfa\x15@\xc7m\xb7\xf0M]\xb9;\xde\xdee\x98\x82\xe6\"\xbdހ$v<)PvT\x88\xde2\xd7\x1fi\xbbm\xd0:o\x96}\x9e\xd7_27\xf7y\x81\xb8<\x16\xf4W\xa9\xe6\xa4`\xc2\xfeCE\xee\x16\xf0\xc2Ǔƿ\x95\xf2\xe16\x12\xc4\xf6\x06\xffCݰY\xea`\xc2\r\x1b7\x8c\xaed\xe5W\xdf\xeb\x806\xbe\xac\x82'\xf3\x9fy\xba\x890\x8f\xf8\x83\x1e:\x83\x19\xdd\x1f:\x90F]\x81\xeby\x00\xd6m\xb8Ɍ\xf3\xfd\xfc\x10\xf2\xc1\xad\x89\r\xec\xd6\xcd\x05>\fh\xce#\x18\xe8(\xacHE\x81\xd4\a_\xb4\r\xfa)\xb3^O\xe6\xa1`\xb2G\xe3\x1f\x9a\xd6Ctt\xc3l\x85{\x03\bv\x82\xc0\xf3N\xd8\xf1\x9a\x8a\x11\u1ff1m\xea\xb3\vZ\x13\xb7P%6\x98\xa5\x8b\xef}_\x90O\xd0_\xaeX\x90\xbfVPEh\xb0\b\x17\xc3\xdd\x1a\xaa\xfa)_\xb7\r\x1er\xac\xe8@m\x8c4Y\x8a\x1b%7\nt_X\x17\xe4o\x94\x19&6\x1f\xa5\xba\xe1Ն\x89\xcf\xc3[~\x8e5\xbe\xa1\xca0+\xecn<\xb1\x812A9\xfb{̮\xb5_\x8e\x03\xba\x1e\x9c`-H\xc20\x86^\xbc\a\x1b\xe3\x0e\xe6\x05\xa2&\xb4\xf4t=%^\t<\x19\xb3\xa9u,\xd1\xc4\"\xa1\xdb\v\xf2IF\r\x83/\x87b]\x986$\x03m\x16\xb0^Ke\xdcj\xf5bA\xd8:$\x1f\xac\xcd\xc1\xbc\x99\xbb\xab\x92\xb0\xd82s]hҸ/Lz+\xf4\xc2x\x94}A\xf7ne\x8afYe#\xacKm(\x8f\x048\xcf2\xfc\x98\xe5\xb1\xca\a\xf9/\xcfZ\xc9[\xb6\x01\xf5\x93\x8e؏#)\x1e\xa6\xe1\xa2>nQ\x04A\x1e\x153\xc6\xc6T\xf2H)\x81'\x95\xb1\xb1\x15\xe7D[R\x9f\x94}$Ό.\x87Kr\xd2P\xbe\xab\xa1\f\x99g\x8f5\xde̸B\xda\x10\x1b\xf7b\xf5\x91oeٜm\xa9\xd8\f\x9eP\xb0U\xb2\xdal\x83$\x0f\x04\xd3$\xaf\x00\x93\xb5hRt\xb8X\xd8TJ\xb4J\t\x8el\xfb&A\x18p\xb84{ U9\xf7\x17\xf7\xfa{\x99/\xfd\x1d(\x8b\xb5\x92\xc5\xc2\xf7\x8b\xb9Թ_\xc9WL\xda\xc8\xc5l\xa3T'.j\xf7\xd7\f\xa0$\x94%\bB\xb5\xef9ᤨ\x93\xdd\xd4o\xd65\xdcH\xcd\x12\xa2\xfd(\xc7\xff\xda\x06\x10\x18^\x86\xbf\xbb\xcc\xf03\x18\xec3\x86\xc7g\xbf\x05\x1fvT\x187\x9d\xa8]\xe4\xcc9\xb1٤\x89\x8c\xb6\x8e\xedYI\x9a\xdb\x0e\x84\x91\xfc\fv\x17gѭ/\xd7p\a\x81]\xfb\xebWk\xc0s\xa2\x99\b\x17_\xbb\xd2\x0f'\xfdѕ@\x81\x17UJ\x15\xaf\xc6<\x9ep\xe9\"\xf4\xba\xb9\x96]\x1dI|8y*~\x7f\x00\xe3`S7\xdeKZ7\t\xd3\xe7?\xb0\xd8z\x00\x96\xf1f\x16\x95?\xfe\ue6f5wIS\xbd8E\x8e\xcd\xfcpR7<\x85\xeb\xdeCz\xc3\xc1j\x9b\x06\xe8N*'\xe9\xdc\xee\x8cٴs\xa6\xd2\xc2\x15\xef\xe7\xc9%\xedΘD{\xb1\f\xdayQ~\xa4xA\xf4IZ\xfb7\xffm$\x85\xe6\xc1\x9e;\x89\xd6ʡ\x85\x81\xbfj\x16-\xeas{?\xa2\x9d\xce[\xd6\xc2\xf7\xe4\x7f\xf9\xff\x00\x00\x00\xff\xff<\x82OF\xb8\x82\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=]s\x1c)\x92\xef\xfe\x15\x84\xeeav7\xba\xe5u\xdcG\\\xe8\xcd#\xdb;\x1d3ck-\x8d\xf6\x99\xae\xca\xeefDA\rP-\xf7\xde\xdd\x7f\xbf \x81\xfa袪\xa8VK\xe3\xdd5/\xb6\xba !?I\x92\x04\x96\xcb\xe5+Z\xb2{P\x9aIqEh\xc9\xe0\x8b\x01a\xffҗ\x0f\xff\xad/\x99|\xbd\x7f\xf3ꁉ\xfc\x8a\\W\xda\xc8\xe23hY\xa9\f\xde\xc1\x86\tf\x98\x14\xaf\n04\xa7\x86^\xbd\"\x84\n!\r\xb5?k\xfb'!\x99\x14FI\xceA-\xb7 .\x1f\xaa5\xac+\xc6sP\b\xa2\x1d\x8a\x1cB\xb5\xc4[{:\xae\x0f\xf81\xa6\x06\xae\xac6-\x88L\x93\x8b\v\"\x15\xb9p\xae\xc9\xc5µ\xae\x187K&\xda}<2\xceC/\xf3\x90w4t\f\xd5w\xf2\x83v\xcat\x12-\x06`\xb5H\xf3\xb8\x03\xb3\x03EJY\xbb\x02\x1bƁ\xe8\x836Px\u0084\xe9\xd5\xe3\x13\xe9\t\x8d\n\xe7\x1e\x84\xb6t\xf5\x88\xf4\x91\x17\x15\xe7t\xcd\xe1\x8a\x18U\xc1\x00m\xd6Rr\xa0b\x828\x9fA\x1b\x96\x9d\x834\x0eR\x840\xca\x7f\xe8P\x00\xbd\t\xfa\x00\x84F@{\x9aY\xb7\x85\xf3\x16a\xbbT\x89\x8e\xa9T\x90\xd9\xe9\xec\xcaO\x93\f8N\xcdB\x12.\xc5\x16\x94\xeb\xddZ\xbd `\n\xac\xc0\xe5\xc4\xce@\n\xb8\x9dfɦ\xb2\x93\xd3%\xb1\xda=(\x03Lh\x034\"\x9cO\xe0\x0f|\xb1\xd6\x19\xf2k\xe7\x91\xdeZ\xc7:\x0fˉ\xdet\x92§\xf7\xa3\x10\xbd\xdb\xc2Y\x86ޱw\x84\x97\xe8\xd0\xc7Ĵ\xf1^\xecԄk\n\xcbJ?\xec\xc6-\x19\xb5\a\x1a\x8cmt\xf1\xa7\x8b\x05r\xb8\xdbk\xb7\x0fM\xa8\x82\x9a,\xc9v\x13\x8a\xd2\x1c\xfa\xb5\x99\x81\"B\xc5Q{\x92\xc8O\xaa\x14=\fp\xb3^\x18\x9d\x91\x9fC0\x8f8*B\xb5\x17\xe6\xe9q\xbf\xff\xcc\\=\x0f\x1f5\x86\x01(\x13\x96\x7fvE\xdea\x9fv\v[K6!M\x04\x9e\xf3\xeb \xc75\xeb\b\xb7~'b\x9dE懄\xbc\x96-/\xbc\xff\x90\x94\xdaI\xf90E\x9d\x1fl\x9df\xb5H2\f7\x915\xec\xe8\x9eI\xe5Qo\xa6Z\xf8\x02Ye\xa2ZO\r\xc9\xd9f\x03\xca\xc2)wT\x83v\xf1\x83a\x82\f\xafkHˌD?\x1e\xe1\xd10Ҳ\t1\x1f\x1a\xba\xf5#\x8eg\xc9P\xec@\xad{\x8d\x93q\xce\xf6,\xaf(\xc7y\x99\x8a\xcc\xe1C\xebqŬ\xcc\b\x93{c\x8eJ\xa6+\xce!\bHY&u\x96\x90R\x80\xf5y\v\xbb&\xe8W\x1d\xc6|M\xad\xaf\"\x87\xb0'\xc8,Uqо\xab\x1c\xdd\xc8\xc6f,\x1a\xa6`\x84\x86p\xba\x06N4pȌTq\x8aL\xf1ٕ\x14#8@Ȉ\xe5\xeb\xae4\x1a\x04F@\x12\\\xc2\xedX\xb6s\xae\x9e\x15\"\x84Cr\t\xd6\xe13\x84\x96%\x8fL\x17M\x19e\xbe\xefdLכ2\xa1\xf5\xc7\xf0b\xfaߔ\x04\x9bٔ(i\x1b\xfd\xeaR\xb6\x16\x87\xf8\x9a\xb6)\xff\x9c\x84\r\x96\xff\x04\xa1\x1d\xd1~\x82\xe1\xb2d\x99\x1e\x94[KU\x06\xfaҺS\xe8\xe9,\b3\xe1\xd7)M\xe8\xf8\\\xbd(b\x87\b_7o\xe6\v}\"kRt\xe2\x99\x18Sw\xf1\x0f\xc8\x17\x9c2n\xfd\x8c\x91̓\x9fڭ\x16\x84mj\xa2\xe7\v\xb2a܀:\xa2\xfeI\xa6>p\xe6\x1c\xc4H\x99\xf5\b\xeek\x98l\xf7\xfe\x8bu\xc1t\xb3\x85\x97H\x97\xe3\xc6Α\r\xde~wz\x9e\x80K0\xbe\xcf\\\xb4U_⊩\xfd\v\xbaVo?\xbe\x8b\xaf\xaf\xda%A\xf2z\x88L(\x9d+o\x8f0j\x8fϻ\xf0\xe1\v\xfa@\xf5\x02\xc8Ū\x17\x84\x92\a88ׅ\nb\xf9CC\xe5\x84\xee\x15\xe0f\x15\xca\xd9\x03\x1c\x10L|\xf7\xa9_R\xa5\xc1\x95\a8\xa4T;\xa2\xa1\x1d\x13\xd3~W\xcd\xd2\xc9\xfe\x80\x84\xc0M\x87T1pūBd\xaf'^\x12mI(\x81\xf6'\xa0\x99$*\xed>\xda۷(\x01\xdfi\xc7K\xab1;V\xa2Yň\x83\xdc$3ԕ{\xcaY^w\xe4td%\x16\xe4\xa34\xf6\x9f\xf7_\x98\xf6;\xbc\xef$\xe8\x8f\xd2\xe0/\xcfBQ7\xf0\xe7\xa4g\xd8\xf1\xb1\b9+o\t\xd6ޣts\x9a\x95\xb6\x9a\xf6L\x93\x95\xb0\xcb\x15G\x92Įp;\xdau\xe7:**\x8dۋB\x8a\xa5\v\xdb\xc4z\xf2\xf4\x96\xaaC\xee'w\xea;\xbc\xb3\x93\x85\xfb\xe26\xc59\xcd \x0f\xdb5\xb8[K\rlY\x96\xd8_\x01j\v\xa4\xb4&\x8d\xcf\xd2\xealB\xad \t\x93U\a6s\x87\xab\xa6\x10\xe5\t\xe4\xc0Y\x1c]\x9cI\xee\xd2<\xc7\x04 \xcaof\xcc(3da\xaeih\x8d\xddM\xc1\x05ŭ\x96\xff\xb13-j\xd3\xff\x91\x922\xa5/\xc9[\xcc\xf5\xe1\xd0\xf9\xe6\x83f-0\t]b\xae\x8e\x95\x9f=\xe5v\xee\xb7\x06\\\x10\xe0\xce\x13\x90\x9b\x9e_\xb4 \x8f;\xa9ݴ]o\xe2\\<\xc0\xc1\xed\x18Nv\xd962\x17+q\xe1|\x88\x9e\xc1\xa8\x1d\x0e)\xf8\x81\\\u0dcb\xa7\xb8R\x89\x92\x9aX\xad#\xa2\x05-\xd3$\x14s\xadR\x1du\xbb`\rN\x88mX\xe7\x10Y'{\f\xdb$\x11-\xa5\x8el\xe4\x0f\feBxo\xa46.^\xd6\xf1\x99\xa3\x015\x19\x82h\x84n\\b\x97T!\v\xc7\x1a\xe5\xa9\xd0o\xbb\xdc\xed@\x83߯\xf0\x819\aԮ\xec.\x1a\xfdv\xd6\xfe\xc2\xed\x97`'4C\x8f\x05ۖJf\xa0\xa3{\xd9MI\x98/\"Y\"m\xdc\xeb\x98#u\xab$\x97\xab2\x1e\x02\r%\xdd嵄\x98\xb9^x\xff\xa5\x15\x10\xb5\xbao\xff\x9e\x92\xb1\xb9\xe3\"\x98KY\x14\xf48\x7f+i\x88\u05eee\xd0\x06\x0f\xc8->ԶBK\x90:\x97\xd7\x02\xf858\n\x05\x13+쀼y\x06\xc7\xc2\xdb\xd0X\xb2I\xac\x9c\xe6\xca^\x87N\x1a\xee\xd4?8U.%n\x15(\xe80\xaf\x1fUG?TH\xd3\nH\xccp7K\x99\x7f\xa7Ɇ)m\xdaC\xd0\x03i*Q03\x17^\xe2\xbdR'\xad\xbb>\xb9\x96G\td>o\xcd\x11&\x11s\xdc_\x02\xc26\x84\x19\x02\"\x93\x95\xc0\x00\x8e\xd5c\xec\xc2\x11\xd7YX\x96\xaa$i\xdaO\x06s\xd0be\x89\x92\xc2\xc4h\xa4\xa7]\xfd\x03e\xfdD\xb5X\x99\xc963\x94\xc5\x16+\xa7\xe9DHqkg*\x16\xf4\v+\xaa\x82\xd0\xc2\xf2\b'sV@\x97\xe9M\xe2\x9bm\x81ӄ\x91VcJ\x0e\x06|\xf2Z\xe2\x182)4ˡ\x9e\\\xbd HA(\xd9P\xc6+\x95h\x01g\x91w\xceR\xc4[\x82\xf3\xad1\xd2:_\")\x12\xa2\xb9\x89\xbe\xe2\xb85.U\xba\xc77\xe5f)\x98\xefe\x95\x8aIL\v<\xb3\xa3\xe5\x13)\xa98|\xf3\xb4R\x87\xfa\xcd\xd3\x1a+\xdf<\xad\x89\xf2\xcd\xd3\xfa\xe6i\xa5\xd4\xfc\xe6i}\xf3\xb4\xda\xe5_\xc2Ӛ\x1a\x91;\xe88\xf0qr\x14\t[\xd5cC\x1c\x81\xef\x93+|\x0e\xf8\x93r1WqP\x91\xc4\xff\x81\xb4\xee\x98\xd1j&\x8f:9\xd3jM\x90yw\xeej\u0095|B\xd6}\xe8\xf4|Y\xf7\xabQ\x88gʺ\xf7Þ\xf6\xb1Oʹ\x0fD\x99\x97\x9d\xbd\xf0\x89\x1a\x05\xd0\x10Vw\xdb\xf01\xbc\x86$d\xa2\xff\x17N\xcc\xede\x8d\x9dQ>\x9e=\x8b?YF\xa2,\xbd\xf8\xd3\xc5\xd7G\xfe\xf3\x10|\x90\xc4}\xda\xf9\x83\xdf\x11\xa8v\x05\xdaN\v\xebf\xe1}\x9db|\x16\xb9M\xcdį\x89\x18\x81\xd5\x15\xc9#*~\xad\xb6\xc0@\xf1\xa9\xf43\xd2\x13N\xaa\xae\"p\x92ΪR}\x10\xd9NI!+\xed\xa3\x12\x16\xd6\xdb̝\xf4\x0f c\xc2\x1a\xd5\xf0\xff ;YE2\xc1G\xc87\x91\x118\x8d|'9\xd0oB\x83\xa1\xfb7\x97\xdd/F\xfaT\xc1\xa1\xb3͏;\x10\xb8\xc3.\xb6\xed\x03\x00\xe1\xa2\x06\x7fc\xc1\xb1\x80E\x00IE\x04\xe3N\xf2\xeak\x1e\xdarG>\x95.\xf64\xdb\xef\x18\x8f\xa9\xa4%\x13\x9e\x9cB\xd8M\x11\x1c\xf0K\xe7\xeev\x9f\xe5\xc8\xc4\xef\x92\x1a8?!0%\"6\x91\xfcwB\xca_bn\xf1\x93\xb7\xe7S\x92\xfa欘\x9f-\x81\xef\xfci{I\xf4\x99NћC\x9dgO\xc7{\xc1$\xbc\x97I\xbdKL\xb8;_\xe6|Z<\xf6\xa4̱\xe9\xd0\xc1p\xd2\xdcd\xaa\xdcdha\n\xb1\xd9(M\xa6\xc0\xcdI|\x9b\xe4N\x9a\x9a\xbdXjۋ%\xb4\xbdl\x1aۨ\x14\x8d~\x9c\x93\xa8\x16\xbf\xaf\x87LN\xb6\xfc\xa5\x84\xedT2H\xd5q_OZ_}:\x82a\x19\x1f\\\xbb\x17\U000912ca\x1bVr\xdcHݳ<\x1al0;8\xd4\x17h\xfc*\xf1詿\t\xe6\xd3\xe7Zj/\x8f<}\xaa\xc9#pNhL\xafz\x98g\ue2aaL.\xc1\xceGV;\xfd\xc5 \xfe^\xab\x85\x13w<]\x8b\xb3Z\x11\v1Q1|\x8b\xcc\xe0đboz\x1e\xac\xf3\xc3\xf1\xb7\xdf*P\a\x82\xf7\xd8\xd4~Ns\b\xcc+\xa6\xb6\v\xb1`*\xbc\xd9\x1a\x8a\x9f\xf7\x9c\xfeF\x95\xc9[\xe1f\xdd\xe3\xf1`\x1bk#\x9aE\x8d5|\"vq\x13\t\n\xd6o.d\xdd:\xd2l\xcaAN=-\xf5\xbcK\x9c\xf9\x8b\x9cI\xaf\"\xdd\xf3\xfb\x9dNA\x9dr\xfa)-\x01`\xf2\xb4\xd3s-y\xa6\x16=\xc9~^\xdai\xa6y\x9b\x85\xcfxz\xe99N-%R*\xe5\x94\xd2<:\xbd\xc0\xa9\xa4\x17=\x8d\xf4R\xa7\x90\x92O\x1f%\xa5\xb8$\xef\x02\xa7\xa6\xa8\x9cx\x9cfz\x8fw\xfc4Q\xc2)\xa2\x84\xdd\xdfi$O@/\xe1\x94м\xd3A\t\xa9R8*\x7f)k\x9b\xee@\x8e\xf6;\u00ad\x7f\xb6V\xc7_\xc6\xe9\xc1\xdf\xc0\x8aw\xed\x0em_ZIky\x1b\x9d\xbd\xa8\xc6\xfd\xe9:\x93\xfe\x02^\xb7]\xa5\xa1\xa4\n/u^\x1f\\:Ktj~O\xb3\xdd\x11\xf4\x1d\xd5d#UA\r\xb9\xa87\x00_;\xe0\xf6\xef\x8bKB>\xc8:'\xa2}/\x8ffE\xc9\x0fv\x85B.\xda\rN\x93\x80\xa8\xb4\x85\xden$gY\xc4w\x8b\xde\xcd\xe4*\xf7.\xcb\xc0\x1b\xa3\xb2v\xca@i+\xc6]7t\xf3\xbaW`n$\xe7\xf2q\xe6ڟ\x96\xec/x\xa5\xf9\x13\xa2CooV\b#\x88\aޑ^'g\xd5ج\xc1N\xcb\r\x9eC\xba\xbf\xdat v\xf3\x1c۷\x06C\xee.\x88\x0en\x817\x9d\x99\xb4\xd6\xe5f\xe5\xc61ԋ\x95\x19*\x0eDbF\x8d\xd91\x95/K\xaa\xcc\xc1%j,:c\bs\xe9Xtgp\xf6\xe8_z\x1d%o\xb8\xeb\x1aw(\x0few\xd3\xf7\x98v\xa7\x8cc\xf8\xf4\xe2\xe4\xb9\xc53\x8ec\xd8-Y\"\xa5\"?G3\xbf\xce\x165\xd3\xfef\xe2\x9f\xe5\x1e\xdeE\xa3g\x1d\xf2\xdc\x1eU\x8f\xa4g\x05\x88\xee\xd2\xdd\xc1,\xd55\xe0\x85\xbc\xfdOOȷ\n]\xfb;UO\t\x94\xddvAD\xf0\v7̆\xceb\xf6\to\xc6?\x90\x9b{\\\xa3զͫ\xa8_\xa3\x85PY\xd8\f\x8e\xc0\xf1\r\xbe?\x7fj\x9a6R\xd1-\xfc$\xdd\xe5\xe3Sl\xef\xd6\xee\\J～\x90?\x1a\x94&v\x01\xaf\xbf\x06\xfd\bX\x93\xf3ݻ\xd4؎r\xe65\xcd\xc6\xf0S\xf8~w\xf7\x93\xc3ʰ\x02.\xdfU.\xdd\xc1\xdaD\r\x96\xc4\x01[\aim\xff\xbb\x93\x8fx\xf9o<\x8e\x19\x1e\x93h\x90Q\x80\xc9昂8\v\xa5\xaa\xe4\x92栮\xa5ذ\xed\x04v\xbft*\x1fM\xb3\x19\xfe葫\xe7\xa8\x00\xff\xcc9\b\xd6\xe7\xe1\x1c\xf8\a\xc6A\xbba%\x18\xe0\x9b~\xab\xda\x1eW\xc5\xda\xf9p\x1b\xfb\xb1\xee``\x8esha(\xba\x04e\xbd(\x17\xb4\xaet\x90\xd5a\xc4\x1b\x8e0a`\v\xfdU\xe0\x88\x05v\xb7J\xe3\xf4\x19\xcc\t\xaee~\x8cŷ:\xc8\xdf\x0f\xb7<\xe2d+\xe4\x15\xbbq\xcf9!7\xf7ךT\"\xc7p\xf1\xfd_ngIݾss}\xd0\xd6)\xa3z\x1fo\xd5r\x8e[\xf6\xc2y\xc7r\x13A`\bN끔Gf\xfc\xc5]\xe7\xbdiuh\xc93\xf4\xf4\x03^\xe9?\xfd\xf8\x83\xbb\xf9\xdf?\x19\xe3ձRxM\xaa\x7f\x15\x00\xaf\x15}\xc2\xfb\x0f\x9d\xe4/\xfd\xd6\x18(J\x13\xf35\xa6\xcd\xe1\xf7c\x00k?M\x1a\xca[ZIC\x85\x98\xa7\xad\x0f\"\x1bK,\xf3\xd6h\x84\x9bc\xfa\x18#\xc0\xb5?\x0fq6\x02\xd4\x00\x87\b\xa0\xab,\x03\xad7\x15\xe7\x87\xfa8\xc6WB\x8d\x0f\x94\xf1\xf3\x91\xc2A\x1b\x14\x04\x8b\xde(\xa4I\x84}\xba7\x88\x1bR\x1bZ\x9c\xf4`\xc3u\x1f\f\xbee\xa4\xf2VR%\xad\xc7Nu\xc3\xfe\xd8\xe4Ҁs-q\x91e\xa1AN`\x0f\x82\xd8\xd9ّ8<\xc65\x13\x8a?\xe1\xeaf\xb80߅PH\xf4\xc5&\xe2\xa3\x1d\x1a_\x06\xfaN\xd701W\x14\xdf3\xe9\x13\xa1\xef\xfc\xbahŕ\xf5\xfeaiA\x9c\xe6\xb5\x0e\xbd\xe6ҝ\x17\x9ef\xe4\xaeoWC\xe0N1q\xfd\xe7^\x9e\xa8\xc6}t\x9fd\xd2\xfa\xe8\xce2h\x11\x88\xb5\x8c\x9f\x1fwT\xf5\xd3.uǖ\xce\xe1\xc8\xc2\x19:ʹ?\xe8X\x80\xd6t\x1bns\x7f\xb4K\x8f-\bp\xe19\xb7y\x12\x01ڜ\x8a\xeb\xdee\xeeT\x86f\xa6\xa2\xbe\x83\x90\xe0۪\xf5\x9d&\\Ơ\xe2\x83.,<\xa1\x16\xd6d3\t\xf5\xa5d*e\r\xf7\xbe\xaehi\x83\x9e0r\xa7y\xf4\x0e8\xdb\xe2\x93N\x96s[\xaa\xd6t\v\xcbLr\x0eh\xad\xfb\xe3zN]\xf7g\x0f?\x03Փ\xa8}h\xd7\xf5;\x80\x8e\xdbn㛺tw|\xd6\xcc0\x05\xcd\v\x83\xbd\x01I\xecx\x96\xa3\xec\xa8\x10}~\xaf?\xd2vݠu\xde,\xfb8\xaf\x7f}oѼ\xa8\x15\x19gA\x7f\x95jA\n&\xec?T\xe4n\x03/4\x9e5\xfe\x9d\x94\x0f\xb7\x11'\xb67\xf8\x1f\xea\x8a\xcdV\a\x13n\xd8x`t-+\xbf\xfb^;\xb4\xf1m\x15\xbc\x99\xff\xcc\xcbM\x8492\x1f\xf4\xd0\x19\x8c\xe8\xfeЁ49\x15\xb8\x9e\a`݆'\xde8?,\x8e!\x1f='\xd9\xc0n\xbd\\\xe0݀\xe6>\x82\x81\x8e\u008eT\x14H}\xf1E۠\x9f\xb2\xea\xf5d\x1er&{4\xfe\xa1\xa9=DG7̖\xbb7\x80`\xc7\t<\xef\x82\x1d\x9f\xa9\x98\x10\xfe\x1b[\xa7\xbe\xbb\xa0\xb5p\vYb\x83Q\xba\xa1\x97\xee>B\x7f\xbbbI\xfeZA\x15\xa1\xc12<\fwk\xa8\xea\x87|\xdd1x\xc81\xa3\x03\xb51Re%n\x94\xdc*\xd0}a]\x92\xbfQf\x98\xd8~\x90\xea\x86W[&>\r\x1f\xf9\x19\xab|C\x95aV\xd8\xddxb\x03e\x82r\xf6\xf7\x98]k\x7f\x9c\x06t=\xb8\xc0Z\x92\x84a\f}x\a\xd6\xc7\x1d\x8c\vDMh\xe9\xe9z\x8a\xbf\x12x2eSk_\xa2\xf1EB\xb7\x97䣌\x1a\x06\x9f\x0eź0\xadK\x06\xda,a\xb3\x91ʸ\xdd\xea咰M\b>X\x9b\x83q3\xf7\x88'a\xb1m\xe6:Ѥ\x99\xbe0\xe8\xadp\x16ƫ\xec\vzp;S4\xcb*\xeba\xbdֆ\xf2\x88\x83\xf3$ÏQ\x9e\xef\xf1\xc1\xca_\x9e\xb4\x93\xb7j\x03\xea\a\x1d\xb1\x1fGR\xbcL\xc3y}ܢ\b\x82<*f\x8c\xf5\xa9\xe4H*\x81'\x95\xb1\xbe\x15\xe7D[R\x9f\x14}$Ό\xae\x86Sr\xd2P\xbe\xab\xa1\f\x99g\x8f5\xbe\xccX\xbf\n곏|-\xcb\xe6lG\xc5v\xf0\x86\x82\x9d\x92\xd5v\x17$y\xc0\x99&y\x05\x18\xacE\x93\xa2Ë˦R\xa2\x95J0r\xec\x9b\x04a\xc0\xe1\xd2\xec\x01\xdf/u/\x1a\xfb\a\xab_\xfb7P\x96\x1b%\x8b\xa5\xef\x17c\xa9\v\xbf\x93\xaf\x98\xb4\x9e\x8b\xd9E\xa9N\x9c\xd7\xee\x9f\x19@I(K\x10\x84j\xdfs\xc2MQ'OS\xbf٩\xe1Fj\x96\xe0\xedG9\xfe\xd76\x80\xc0\xf02\xfc\xdde\x86_\xc1`\x9f1<>\xf9#\xf8\xb0\xa7¸\xe5D=E^\xb8I\xecb\xd6BFۉ\xedIA\x9a\xdb\x0e\x84\x89\xf8\fv\x17gѭO\xd7p\x17\x81]\xfb\xe7Wk\xc0\v\xa2\x99\b/\x82\xbb\xd4\x0f'\xfdѝ@\x81\x0fUJ\x15\xcf\xc6\x1c\x0f\xb8t\x11z\xd9X˾\xf6$ޟ\xbc\x14\xbf?\x82qt\xa8\x1b\xdf%\xad\xab\x84\xe5\xf3\x1fXl?\x00\xd3x3\x8b\xca\x1f\x7f\xf7\xc3\xda\xfb\xa4\xa5^\x9c\"c+?\\\xd4\r/\xe1\xba\xef\x90\xdep\xb0ڦ\x01\xba\x8b\xcaY:\xb7?c4휡\xb4\xf0\xf6\xfdybI\xfb3\x06ў-\x82v^\x94\x1f)>\x10}\x92\xd6\xfeͷ\x8d\x84\xd0<\xd8s\a\xd1Z1\xb40\xf0\x17\x8d\xa2E\xe7\xdcޏh\xa7\xf3\x96\xb5\xf0=\xf9_\xfe?\x00\x00\xff\xffY\xa1\x05sу\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xccZK\x93۸\x11\xbe\xebWt\xed\x1e\xf6\xb2\x94줒J\xe96\x96\x93*W\xc6\xf1\xd4hvr]\bhJ\xb0@\x80\x01@\xc9\xca㿧\x1a\x0f\x89\xe2C\x0f;q\u008b-\x12h\xf4\xf3\xeb\x0f\xc0\x14E1a\xb5|E\xeb\xa4\xd1s`\xb5\xc4/\x1e5\xfdr\xd3\xed\x1f\xdcT\x9a\xd9\xee\xedd+\xb5\x98âq\xdeT\xcf\xe8Lc9\xbe\xc7Rj\xe9\xa5ѓ\n=\x13̳\xf9\x04\x80im<\xa3\u05ce~\x02p\xa3\xbd5J\xa1-֨\xa7\xdbf\x85\xabF*\x816\b\xcfK\xef\xdeL\xdf\xfe~\xfa\xbb\t\x80f\x15\xcea\xc5\xf8\xb6\xa9\x9d7\x96\xadQ\x19\x1eENw\xa8К\xa94\x13W#\xa7\x15\xd6\xd64\xf5\x1cN\x1f\xa2\x84\xb4z\xd4\xfc]\x10\xb6\x8c\xc2\x1e\x93\xb0\xf0]I\xe7\xff<>\xe6Q:\x1f\xc6ժ\xb1L\x8d\xa9\x15\x86\xb8\x8d\xb1\xfe/\xa7\xa5\vX9\x15\xbfH\xbdn\x14\xb3#\xd3'\x00\x8e\x9b\x1a\xe7\x10f\u05cc\xa3\x98\x00$\xd7\x04i\x050!\x82\xb3\x99z\xb2R{\xb4\v\xa3\x9aJ\x1f\xd7\x12踕\xb5\x0fΌ\xb6@2\x06\xb25\xe0<\xf3\x8d\x03\xd7\xf0\r0\a\x0f;&\x15[)\x9c\xfd\xa2Y\xfe\x7f\x90\a\xf0\xd9\x19\xfd\xc4\xfcf\x0e\xd38kZo\x98\xcb_c\x8c\x9eZo\xfc\x81\fp\xdeJ\xbd\x1eR\xe9\x919\xffʔ\x14A\x93\x17Y!H\a~\x83\xa0\x98\xf3\xe0\xe9\x05\xfd\x8a\x1e\x02r\x11B\xf6\x10\xec\x99K\xeb\x00좔\xe0\xa3aMUo\xad3\xb5I\x15x\xedH\x89\xfaӛ\xa4}Kl\xce\xef)\xb7x\x14\xe9<\xab\xea3\xb9\x0fk\x1c\x13v\xe6\x8a\xf7X\xb2F\xf9\xb6\xa9\x14%\xd5\xce\xcbs\xb3j\xe4S\x11g\x9d\xad\xf8\xfe\xec]\\ue\x8cB\x16\xa5\xc4Q\xbb\xb71\v\xf9\x06+6O\x83M\x8d\xfa\xe1\xe9\xc3\xebo\x97g\xafa(\x91:EA\x81c\xad\xd8l\xd0\"\xbc\x86\xfa\x8bqsɴ\xa3L\x00\xb3\xfa\x8cܟ\x82X[S\xa3\xf52\x17K|ZX\xd4z\xdb\xd1\xe9\x9f\xc5\xd97\x002#\xce\x02A\xa0\x841\xafR\xfd\xa0H\x96\x83)\xc1o\xa4\x03\x8b\xb5E\x87:\xc2\x14\xbdf:)8\xed\x88^\xa2%1Tۍ\x12\x84e;\xb4\x1e,r\xb3\xd6\xf2\xefG\xd9\x0e\xbcI\xc9\xec\xd1y\b\x15\xaa\x99\xa2dm\xf0g`Zt$W\xec\x00\x16iMhtK^\x98\xe0\xbaz|\xa4j\x90\xba4s\xd8x_\xbb\xf9l\xb6\x96>#47U\xd5h\xe9\x0f\xb3\x00\xb6r\xd5xc\xddL\xe0\x0e\xd5\xcc\xc9u\xc1,\xdfH\x8f\xdc7\x16g\xac\x96E0DGH\xadď6a\xba;[\xb6W\xd2\xf1\t\x90zGx\b^c\xcaDQ\xd1\xc4S\x14\xe8\x15\xb9\xee\xf9\x8f\xcb\x17Ț\xc4HŠ\x9c\x86\xf6\xfc\x92\xe3Cޔ\xbaD\x1b\xe7\x95\xd6TA&jQ\x1b\xa9}\xf8\xc1\x95D\xed\xc15\xabJzJ\x83\xbf5\xe8<\x85\xae+v\x11\xba\x18\xac\x10\x9a:\x80Dw\xc0\a\r\vV\xa1Z0\x87\xdf9V\x14\x15WP\x10n\x8aV\xbb7w\aG\xf7\xb6>\xe4\x9e:\x12\xdaA4X\xd6\xc8\xcf\xeaN\xa0\x93\x96*\xc33\x8f\xa1\xba:\x0eJP1ޔ\xf33\f\x12\xf40\xceѹ\x8fF`\xf7KG\xe5\x87\xe3\xc03\x1dk\xb4\x95t\xa1\xbdBil\xb7\xf3\xb0#\x92\xb7\x9f\x8cx݀\x03\xa0n\xaa\xbe\"\x05<#\x13\x9f\xb4:\x8c|\xfa\xab\x95\xbe\xbf\xd0H \xe9\x89*.\x0f\x9a?\xa1\x95F\\1\xfe]g\xf8\xd1\x05\x1b\xb3\x872\xe4\xbf\xf6\xea@\xd8\xe5\x0e\x9a\xf7Q;?\x0fO\x1f2\x82\xc7\xdaJ\x85\x99|5\x85\x87TԦ\x847 \xa4#\"\xe1\x82о\xb3t\xa3\x02ј\x83\xb7\xcd]\xe6s\xa3K\xb9\xee\x1b\xdd\xe6Fc\x19sEt\xc7s\x8b\xb0\x12\xa1\x16eGm\xcdN\n\xb4\x05Շ,%O\x9a46v\x90R\xa2\x12=l\x1a\xad\xb2`\x8aEAE\xcdԕ\x18.\x8e\x03\x03\x93fR\xc7\f>\t\bXc\xabԚ\xb5G-\xb0\xdbm\x826&\x00\x9aC\x01{\xe97\x11)\xd5P\xdd\xc1\xc5ڣg\x8b\x87\xa1\xd7\x1d\xdd_6H#c\xe3Ep\xc8-\xfa\x90m\xa8(}(\x95\xa6\x00\x1f\x1b\x17\xb0\xb6\x8b\x13\xf9\t\x84/\xcf\xde\xe2\xa1\xefh\xb8\x16\xdcD\x85FT\x0e$j\x0e?\xfcpݤ^w\xcb\x0fQ\xf7l\xa8\xc5\x12-\xea\x1e\x9b\xc8\xcfK\xe8Q\x944\x94aX\x96Ƚܡ:\x84\x9eD\xe0\xf93\xac\x1a\x0f\xa2\xc1\x105Ʒ{f\x85\x03n\xaa\x9ay\xb9\x92J\xfa\x03H7\"\x9f)e\xf6(Rı\xaa\xfda\n\x1f\xb4\xf3LstG\x1eD\x1e\x8b\xa9\xc0t\x1c\x95\xaa8\x10:f\x8700\x8a\xaf\x8c\xf3\xc0\xd1R:\xaa\x03\xec\xad\xd1\xeb1c\a\xda!\xed\x01\xadF\x8f\xa1#\n\xc3\x1d5C\x8e\xb5w3\xb3C\xbb\x93\xb8\x9f\xed\x8d\xddJ\xbd.H\xc1\"\x81\xcf,\xec\xecf?\x86\x7f\xbe&\vL\x1dq\xe2\x86\xe4]\x86Z?\x10\xbd\xf5\x1b\x8c-b\x19s\xd0X \x02A\xa9]\xa5܍\xc8:TvC\xbc\xbc\xfd\xe4\x90\x0f\xf5\x8f-\xf6[\xc7\x05P\x01\xf8R\x9c|[T\xac.\xe2h\xe6M%\xf9\xa4km\xcc\xfb\xcb\xf8\x937+R\vɉܞ\xe3F\xdeĉ\xb3=̀\x1b\xba\xbb\x9c1\xb4\x1cvS47q\x85+\x1a\x7fj\x8f=m}#t\xa7\xfe\xef\xd0\x13\xeft\xa0\x91\xf8\x01\xb3}?\a\xc0\xe4FkB*o\x80\x1d\xdb\xc0O\xae\xdb\xff\xeeD\xcfU÷8\xe0\xf8\x9e)\xef\xc2\xc0\xec\xe38\x8dti\x1c\x86\xc6tM\r\xb8^\x11\x9c-\xd0ޢ\xcb(\xf2-\x1eH\u0091[0X<\xc0\xaa\xd1BaVu\xbfAM\xdb1Y\x1e\x88\xec\xbf<.\xb3c\x03\x01K[\xa7\xec\xde1 yO\xbb\x00JA1\x87_\x1c\xa6u\x9f\xb1\x04\xa9\x9dG\xd6#\xe9\xf1\x89\xbdq\x0e\xab\xc3\x00\u05fa\xd9A\xcfX~\xbb\x8f\x82\xae\xe4\xa1\xd4 8\xc6\xc4J\xb0\x92\xfa{\xde\x0f-\x1e\x02\xc4\x12\xdf \"}\xe6ґe\xeetth\xd0i\xf1\fdR\xc7\x02a\xd5\xd8\":\x1fR\x01\x8by9H䇃q\xb9.\xe0\x12\xb3\xe89\xfb>v1*\x13\x80\xdd\xc80\xe0z\xb2\xc0E\xa6\x017\xb0\x8d\x9e\x99\xa39\x05w\xb2\x0e\xf8\x0e\xcc\x03\xfe\xfb\xec\x03\xeef \xf0\xddY\bܖ)\x97\xd9\b|\x13#\xb9\xe0\x8bK\\\x05\xae\xf2\x15\xb8\xc8Y`\x94\xb7\xc05\xee\x02w\xf2\x17\bx\x82\xa5\xfcr\x032?\x85\x81\xb9\x93\xd6\xcco\xa8kH\x81\xc0\x06\xfaj<\xa1\x18q\xd0q\xd3\xfb)\x85\xef+\xfa\xee%\xd2\x17չ\x87\xf7e@\xbfB\x8c\x9eҰ\xa3\x17\xf2\xef\x04 \xe7\a c\x04mТ\xdd\xf1\xb4\xfdO\xf1X\x81\x0f\xa0\xf8\x992\xaf\xfd\x19\x17\x8e'\xf2\x99\xff\x10K\xa3Ͱ\xb1\x16]m\xb4\xa0\xb6w\xdb\xe1\xc4I\xe5\xff\xdc\x11\xc5pX\x8bs\xfa\xda\xf9\x96\xa3p\xd3\xf9\\\xb8߸\xfb\x84.\xde\xfa\xb4Ͽ\xccʡݵ\x0e\xe9:6~\x97\xb3\xb9\xc1\xce\xd6:\xb0#\xaa\xa4\xa1\xd1\xe1\xc8\"4\xad\xe9d`F\x9b\x17\xfa\xd0<\xa4\x03m\xf64\xb9%-v=\x13\xe9M8\xb4dZ\xa4\xe3b\xfa4 y/\x95\xa2\x1ef\xb12\xe4,\xd4^Zj\x96,\xb4\xb1\xddo\xa6o\xfewg\x81\x8a9\xbf\xc6Qq\xff\x99\xa6\x00[\x99\xc6\x0f\xf4\xfeV\xc2\x0f\xd6t\xb8e\xbfG\xc7\xf0\xb7\x03\xd7\xe8\t\x8d\xc9\x11፵\xe1\xb2._\"ݱ\xd1\x1cC\xe0\x87Ο8\xb4\xbf\xf5\xff\x00\xe2\x06\xbb\x06\xbbt\xefe촭\xb8&'\xb7\xdf4\xab\xe3\x15\xec\x1c\xfe\xf1\xafɿ\x03\x00\x00\xff\xff%\xff\\)\x99#\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcVMo\x1b7\x10\xbd\xebW\f\xd0kwU\xa3hQ\xec\xadqr0\xda\x06\x82\x1d\xe4N\x91#-c.\xc9\xce\f\xe5\xba\x1f\xff\xbd \xb9+K\xab\x95\x93\\\xb27\x91Ù\xc7\xf7f\x1e\xd54\xcdJE\xfb\x11\x89m\xf0\x1d\xa8h\xf1/A\x9f\x7fq\xfb\xf8\v\xb76\xac\x0f7\xabG\xebM\a\xb7\x89%\f\xf7\xc8!\x91Ʒ\xb8\xb3ފ\r~5\xa0(\xa3Du+\x00\xe5}\x10\x95\x979\xff\x04\xd0\xc1\v\x05琚=\xfa\xf61mq\x9b\xac3H%\xf9T\xfa\xf0C{\xf3s\xfb\xd3\n\xc0\xab\x01;0\xe8Pp\xab\xf4c\x8a\x84\x7f&d\xe1\xf6\x80\x0e)\xb46\xac8\xa2\xce\xf9\xf7\x14R\xec\xe0e\xa3\x9e\x1fkW\xdcoK\xaa7%\xd5}MUv\x9de\xf9\xedZ\xc4\xefv\x8c\x8a.\x91rˀJ\x00[\xbfON\xd1b\xc8\n\x80u\x88\xd8\xc1\xfb\f+*\x8df\x050^\xbb\xc0l@\x19S\x88TnC\xd6\v\xd2mpi\x98\bl\xc0 k\xb2Q\nQ\x1fz,W\x84\xb0\x03\xe9\x11j9\x90\x00[\x1c\x11\x98r\x0e\xe0\x13\a\xbfQ\xd2w\xd0f\xbe\xda\x1a\x9a\x81\x8c\x01\x95\xea7\xf3ey\u0380Y\xc8\xfa\xfd5\b,J\x12O J]\x1b<\xd0\t\xbf\xe7\x00J|\x1b{\xc5\xe7\xd5\x1f\xcaƵ\xca5\xe6pS\x99\xd6=\x0e\xaa\x1bcCD\xff\xeb\xe6\xee\xe3\x8f\x0fg\xcbp\x8euAZ\xb0\fjB\x9a\x89\xab\xacA\xf0\b\x81`\b4\xb1\xca\xed1i\xa4\x10\x91\xc4N\xadU\xbf\x93\xe19Y\x9dA\xf8\xb79\xdb\x03Ȩ\xeb)0y\x8a\x90\v\x89cS\xa0\x19/Zɵ\f\x84\x91\x90\xd1\u05f9\xca\xcb\xcaC\xd8~B-\xed,\xf5\x03RN\x03܇\xe4L\x1e\xbe\x03\x92\x00\xa1\x0e{o\xff>\xe6\xe6|\xef\\\xd4))\x94\xe4\xb6\xf3\xca\xc1A\xb9\x84߃\xf2f\x96yP\xcf@\x98kB\xf2'\xf9\xca\x01\x9e\xe3\xf8#\x93h\xfd.tЋD\xee\xd6뽕\xc9Rt\x18\x86\xe4\xad<\xaf\x8b;\xd8m\x92@\xbc6x@\xb7f\xbbo\x14\xe9\xde\njI\x84k\x15mS.⋭\xb4\x83\xf9\x8eF\x13Ⳳ\x17\xddS\xbf\xe2\x02_!O\xf6\x84\xda#5U\xbd\xe2\x8b\ny)Sw\xff\xee\xe1\x03LH\xaaRU\x94\x97\xd0\v^&}2\x9b\xd6\xef\x90\xea\xb9\x1d\x85\xa1\xe4Dob\xb0^\xca\x0f\xed,z\x01N\xdb\xc1\nO\x1d\x9b\xa5\x9b\xa7\xbd-\xb6\x9b\x1d E\xa3\x04\xcd<\xe0\xceí\x1a\xd0\xdd*\xc6o\xacUV\x85\x9b,\xc2\x17\xa9u\xfa\x98̃+\xbd'\x1b\xd33pEڅ\xe1\x7f\x88\xa8\xb3\xb8\x99\xdf|\xda\ueb2ec\xb5\v\x04O\xbd\xd5\xfd4\xfc3\x9a\x8eFq\xce߲1\xe4\xef\xc5n\xe7;W/\x0fEdK8k\xd8\x06.\xbc\xfbu^\x8a\xa9~%3\xd5\xd1Gnt\"*\xcdw\xf4y\xb5t\xe8K\xb9@\xa2@\x17\xab3P\xefJP\xf9Ǡ\xacgP\xfey<\b\xd2+\x81'\xa4\r\x97\x95\x1ax\x8fO\v\xabw~CaO\xc8\xf3\x96ϛ\x9b\xca\x1e\xce߃WXZlʋE\xceVhNXd\t\xa4\xf6\xa7\xbcr\xda\x1e\x9d\xbe\x83\x7f\xfe[\xfd\x1f\x00\x00\xff\xff\xbeM\x1a\xea\xb1\n\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcWMo\xe36\x10\xbd\xfbW\f\xd0K\v\xac\xe4\x06E\x8b·\xd6\xd9C\xb0\xe96\x88\xb7\xb9S\xd4HbC\x91,9t6E\x7f|1\xa4\xe4\x0fYv\x9c\xcb\xea\xe6\xe1p\xf8\xe6\xcd\xcc#]\x14\xc5B8\xf5\x84>(kV \x9c¯\x84\x86\x7f\x85\xf2\xf9\xd7P*\xbb\xdc\xde,\x9e\x95\xa9W\xb0\x8e\x81l\xff\x88\xc1F/\xf1\x16\x1be\x14)k\x16=\x92\xa8\x05\x89\xd5\x02@\x18cI\xb09\xf0O\x00i\ry\xab5\xfa\xa2ES>\xc7\n\xab\xa8t\x8d>\x05\x1f\x8f\xde\xfeX\xde\xfcR\xfe\xbc\x000\xa2\xc7\x15\xd4\xf6\xc5h+j\x8f\xffD\f\x14\xca-j\xf4\xb6Tv\x11\x1cJ\x8e\xddz\x1b\xdd\n\xf6\vy\xefpn\xc6|;\x84y\xccaҊV\x81>ͭޫ\xc1\xc3\xe9\xe8\x85>\x05\x91\x16\x832m\xd4\u009f,/\x00\x82\xb4\x0eW\xf0\x99a8!\xb1^\x00\f)&XŐ\xdd\xf6&\x87\x92\x1d\xf6\"\xe3\x05\xb0\x0e\xcdo\x0fwO?m\x8e\xcc\x005\x06镣D\xd4\x7f\xc5\xce\x0e\xd3\x04@\x05\x100\xc0\x01\xb2;\x84 \f\bO\xaa\x11\x92\xa0\xf1\xb6\x87J\xc8\xe7\xe8\xc0V\x7f\xa3$\bd\xbdh\xf1\x03\x84(;\x10\x1c%;\x1c\x9c\xa5m\v\x8d\xd2X\xeel\xce[\x87\x9e\xd4Hy\xfe\x0e\x1a\xea\xc0z)\v\xfe8\xf1\xbc\vj\xee,\f@\x1d\x8e\xe4a=p\x05\xb6\x01\xeaT\x00\x8f\xcec@\x93{\x8d\xcd\xc2\fٔ\x93\xd0\x1b\xf4\x1c\x06Bg\xa3\xae\xb9!\xb7\xe8\t\x1aE\xaf\xcb41\xaa\x8ad}XָE\xbd\f\xaa-\x84\x97\x9d\"\x94\x14=.\x85SEJĤQ+\xfb\xfa;?\ff8:\x96^\xb9!\x03yeڃ\x854\x1d\xef(\x0f\xcfK\xee\xae\x1c*\xa7\xb8\xaf\x02\x9b\x98\xbaǏ\x9b/0\"ɕ\x1aZl\xe7z\xc2\xcbX\x1ffS\x99\x06}ޗڔc\xa2\xa9\x9dU\x86\xd2\x0f\xa9\x15\x1a\x82\x10\xab^Q\x18{\x9dK7\r\xbbNR\x04\x15Bt\xb5 \xac\xa7\x0ew\x06֢G\xbd\x16\x01\xbfq\xad\xb8*\xa1\xe0\"\\U\xadC\x81\x9d:gz\x0f\x16Fy&j^\x01\x128\xe1[\xa4\xa9u\x82\xe5Kr\xe2\xe3_:q,X\xdfcٖ\xac9a\x00\x92\xf5\xe8\x87i\xa1.a\x80\xd9F\x9fE2\xf67\xd3\xc0\xbc\xb2\xa0\xb0\xd8\x1db:=\x9a?4\xb1\x9f?\xa0\x80\xdf\x13\xe6{\xdb^\\_[C<\x17\x17\x9d\x9e\xac\x8e=n\x8cp\xa1\xb3o\xf8\xde\x11\xf6\x7f:\xf4\xf9\x1a\xbe\xe8:\xde滫\xef\x82c\xd4g\xcf}D\xbeA\xf0|\xa6\x83\xc3UQ\xae\xc04x^\x95\xe8zs\xf7\x1e\nϸ\xbf\xa3Hw\xa6\xb1o\xa4\xb8w\x9c\xf5;#\x03\xe3\x97\xde\x10o\xf74\xbfBƞ\xe6-\xf9\xeeD\xf8\x14+\xf4\x06\t\xc3^\xa9_\x14u\xb3\x11\x01^:%\xbb\xb41\r\x04_\x02!X\xa9\xe6$\xf5\n\xf8\xac#\xca\xe3\xccP\x16iXg\xcc\f\xfe\xc4|F\xfd\xce\x1dP\f\x8at\x95\x82\x92\xa0\x18ޡ\xa1\xc9\x7f\xa4ZF\xef\xd3\x15\x95\xad\xfc2\x99n\xb8VDG\xe5\xf9\xeb\xf1\xfe\r%\xbd\xdd{\xa6\x17\xb7P&\xa3q\x1e\x8b\xa0Z~A\xf1\x1akiҸS2\xf2w\xfc\xc2;&j\xb6\xa2\xf8թ<\x80o@\xfc\xb8ŝ\x8f&\xdf\xf3\xd37l\n\x88\x81\x9f[ \x85\x99\xc1X!Ԩ\x91\xb0\x86\xea5\xdf\\\xaf\x81\xb0?\xc5\xddX\xdf\vZ\x01\xdf\xff\x05\xa9\x9962QkQi\\\x01\xf9x\xae\xcbf\x13w\x9d\b3cx\x94\xf3\x03\xfb\xcc5\xc6n\x18/v\x06\x9c\xbd_\n\xf8\x8c/3\xd6\ao%\x86\x80\xa7ct6\x93\xd9!81\x06~\xa4\xd5\a,\r\x7f\x19\x06\xcb\xff\x01\x00\x00\xff\xffx\xae@\xbaJ\x0e\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4:Ks\x1b7\xd2w\xfd\x8a.吤\xca$\xe3|ߦ\xb6x\xb3\xe5͖v\x13\xafʔ}I\xe5\xd0\x1c49\x88f\x00,\x80\x11\xcd\xcd\xe6\xbfo5\x80\xe1\xbc@R\xa2\x93\x18\x17\x89x4\xfa\xfd\xc2\xccf\xb3+4\xf2\x03Y'\xb5Z\x02\x1aI\x1f=)\xfe\xe5\xe6\x0f\x7fus\xa9\x17\x8f/\xaf\x1e\xa4\x12K\xb8i\x9c\xd7\xf5;r\xba\xb1\x05\xbd\xa1\x8dT\xd2K\xad\xaej\xf2(\xd0\xe3\xf2\n\x00\x95\xd2\x1ey\xda\xf1O\x80B+ouU\x91\x9dmI\xcd\x1f\x9a5\xad\x1bY\t\xb2\x01x{\xf5\xe37\xf3\x97\xdf\xcd\xffr\x05\xa0\xb0\xa6%\x18-\x1eu\xd5Դ\xc6\xe2\xa11n\xfeH\x15Y=\x97\xfa\xca\x19*\x18\xf6\xd6\xea\xc6,\xa1[\x88gӽ\x11\xe7;->\x040\xaf\x03\x98\xb0RI\xe7\xff\x99[\xfdA:\x1fv\x98\xaa\xb1XM\x91\b\x8bN\xaamS\xa1\x9d,_\x01\xb8B\x1bZ\xc2[F\xc3`A\xe2\n \x91\x18К\x01\n\x11\x98\x86՝\x95ʓ\xbda\b-\xb3f \xc8\x15V\x1a\x1f\x982\xc2\x0f\x9cG\xdf8pMQ\x02:xK\xbbŭ\xba\xb3zk\xc9E\xe4\x00~qZݡ/\x970\x8f\xdb\xe7\xa6DGi52w\x15\x16Ҕ\xdf3\xca\xce[\xa9\xb69$\xeeeM \x1a\x1b\x84\xca\xd4\x17\x04\xbe\x94n\x82\xdd\x0e\x1dch} ;\x8fKXg\x88\xcecm\xc6H\xf5\x8eF\xac\x04z\xca\xe1t\xa3kS\x91'\x01뽧\x96\x92\x8d\xb65\xfa%H\xe5\xbf\xfb\xff\xe3\xecH\xfc\x9a\x87\xa3o\xb4\x1a\xf2\xe65\xcfBo:b²ڒ\xcd2H{\xac>\x05\x11\xcf\x00^\xf7\xceGL\"\xdc\xfe\xfcYTnUa\xa9&u\x19B\xb2;=Ŧ\x0f\xba\xbfj\xac\xd4V\xfa\xfd\x12^~\xf3T4\xd9>@o\xc0\x97\x04IyV^[\xdc\x12\xfc\xa0\x8b\xa8h\xbb\x92lR\xb4u\xd2\xfeR7\x95\x80u+\x18\x00\xe7\xb5\xcd*\x9b\xa1b\x1eO%\xb8-ؑ\xc6\r\xef\xfc#\f\xa2\xb0\x84Y\x83h\x9d\xe6<\xec\x90Z\xe5\xad\xe2Ֆ\x9ed\x11}\x96*-\xe8\xc0?\x9a\xa0%\x1d\x18\xab\vr\ue1212\x8c\x01\"o\xbb\x89\xb3\f*)\xeci\xf1iL\xa5Q\x90\x05\xaf\xa1D%*b2\x10\xbcE\xe56IE\xa6\x02l\x8f\xdd\xef\xcd\x10\x95\xf7i\xe1\x18:q\xd7\xe3\xcb讋\x92j\\\xa6\xbdڐzuw\xfb\xe1\xffV\x83iVcm\xc8zن\x8f8z\xc1\xb17\vCr\xff;\x1b\xac\x01\xf0\x05\xf1\x14\b\x8e\x92\xe4\x02\x1bR \x91p\x8a\xec\x91\x0e,\x19K\x8eM+h\x94\xde\x00*\xd0\xeb_\xa8\xf0\xf3\x11\xe8\x15Y\x06\xd3\xdaB\xa1\xd5#Y\x0f\x96\n\xbdU\xf2?\a؎y͗V\xe8\xc9\xf9`\x8cVa\x05\x8fX5\xf4\x02P\x89\x11\xe4\x1a\xf7`\x89\xef\x84F\xf5\xe0\x85\x03n\x8cǏ\xda\x12H\xb5\xd1K(\xbd7n\xb9Xl\xa5oS\x86B\xd7u\xa3\xa4\xdf/B\xf4\x97\xeb\xc6k\xeb\x16\x82\x1e\xa9Z8\xb9\x9d\xa1-J\xe9\xa9\xf0\x8d\xa5\x05\x1a9\v\x84\xa8\x906\xcck\xf1\x85MI\x86\x1b\\;\x11t\x1c!\xd2?C<\x1c\xfb\xd9\b0\x81\x8a$vR\xe0)fݻ\xbf\xad\xee\xa1\xc5$J*\n\xa5\xdb:\xe1K+\x1f\xe6\xa6T\x1b\xd6y>\xb7\xb1\xba\x0e0I\t\xa3\xa5\xf2\xe1GQIR\x1e\\\xb3\xae\xa5g5\xf8wCγ\xe8\xc6`oBZ\x05k\xb6%\xf6\x00b\xbc\xe1V\xc1\r\xd6Tݠ\xa3?YV,\x157c!\x89wg\xf9\xc3c#\xa9\x12!s8\x7fwVsy\xdcn\"\x12!\"x\r\bFRA\x83h\fR9O(\xd2$;AKi\xedE\xf4\xf4G\x91\xe4\xd1Em\x96\t G\x1e)\xe0\x1f\xab\x7f\xbd]\xfc]G:\x00\vN\xcdB\xad\x17\xf2\xed\x17\x87zO\x90\x93\x96\x04Wo4\xafQ\xc9\r9?O\xd0Ⱥ\x9f\xbe\xfd9\xcf?\x80\xef\xb5\x05\xfa\x88\\5\xbd\x00\x19y~\bf\xad\xdaH\x17\t?@\x84\x9d\xf4e@\xd4h\x91\b\xdc\x05\x12<>\xb0%G\x12\x1a\x82J>d\xec'\x8e\xeb\x90\xcduh\xfe\xca\xd6\xf3\xdb5|\x15\x9d\xd75\xff\xbc\x8eh\x1cҖ\xbe\x81u\xe8D+\xb3r\xbb\xa5.\xef\x9f(\v\x87Y\x0eP_\x83\xb6L\xab\xd2=\x10\x010\xcb)\xc6\a\x12\x13\xf4~\xfa\xf6\xe7k\xf8jȃ#WI%\xe8#|\xcb\xde'\xf0\xc6h\xf1\xf5\x1c\xee\x83\x1e\xec\x95Ǐ|SQjG\n\xb4\xaa\xf61\x01~$p\xba&\xd8QU\xcdb\x82(`\x87{Л#\xf7\xb4\"b\xd5D0h\xfd\xc9$1\xf1\xe1\xb4\xd1L\xb3\xa6v<\xcd^B\x16\xf5$\xeb\xfdl\x19\xc8\x139\x11ʅO\xe0D\xbf\xf4\xba\x80\x13\x0f͚\xac\"O\x81\x19B\x17\x8e\xf9P\x90\xf1n\xa1\x1f\xc9>J\xda-v\xda>H\xb5\x9d\xb12\u03a2\xd4\xdd\"t\xbb\x16_\x84?\x97\x12\x1e\xdaT\x9fJ}\x00\xf2\xf9X\xc0\xb7\xbb\xc5%\x1ch\xb3\xfb\xa7Ǯ\xa3|X\xa5\x84s\f\x93m~Wʢlk\xbd\x9e\xb7\xadQDw\x8cj\xff\x99l\x87\xf9\xdcX\xc6h?K\xad\xda\x19*\xc1\xff;\xe9<\xcf_\xc2\xd8F~\x92sy\x7f\xfb\xe6sZT#/\xf1$Gj\x988>\xce:\xacf5\x9aY܍^ײ\x18\xed\xe6\x1c\xfeV\xb0\x906\x92\xec\x99\xf4\xef\xdd`s\x9b\xa0f\xaa\x81Þg\xe5\x9f\x1e\xb7\x99\x84\xaf\xdf\xc5>\x95\x16\x9e\xe4\xd7yU\xb8ǭ\x03\xb4\x04\b5\x1aֈ\a\xda\xcfb\xc6aPr\xba\xc0\x19\xc1\xa11\bhL\xc51=f\x11\x19\x88)\xffM\xecA\x17\xe8;Ɛ\xac(ۮԊ\xbc\x97\xea32\xe7\xfd\b\x91ߗQ\x87\x9e]\xa1\xd5FnS\xb7s\xca)\xd5T\x15\xae+Z\x82\xb7ͱ\x9a\xeb$#\xefy\xcbi\xfa\xdf\xf7\xb6\xb6\x1a~\xa6\xc1\x98\xa7j\xd0v\x9c\x12C\xaa\xa9\xa7\xa8\xcc\xe0A\x1b\x89\x99yK\xceO\xac\x97\x17\xae\xaf\x9fccQ)/)\xb9c\x19\x9c\xabJ\x93\xa2\xa7\x04\xbe\xadL\xbd\ueabc\xacП\xe1\x1b\xb8\xba\xe7rd\x88\xf7,\xdf.\x19\xed\xe9u\x97\xdb)\xa3\xc5hf\xe8\x06G\x8b\x91\xbe'\xf5\x90BC\xfb\x19]\xa4\xf8Ȗx\x1a\x83\xa3o\x9f\xde8\xed\xbe\xb4\x8fą\x9d\xf1$\x0e\x8d\xfeK$\xfej\f$\xf4~\xadHF!k:\x94\xfeC_\x17\x8b\xbb5\x81\xb1d0\xdb\x15\x82йw\xa1\x85\xf9\xa5\x8b\xc0\xa4\x83Ƒ\b\x1d\xb4\xc9\xdd\x13\b\xed;\x93@O3>\x7f\x99\xbf\xc87\xa6\xe2\x9b_\xff\xa5\xe4\xa2.\xd5\x14̔\x85\xd8r-<ᴏ\x8d9\x8eu\xe0\x0e\xfc\x8a\xd0H\x84*\x94\x8b\xe4\rʊ\x04\xb4/\xd9τ\xb2\xa6\r\xa78\xd1ǵ}\x9c\x84\xde\xf1\xfa\xef\xb4$3L\x98&<\x7f\xa40\xc7O\x8dg$y;\xda\x0e\xa5\xae\x92\xbcTS\xafɲa\x86\aOP\xb4㺿(Qm\xb3N\xae}\xb0#\xa8\xd0yXw\x1f\x06\xe4\x88\uffd8\x8e)\xeb\xbfpv\xa3&\xe7p{Ν\xff\x18w\xc5\xce]:\x02\xb8֍\xcf\xdb\xef\x97.\xb9\xa0\xe7u\x0f\xb3M\xb1\xa1\xf7C_\xb6\xcen\xd3TU8ӏ\x1b\xdd\a\x1c\x01\xab5\xe53\xfe\x13\xad\xc3S\b\x96\xe8α\xea\x8e\xf7\xe4\xfc\xf1!؝t\xc8p\"\xb0\xbf\xa5]f\xb6\xf5s\x99\xa5\xbb\xe4<3K\x93/1\xfa\x8b\xb17\x9e\xe3\\\xbb\x96\x85y\xf8\xce!\xb3\xf6}\xf0*\xcfbv\xc2\xef\x12\xb7y\xe8\xadw\x96\x17>[\x98\xd8\xdf0\xff@%\xfab\xcb5!\xba\xf3\xad\x06EH\xa9\x91\x96\x9e\x04\x82\xeb\xf2\x1a\x84t\xa6\xc2\xfd\x81\x96P\xfa\xb1\xa9\xe6\xdfG:\x8bj=\xa6\xa1c\xa9\xec\xe9\x0e\xf7\xe1k\x91|]{\xda_\xc0\x19\x9f\x11\xd6\xf5qg\xf8{\xdcp\"\x15w\n\x8d+\xb5\xbf}sF5V\x87\x8d\xad=vee\b,\xe1\xe9-mJ\xaa\x90A\xb5\xf3n\xcfr\x16Ï\x87.\xd1\xe2\xd5\x00\u0099\xb8\x9f\xbee\xcaE\xd7\x15{\x01v@\xe1a\xf7f\xfc\x05NjC\x90A\x9f\x1a\xe41\x1e\xe5\xba\nZ\x85:B\xdb\xe9+;\x9c\r\xe4C\x82\xfe\xcc\x18\x9eU\xa7\xc9d\xc0\\\xf4`\xa77\xcd\xfeL\xb3><\xf7/\xe1\xd7߮\xfe\x17\x00\x00\xff\xfff=C\x19\x96(\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4Z͒\x1b\xb7\x11\xbe\xefSt\xad\x0f\xb6\xab4d\xa4$\xae\x14o\xd2*Nmbo\xb6DI\x17\x97\x0f\xe0\xa09\x03s\x06\x80\x01\f\xb9\xb4\xe3wO5\x80\x19\xce\x0fH.\xa9\x925\x17i\xf1\xd3\xf8\xf0u\xa3\xbb\xd1`\x96e7L\x8b\x8fh\xacPr\x01L\v|r(\xe9/;\xdb\xfc\xc3΄\x9ao_\xdel\x84\xe4\v\xb8k\xacS\xf5;\xb4\xaa19\xbeŵ\x90\xc2\t%ojt\x8c3\xc7\x167\x00LJ\xe5\x185[\xfa\x13 W\xd2\x19UUh\xb2\x02\xe5lӬpՈ\x8a\xa3\xf1\xc2ۥ\xb7\x7f\x99\xbd\xfcn\xf6\xf7\x1b\x00\xc9j\\\x80V|\xab\xaa\xa6F\x83\xd6)\x83v\xb6\xc5\n\x8d\x9a\tuc5\xe6$\xbc0\xaa\xd1\v8t\x84\xc9q\xe1\x00\xfaQ\xf1\x8f^λ \xc7wUº\xff$\xbb\x7f\x10\xd6\xf9!\xbaj\f\xab\x128|\xaf\x15\xb2h*f\xa6\xfd7\x006W\x1a\x17\xf0@P4ˑ\xdf\x00\xc4}zh\x190\xce=s\xacz4B:4w$\xa2e,\x03\x8e67B;\xcf\xcc\x18\"X\xc7\\c\xc16y\t\xcc\xc2\x03\xee\xe6\xf7\xf2Ѩ\u00a0\r\xf0\x00~\xb1J>2W.`\x16\x86\xcft\xc9,\xc6\xde@\xf1\xd2w\xc4&\xb7'\xcc\xd6\x19!\x8b\x14\x8a\xf7\xa2F\xe0\x8d\xf1\xaa\xa5\xfd\xe7\b\xae\x14v\no\xc7,A4\xceo<\r\xc6\xf7\x93H\xebX\xadǨzS\x03,\xce\x1c\xa6@ݩZW\xe8\x90\xc3j\xef\xb0\xdd\xcaZ\x99\x9a\xb9\x05\b\xe9\xbe\xfb\xdbq>\"a3?\xf5\xad\x92Cr\xdeP+\xf4\x9a\x03\x12\xd2V\x81&ɐr\xac\xfa\x14 \x8e\x04\xbc\xe9\xcd\x0fH\x82\xdc~\xfbY(dz\xa0\xd6\xe0J\x847,\xdf4\x1a\x96N\x19V \xfc\xa0\xf2\xa0\xc2]\x89\x06\xfd\x88U\x18A'\x18\x04\xe9N\x99\xa4\xea4\xe6\xb306\nke\x8d\xf47\\\xe8\xb3\xd8Wn\x90%\xed\xabuE3?B(\x996\xb2\xd7\x05>\xcb\xc0\xfaDJű\xc7\xda\x04\x97\xb0\xa0\x8d\xca\xd1\xda\x13\x86OB\x06H\x1e\x0e\rg)*яi\x015\xbaR\x8c\xa3\x01\xa7\xa0d\x92W\x18t\xe8\f\x93v\x1d-c\xaa\xc2v\xda\xfb\xbd\x1eB\xf9\xd0\xca\xeb\xf5L0\x85\xa1ۗ\xc1\r\xe6%\xd6l\x11\xc7*\x8d\xf2\xf5\xe3\xfdǿ.\a\xcd@\xb4h4N\xb4\x9e9|\xbd\xc0\xd3k\x85\xe1\x9e\xff\x97\r\xfa\x00h\x810\v8E \xb4\x9e\x8b\xe8_\x91GL\x81#a\xc1\xa06hQ\x86\x98D\xcdL\x82Z\xfd\x82\xb9\x9b\x8dD/ѐ\x18\xb0\xa5j*N\x81k\x8bƁ\xc1\\\x15R\xfc\xd6ɶD8-Z1\x87\xd6\xf9\x83h$\xab`˪\x06_\x00\x93|$\xb9f{0HkB#{\xf2\xfc\x04;\xc6\xf1\xa3\xb7&\xb9V\v(\x9d\xd3v1\x9f\x17µ\xe18Wu\xddH\xe1\xf6s\x1fYŪq\xca\xd89\xc7-Vs+\x8a\x8c\x99\xbc\x14\x0es\xd7\x18\x9c3-2\xbf\x11\xe9C\xf2\xac\xe6_\x99\x18\xc0\xed`ى\xa2\xc3\xe7\x83\xe8\x05ꡨJ'\x81EQa\x8b\a-P\x13Q\xf7\xee\x9f\xcb\xf7\xd0\"\t\x9a\nJ9\f\x9d\xf0\xd2\xea\x87\xd8\x14rM\x86O\xf3\xd6F\xd5^&J\xae\x95\x90\xce\xff\x91W\x02\xa5\x03۬j\xe1\xc8\f~m\xd0:R\xddX\xec\x9dOY`E\a\x8a\xfc\x00\x1f\x0f\xb8\x97p\xc7j\xac\xee\x98\xc5?YW\xa4\x15\x9b\x91\x12\x9e\xa5\xad~\"6\x1e\x1c\xe8\xedu\xb4Y\xd4\x11Վ\xfd\xdbRcN\x9a%ri\xaaX\x8b\x18I\xd6\xca\x00\x9b\x8c\x1f2\x95v\x01\xf4%#\xcax\xd09\xb3\xa3\xefMJP\x8bX\xf6\x1cy\x8cw6\x06\xaaj\x18\xa8\xfa\xdf$F\x1a\xd4\xca\n\xa7\xcc\xfe\x10)\xc7&qT;\xf4\xe5L\xe6X]\xb3\xbd;?\x13\x84\xe4\xc4;v&M\xce(H\xf5@\x95,\x14\x1d\xb2\x89:\xe0\xde\xd18\xb2s\x8b.\xbdYy4\xb2\t\t\x87\x1c\x13\xfa\xb9\xe4x\xdb+\xa5*dc6\xb5\xe2g6\xfd\xa8\xa2\xe30\xb8F\x83>\xfe\a7\xab\x95wƎ\tٺ\x8f\x90r\x83S\x89}\xac\xc8\xdd\x1cS\xcdq;\x84\x13!)\t\xf8\xf5\xe3}\x1bvZˊ\xd0'\x91\xa5\xcfO\xd2,\xe8[\v\xac\xb8\x0f\xd4\xe7\xd7NZ\b}\xf7\xeb\x00\xc2\xfb^\xa7\x80\x81\x16\x98\xe3 \ue050\xd6!㱑܍\xc1\xd8\xf7\"\xf8ԣ \xe9;\xc4GR\t0\xf2\xf1\x82ÿ\x97\xff}\x98\xffK\x85}\x00\xcb)\x13\xf2w\x15\xacQ\xba\x17\xdd}\x85\xa3\x15\x069\xdd>pV3)\xd6h\xdd,JCc\x7fz\xf5s\x9a?\x80\xef\x95\x01|b\x94\xf4\xbf\x00\x118\xef\xc2Fk5\u0086\x8dw\x12a'\\\xe9\x81j\xc5\xe3\x06w~\v\x8em\xe8Ą-4\b\x95\xd8`\x9a}\x80[\x9f<\x1d`\xfeN.\xe5\x8f[\xf8&8\x89[\xfa\xf36\xc0\xe8\x12\x84\xbe\xd79\xc0q%s\xe0\x8c(\n<$\xda\x13c\xa1\x80F\xa1\xe0[P\x86\xf6*UO\x84\x17Lz\n\x8e\x18\xf9\x04\xdeO\xaf~\xbe\x85o\x86\x1c\x1cYJH\x8eO\xf0\x8aθ\xe7F+\xfe\xed\f\xde{;\xd8KǞh\xa5\xbcT\x16%(Y\xedC\xbe\xb9E\xb0\xaaF\xd8aUe!\x15\xe3\xb0c{P\xeb#\xeb\xb4*\"\xd3d\xa0\x99q'ӱ\xc8\xc3\xe9C3\xcdO\xda\xefy\xe7\xc5\xe7+\xcf:\xbd_,\xd6?\x93\t\x9f\x98\x7f\x02\x13\xfd\xab\xce\x15Ll\x9a\x15\x1a\x89\x0e=\x19\\\xe5\x96x\xc8Q;;W[4[\x81\xbb\xf9N\x99\x8d\x90EFƘ\x05\xad۹/\xd9̿\xf2\xff\\\xbbq_g\xf9\xd4\xdd{!_\x8e\x02Z\xddίa\xa0ͣ\x9f\x1f\xbb\x8e\U000b0319\xddX&\x9d\xf9])\xf2\xb2\xbdU\xf5\xbcm\xcdxp\xc7L\xee\xbf\xd0\xd9!\x9e\x1bC\x88\xf6Y,8fLr\xfa\xbf\x15\xd6Q\xfb5\xc46ⓜˇ\xfb\xb7_\xf2D5\xe2\x1aOr\xe4\xb6\x10\xbe\xa7\xec\x80*\xab\x99\xce\xc2h\xe6T-\xf2\xd1hʕ\xef9)i-М\xc9\xfe\xde\r\x06\xb7Y{\"\xeb\xee\xc6\\\x94v[ɴ-\x95\xbb\x7f{\x06Dz\x1b\xd8b8\xe80&\x9d\xad,:\x12's\xcdg\xe0Y\x8a\xdf\x12n+\x89\x88\x86\xb6\x98*U\x88\x9cU`}\x9b\x8c\xc5\xca\b\xb3\x95=\x05\x94\xaaG\x8e\xe1\xf6\xab\x8a=\xbc\xde\x17<\x1c\xf7\xb4C\xc8\xc3\xd1-jeD!$\xab\x0e\x1e\xdb_\x1d%\xab\x99\xff+a\xab5\xd3Z\xc8\xe2\"n\xdb\xfa\xd6\x12\x9d\x13\xb2H$\xfa\xfd\xf2\xfb\xa9\xeb\xc0\xc9sr\xde\x05|\x18\x01\x01f\x10\x18\xed\x89T\xb5\xc1}\x16\xb2N\xcd\x04\xa5\x8c\x94\x15\xc6\xd4z\x85\xc0\xb4\xae(\xaf\v\x99d\xca7\xb5պ\\ɵ(b\xe5tʔl\xaa\x8a\xad*\\\x803ͱK[\xf2\xb8\xf7\v\x85g4\xfe\xa17\xb4U\xf7\x99RezW\x83\x02\xe6t3(\x9bz\n%\x83\x8d҂%\xda\xe9pN\x1c\x13u\xdc\xde^bR\xe1\xe4\x9f\xe1 ܙS\x05\x87\xe88\xe25$^\xb1\x83\xfbHG\xf3K\x1d\x8a\xc1_\x1b\xbaS\r\x11f\xe9\xda\xcah\x8cV\xfcfLZ\xdf\x17\x8f:\x0f\x9et\xdc1<\xf4\xa3\xde@\xc1\xb3\xcaR\xbeP~Ia*<\x87E\xdeC\x1a\xe0\xdaG2\xba`\\]\x9a\xa2;\xacvȻ7\x84k\xea6\xaf\xc7B|A\xd9\xf0xHD\x8d]\x91#ډ9\x94]B\x88\xd1\x065KZ\x04\xf8G\x01\xeb\v\xa3_\xdb MXh,r\xef['\x8b\x1f\x8d\t\x9c9\xcch\xfeu\x0e$]\xec\n\xcfs\xfdW\x98\xab*_S1S\x0eYG\x9b\x7f\x1fj\x1f\x06S\x94\x1d\xe4u\x84\x05q\xc8\xfd\x95\x1b\x94\x845\x13\x15r\xe8\x1e\x9f/f>\x01z\x9a\x8c}N\xf2k\xb4\x96\x15\xe7\x9c֏aT\xa8\xbc\xc5)\xc0V\xaaqG\xac\xf2k\x1b\x8f\xd6E1Y*~\x0eɃ\xe2\x1e\x86<\xfe\xe46E\x93PK\xff\x19\xee\"\x8c\xbe\xa8y\xaeHIcR\xae\xa6\x83|\xda\xd7\xc0\x89\x18\xf6\x80\xbbDk{\x82\x13]\x8f\xd1-$\xba&\xbf\a\xe8w\x86Jr*\xa7i\xfb\x922\xbb\xc7\xf6D\xdf\xf7\xfe\xb8\\\xc4v\xc4w\x8dC\xe8\xeaХ\xaaZ\x1f\xe0\x1f\xc9eS\xafА*V\xa9\x8c\x18\x98\xe4}ͥ\x8a\t\x9d\x846\f\aQ\xb1\x1e\x16\v\xe8\xfe\x94;\x05\\X]\xb1}\xb7\x19\x7f\x83\xa3#\x9d~N8\x9c\xab\xd6WQ\xe49\x92\xb7\x9d\xaeTw?ZH\xdfOOg\xfap&\xdb\xf7\xfdݏ\x11>\xcf\n'\xf2\xce\xe1\x8fC\xae1\x90\xe5@¹`\x11\x7f\xacr\xb9\x8f\x1f.\xf3g\xba\xf7${\x93F\x8f\x9c\xf7d\xc7'\xaf~K\xb3\xeaރ\x17\xf0\xfb\x1f7\xff\x0f\x00\x00\xff\xff;\xa8N\xc3\x13&\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xdc=[s\xdb8w\xef\xf9\x15\x98\xf4a\xdb\x19\xcbi\xa6\x97\xe9\xf8\xcd\xf5:\x8d\xfb}\xebx\xec4\xfb\f\x91G\">\x83\x00\x17\x00\xa5h\xdb\xfe\xf7\x0e\x0e.$%\x90\x84d˛-^2\xa6\x80\x03\xe0\xdc\xcf\xc1\x01\xb2X,\xdeц}\x03\xa5\x99\x14W\x846\f\xbe\x1b\x10\xf6/}\xf9\xfco\xfa\x92\xc9\x0f\x9b\x8f\uf799(\xaf\xc8M\xab\x8d\xac\x1fA\xcbV\x15\xf03\xac\x98`\x86I\xf1\xae\x06CKj\xe8\xd5;B\xa8\x10\xd2P\xfbY\xdb?\t)\xa40Jr\x0ej\xb1\x06q\xf9\xdc.a\xd92^\x82B\xe0a\xea\xcd?^~\xfc\xd7\xcb\x7fyG\x88\xa05\\\x11\x05\xdaH\x05\xfar\x03\x1c\x94\xbcd\xf2\x9dn\xa0\xb00\xd7J\xb6\xcd\x15\xe9~pc\xfc|n\xad\x8fn8~\xe1L\x9b\xbf\xf4\xbf\xfe\x95i\x83\xbf4\xbcU\x94w\x93\xe1G\xcdĺ\xe5T\xc5\xcf\xef\bхl\xe0\x8a\xdc\xdbi\x1aZ@\xf9\x8e\x10\xbft\x9cv\xe1W\xbd\xf9\xe8@\x14\x15\xd4ԭ\x87\x10ـ\xb8~\xb8\xfb\xf6OO\x83τ\x94\xa0\v\xc5\x1a\x83\b\xf8\x9fE\xfcN\xc2B\tӄ\x92o\xb8Q\xbb\x1aD<1\x155DA\xa3@\x830\x9a\x98\n\bm\x1a\xce\n\xc4;\x91\xab\x1e\xa40J\x93\x95\x92u\amI\x8b\xe7\xb6!F\x12J\fUk0\xe4/\xed\x12\x94\x00\x03\x9a\x14\xbc\xd5\x06\xd4e\x04\xd4(ـ2,`ٵ\x1e\xef\xf4\xbeNm\xcc6\x8b\v7\x8a\x94\x96\x89\xc0m\xc1\xe3\x13J\x8f>\"W\xc4TLw[\r\xdb#T\x10\xb9\xfc\x1b\x14\xe6r\x0f\xf4\x13(\v\x86\xe8J\xb6\xbc\xb4\xbc\xb7\x01e\x91Uȵ`\xbfG\xd8\xdan\xdcNʩ\x01m\b\x13\x06\x94\xa0\x9cl(o\xe1\x82PQ\xeeA\xae\xe9\x8e(\xb0s\x92V\xf4\xe0\xe1\x00\xbd\xbf\x8e_\x90xb%\xafHeL\xa3\xaf>|X3\x13$\xaa\x90u\xdd\nfv\x1fP8ز5R\xe9\x0f%l\x80\x7f\xd0l\xbd\xa0\xaa\xa8\x98\x81´\n>І-p#\x02\xa5\xea\xb2.\xff.\x12u0\xad\xd9Y\x1e\xd5F1\xb1\xee\xfd\x80\x02q\x04y\xac\xa88\xc6s\xa0\xdc\x16;*\xd8O\x16u\x8f\xb7O_\xfbLɴ'J\x8f7\xc7\xe8c\xb1\xc9\xc4\n\x94\x1b\x87\xacia\x82(\x1bɄ\xc1?\n\xce@\x18\xa2\xdbe͌e\x83\xdfZЖ\xdf\xe5>\xd8\x1b\xd4:d\t\xa4mJj\xa0\xdc\xefp'\xc8\r\xad\x81\xdfP\roL+K\x15\xbd\xb0DȢV_\x97\xeewv\xe8\xed\xfd\x104\xe2\bi\xbd\x16yj\xa0\x18H\x9a\x1d\xc6VA]\xac\xa4\x1a(\x19;d\x88\xa3\xb4\xf0\xdb洈U\x8b\xfb\xbf\xccq\x99m\xff\x1eG[~\xb3+k\x05\xfb\xad\x05T\xa6N\xfc\xe1P_\xa9\x9ej\x1f6\xcbF\xfb\xd4\x1dE\xb4m\xf0\xbd\xe0m\te\xd4\xeb\a\x1b\xcc\xd9\xc6\xed\x01\x144z\x94\t+D\xd6\xfaؽ\x88\xeeWT\xe0T\x01\x11\xd2$\xe01\xe1\xe0\x11&\x10\x03I\x9a`G\x03ubœ[&D\xb4\x9c\xd3%\x87+bT{\x88F7\x96*Ew#\xd8\n\x1e\xc0\x8b\x90\x15\x81xU\xc3Y\x81$\x8f\n\x05\xf1\xf5\xe7E\x15\xd3VQ\x86]>HΊ\xdd\f\xben\x93\x83\x82\xb4z\xd9\xf5;$K\xa8\xe8\x86I\x95\x12\x03\xa9\xb0kϞwjZZ-\xe9\x81\xec۸\xcc\r'\x91UI\xf9<\xc7\x10\x9fm\x9f\xce:\x90\x02\x1dʸ\x15Omo\xbb\x97@\xe0;\x14\xadI,\x93\x90\xb2E\xd3$\x15i\xa46\xe3t\x1fW]\xa4\xef\x1c\xa5~\x9c`\x9a\x83\x9d%Y\xdd5\xaf\x84\x03Q-\x0e\x06\nY\n\xb0ۨ-Q\xbb\xbeJ\xb6\xae\xef(RȒj(\x89\x14\xa33#\xbb\xb4\x1c\xb4\x9f\xabD\xce\xe8\xf4\xd0E\xb7\x7f\xf4x\b\xa7K\xe0D\x03\x87\xc2Hu\x88\xcc\x1c\x94\xba\x96\xa3XGP\x99ЦC\t\xe860\x01\x92XN\xdfV\xac\xa8\x9c\x87a\xd9\x13\xe1\x90R\x82\xb6\xda\x04]\xe6\xdd\xd8&\xc9\x1c\xf9\xfd$Sڣk3b\xb5\x0f/\xa5Q\xba\x96\xa1\x86\xbb\x96Dm\xa7{\x0ft\x8b\xffn\xe4\xe4\xb6\xff\x7f\"6\x18\x93\x13\x98vB\xfe\t\xba\x9f\xd9<=ʷ\x18ၾ$w+\x02ucv\x17\x84\x99\xf0uN\x12(\xe7\xbd9\xfeĴ9\x9e\xe93I\x93#\x13g\"L\x9c\xe2OH\x174\x19O\xdebd\xd3\xe4\xaf\xfdQ\x17\x84\xad\"\xd2\xcb\v\xb2b܀\xda\xc3\xfeI\xaa>P\xe65\x90\x91c\xf5\b\xe6\tLQ\xdd~\xb7.\x8e\xee\x92`\x99x\xd9\x1f\xec|\xe3\x10A\f\xcd\xf3\f\\\x82\xf12SPc\x1cN\xbe\"6\xbb/\xe8T_\xdf\xff|\x18+\xef\xb7\f\xce;\xd8Ȍйv\xbd\xb7\xa3\xfe\xfa|T\x10~A\x1f(\x06U.\xe7rA(y\x86\x9ds]\xa8 \x96>4tΘ^\x01&\x7f\x90Ϟa\x87`\xd2ٜÖ\xcb\r\xae=C\xc2\xf5O\xb5\x01\x0e\xed\x9a|X\xec\xf0d? \"0\x86\xcfe\x03\u05fc($r'閩KB\v\xb8?a\x9bY\xacҟ\xa3\x9f\xfaD\x0e\xf8I;ZZ\x89\xa9\x98\xcfij@\x99\xc9%\xa8k\xdf(ge\x9c\xc8\xc9ȝ\xb8 \xf7\xd2\xd8\x7f0@\xd3\xc8(?K\xd0\xf7\xd2\xe0\x97\xb3`\xd4-\xfc\x9c\xf8t3\xa0\xa0\t\xa7\xe5-\xc2\xfa9?g\xd3,\xb7E\xdc3M\ue10dW\x1cJ2\xa7\xc2\xf4\xae\x9b\xceMT\xb7\x1a\xd3uB\x8a\x05\xda\xcc\xe4L\x1e\xdfR\r\xd0\xfd\xe2I\xfd\x84_\xad\xb1p\xbf\xb8$3\xa7\x05\x94!\xb2\xc4\xec'5\xb0fE\xe6|5\xa85\x90ƪ\xf0<\x8e\xc8T\xac~7DZO\x9e\xf5\xee\xb7\xef\x8b\xe7\x98/XX\x93\xb3\xf0\x10\x8c\xac3p\xe0uw9\xbf\x9f\x85\x95ٌ^\x81\x13f\xbb\x8e$Gǻ\xe6 \xe5\x05\xe8@+\x8e.\xce,uiY\xe2\x11\x1a\xe5\x0fGX\x94#x\xe1X\xd5\xd0[\xbb3\xc15m\xacZ\xf8okiQ\x9a\xfe\x974\x94)}I\xae\xf1\xa4\x8c\xc3\xe07\x9f\x87\xeb\x81ɘ\xb2\xb1SY\xfe\xd9Pnm\xbfU\xe0\x82\x00w\x9e\x80\\\x1d\xf8E\x17d[I\xed\xcc\xf6\x8a\x01\xc7\xf3\x8a\xf7ϰ{\x7fa\xa7\x9f\x9d\xb2\xafd\xde߉\xf7·8P\x18\xd1ᐂ\xef\xc8{\xfc\xed\xfdK\\\xa9LN\xcd\xec6`њ6y\x1c*\x92\xc9\xfa\xae\r8\xa6\x9f\x9b\xef\x92\xf2\xdeɞ\xdam\x16\x8b6R\x9b\xcf\xe9\xbc\xe1\xc8z\x1e\u0088\xa1g\x9cȱ\xcdF\f>\x8f\x16\xf5\xbdu\"W\x06\x94\xcf%:\x1b\x10\xe2\x8f\x17Ff\xa9S\x99\xfebc2\x90\xc6\xfc\xaeE\xf0\f7\xb9\x83\x9b\x9c%\x1e\xe3\xb0Z\xbc\x1c\xe9\xed\xdf~\xef\xe53\xad\xe4ڿ\xfb\x1bym\x87\xba\x90uM\xf7O5\xb3\x96z\xe3F\x06\x9e\xf6\x80\x1c\xf5պEyε\xc8\x1d\x0f\xe1\xf9喙\x8a\tB\x83\xda\x00\xe5\x19\x8a\x92F\xa6rةVQM\x96\x00\"\xa6\xe8\x7f\x04W\xa2f\xe2\x0e' \x1f\xcf\xe0zDt\x9d\xd3ٽ\x894\x89\x94\x8f\x1f\x9c\xc9jdI\xb6\x15(\x180\xc6a\xde\x1d=U!M/eq\x84C\xda\xc8\xf2'MVLi\xd3_\x82&\xadΥ\xf5\x91\xe4\xb3\xeb\xfe\xcaj\x90\xad9'\x82o\xbbi\x06g\xcd5\xfd\xce\xea\xb6&\xb4\x96\xad3\xe6\x86\xd5\xf1TףwK\x99\x89\xc7V\x98\xbf1Ғ\xa0\xe1`\x80,a\x95>\xefM\xb5B\n\xcdJP\xa1J\xc1\x91\x8dI+\x98+\xcax\x9b:%J\xb5c#`q\xab\xd4I\x01\xf0\x177\xb2\x97w\xac\xe4v\x88\xa0̽\xe3A\x1a\x10\xb6\"\xcc\x10\x10\x85\xc58(\xa7\x92q\n\x8f\fD\r\xcb\xd5sy\n\xdc6\x10m\x9d\x87\x80\x05\n$\x13\x93)\xb7~\xf7O\x94\xf1s\x90\xcdr\xde'\xa9\x1e\x81\x96\xa7\xe4h~\xed\r' t\xab\xf0\xf0\xdf\xe9\x8e-\xe3yk\xb6\x94#\x9c\xb6\xa2\xa8\x00\x95\x90\x18\xea\x06\a\x9e\tm\x80\xe6\xf2\x82\xf5\x8aZ!\x98X\xe7\xd1.;\x11\xda5\x87\ua954\x1c\xe8\xf8)d\xd7,\xae\xdf@\x13\xfd\xdaM\xf3BM\xd4\x11\xc1\x1d\x9b#\x1d\xb2)j\x95\x16\xa1\xc6@\xdd8\x91\x93D\xb5\xa2o]Π\x88\x8e\t\xc3\xfd*^3\xbef\x82e\xd0v@\xd7;\xc1L\xdfy\xb4 \xce\xea<\xda\t\xa2;pJ\x86\xedn\x00\xc0\nh\x88Cp\xed\x91k\x8ep$\x97@hYB\xe9r\x97\xd6\x15\xf1a\x89+|\x1b)nH\xee\xeexO0\x8b\xb2\xa1\r\x82N\xccê\r,Z\xf1,\xe4V,0\x18\xd7G\xeb\x90\x13\xb3T/\x9dޜ\xac\x8c\xe6\xf5K\xbe\x9a\x9e\xd3BC~\xcd\xe7\xa9\xe0?\x9dA\xcbd\xf3\xcdQ\t\x8f).\x98\xd3k\xae\x00{\xe4\xc7\xd9UL\xcd?1\xd8\x1fJ߸b\xe9\x17\x95\xc5ݥA\xf5\x9c\xc2m\x05\xa6\x02\x15J\xb3\x17X\x92^N\x9e\x90v\xc1K\xac\x93\xb3L\x15\\dW\xfe\xb9W9\x87\xd1M\xcb\xf9\x85\xe5m\xda\xf2d8l$\x8a\xd8!geՏ\xa5=\x86\x9c\xea\x8bl<\xf6+-\x86\xf5\x85\xb1\n\"\x14\x18\xca0\xb3\xa7qj\xbfXX\xda;\xdf\x1f\x96S`\xfe/,\xff\x0f/=̨\x94\xc8Gcn\x95fDb\x02V\x82\xc1zh\xec\xea+|?_\xe8\xfbc\xe1\xd4@\xfd\xa5\xf1\x123\xea\xc2f\xa05\x01g\xaf\xde\x04\xadA\xab\x9d+\x10\xed\x80\xcf\x19\xda\xf1ׅ\xbb\x05\x11\xc0\xa4\xf8\xf5k\x05A|}\xf5>\xd3\xe4\x9fI%\xdbDU\xdf\x04\xcaf\xaa;\xe67<(\xf4\xf0\a\n`\xe8\xe6\xe3\xe5\xf0\x17#}\xd9\af\xd1\x12\x800(\xea2\xb3L\x94l\xc3ʖ\xf2 \xb5\xdd\x1d\x02\xc7@\x1d\x9f%\xa0IE\x04\xe3\x8e\x01\xc3\xf8\x01Ñ/\x8d;\x969Z\xc5M\xfb\xa2y\xd5!'ׄ\fk>F\xac\xe1\xb1\xc7\x17\xafR\x05\xfb\x87\xd4z\x1c_\xe1\x91\x13I\xccTs\x9cPÑY,\xf6\xe2\xf3\x96\x9c*\x8dcb\xee\xb3Ud\xbc~\x1dF\x16~\xe6k.\x8e\xc1\xce\xd9\xeb+ް\xaa\xe2mj)2+(^\xaf\x142/\xfa<\xa9\x14`>`\x19\xaf\x82\x98\xad}xQ@sҖfk\x1a\x8e\xa9d\x98\xa5N\x9e\x98\xbdY\xad\u009bU(\xbcm]\xc2$\x17M\xfexL\xe5A\x8c\x93~\xa1M\xc3\xc4\xfa\x90)rYg\x92m\xe6Y\xe6~o!\x03\x9e\xe9\x873]t8\x12\xfa\xba\xeb҉H2\xa4-\x990\xf2\x92\\\x8b\x9d\x87\x9b\x80\xd3\v\x1f\x854\a\x17\xd9첶\x8c\xf3\xfem-\x04;\r\xcaߙԴv\xab\x1a\xf3\xf6\x93t\x95j\xe0\x94\x9f\x148~ك\xd1ώ\xbe\xa5\xe7_\xb7ܰ\x86\x83\xf5\xe86\xacL\xde!3\x15\xec\"\x92\xff&\xf1\x86\xd4r\x87\x90\xbe]\xf4\x9eQ\xec\x9eq\x926\xbf\xc9\x13\xb6\x97Q\xcc~\\\x11{\x06\xcdrE\xf1\r\x8b\xd5߰H\xfd\xad\x8b\xd3g8k\xe6\xe7\xe3\x8a\xd0O>\x81\tG\xfd\xf7\xb2\x84\a\xa9\xcc\\p\xf2\xb0\xdf?q\x92\xda\v\xd8$/\x89\b]\x13\xbb\xc4\x10Ç\x17\xa7m*}\xe8\x19\xdc\xe9_di\xd76w\xc6\xf2\xb8\xd7\xfd\xe0\xae\xf2\n\x14\b\xf7\xcc\xc7\x7f>}\xb9\x8f\xf0S>\xaf\xf7\x8c\xf7\x9e\x97p\x1eL\xe9\x91\xe3\x8f\xe6|1\x93\xc3\x16\xfa\x00\xaf|.B\x1b\xf6\x1f\xf8\xaa\xdb\v\xd2A\xd7\x0fw\b#\xf8i\xf8L\\\xac\xa2\x88'\x96K\xb0\x16+\xa2jT,\xeeV\x03\x88Ê\xdf\xfe3JP\xba'\xb3\x82\xc5d\xa1\xc6\xcb\n\xdeÝ[\xc7\xd8,\x9f\xac\xd3(vD:\x8e\xac\x98*\x17\rUf\x87l\xa3/\x06k\bff*\x9d3\xaaX\x0f\x9f\x01K\xa27\xbc\xfe\x85g\x91\xbbfxڻ\x8f\xbbS\xd61~\xffd\xf6\xe6\xc9+\xaec\xdcb/\x10S\x89\xcf\xc9\x02\x93WK\x93yM\xf4\xf0\xed\xa4\xb4\xcbc\x1c=\xad\xe7l\x14\x1dRM\t0v<\xaa:-h\xa3\xabēK/\xd3u\xf8\x1a\x99\xa1\xa6}\xc9&\x1d\x80\xc1>YQ\xf5\xb4\xd5\x16\x82>\v\xdbFi\xc5a)\xddnm\xb3\xab{a\xfc\xa2\x97)x\x9b#\xe1\xcc\xe7\\N~\xc8šgD\xfd`\xf6˪\xb6CL\x9dp\x18<\xeb\xdae\x14\x19O;\xb1\x99π\xe4\x19\x8c\x13\x9e\xfe@|\xe5\xe2\x8a$_\x04\xc9|\xf5\xe3\x0fE\xf4\x84V\xd3E\x05e\xcb\xe1\xd47\xff\x9ez\xe3\xe7_\xfd\v\xb3e\xbc\xfbg\x91\xdd3\xd0\xd6g\x1e\xbe/\xe8)\xe1!\xf7)9\xe6\xf0ap\xe0\x9e\x17+\xdcK\x94E\x01Z\xafZ\x1e\xaa\x94\n\x05\xd4@\x19\xba3\x1dW|T\x9dM\xdbpIKP7R\xacX\xe2\x84d\x80\xd6\xff\x1at\xde\xe3\xd9\x02?\xb6\xaa{\xdaq\xf2Y\xbc\x17i\xae\x86*\xca9\xf0O\x8c\x83\xfeYn\x85]W\x86@>\xa4\xc6\xf5\xeee\x15\xad\xb2f}GD[/\xad\x93\vƌ\a\x8b+\xa9\xa6+\xa4\x1dޙ0\xb0\x86T|\xbdU\xcc\xc0SC\x95\x06\\Q\xc6\x0e~\xdd\x1b\xe2\xa2\xcf\x15\xa7kW\nW\xb2\x82\x1a\x88\x06\x18g\x18[>\x8e\xd7\b\x8b\xef\xb02I\x8e$\xbd\xb2\x85z\xecJƨX\x8f=/\x9a0\xd5\xc9\aF\x9dE.hc\xf0\x02\f\xd2\x11\x89h<\f|\xb4w\xef\x8d\xd1\x01\xd8qN\xf3e̾`N\x1bZ'\xa2\x84y\xbdss\b\x06\x9f\x05Ve\xaf\xee\xae\xff\xc0b,\xb0#[\xaac1u\xd2\xf7\xee`;0\xe8\xaa[\xd0P\x12\u0600 V\x14)\xe3PNq\xeaWL$\xab\r\xa8\x9ft\x84\x83\x95\x80\x96ş\fU&.\xfdЏYIUSsEJj`aG\x9f溥\x9fIU\xea\xc4\xe3@\xbc\xd9\xe6ţ\b\xd7n\xac\xf5s\xf7\xd1jК\xaeC\x10\xba\x05\x05d\r\xc2\xe2=\xe6\x16\x93\x1eS\xb8\xd2\xe7\x8dE,-\xb5(\xa4\x85i\xa9\x9f\xc0\xb9p\xf1\xf44\xbcO\x8cQ\xeczTE\xa7U\x85\xbf<\xf8\bT\xef?w}\x80\x8bO\xfd\xbe>I\xecv\xec\xceF\xa8+\xf0\xc4\a\x8f\r\x8b\x91uJ\xa6\x8dę\x8f2'\x95\x94\xcfYn\xf6\xe7رK'1\xe1X\t\xafL.ekz~\x8eGxb\x99\xf8\xfc\xe7+\xdb\x17\x84y\xed.P\x8d\xe5V\xf3<\xbd\xcf\x03H1\xbc\x95\x86\xf2`d,_\xc6\x0e\xd5\xc4\x03\x02O\xe1\xf1d\xcew\x17\xfb\x90\xf7^e\xef`W\xddS\x9e^\x13t\xd7\xc7\xc7Ҫ>\xeb\x97\x04\x12_\x01\xed|\x92\xb17\x17\xe7\xec\x1fB\xfd\x84\x8b\xca\xc0\xf1\xe7\xae\xf7\x18\x1e\xdd2\x9d\xc3\f\"\x1di\x12\f>L\x15%ㄥOx\xa9ME\xf5\x9c{\xfa`\xfbD\xb7\xa3g\xae\xa2\x13\xfa8\"\x95\xe9{\xae\vr\x0f\xdb\xc4W\x87,<\xfdB\xa9Jt\xb9\x13\x0fJ\xae\x15\xe8C\xa6[\xe0}F&֟\xa4z\xe0횉/\xe3\x95\xdfS\x9d\x1f\xa82\xcc2\xad[Ob\xecM\xb0q\x89\xdf\xe6G\x8f\xff\xc0\x04\xe5\xec\xf7\x94.\xef\xff87Ä\xbek<\xf2N\xb1P\x01\xf1s\n\xd0k\xe8\x9ft\xcf\xfc\x84y/ɽL\x8a\xb1? fC\xa0L\x93%h\xb3\x80\xd5J*\xe3\xf2\xf7\x8b\x05a\xab\xe0 Y\r\x81q\xa2{͞\xb0T\xe2=\x1e\xbd\x05\x87e\xe5S\x89\n\xad\x0e\x86\x9c5ݹ\x8c$-\n\x1b\x13\xc0\amh*6y\x91\x9e\xc6P\xd5\xcbJ\x8e\n\xb9\xeb\xf7\x8f9\xbe\xa8>\x10\x9cC\x1d^gw\x06\x9d\x8f\x9di\r^\xcb \xdab\xef\x14eB\x9c\x1a\xbb\x1b\x0f\xbb\xf3L\xcd\xd7\beL=\xfa\xfd\r\x1e\xe2\xf6\a\xac\xbe\x93%[QQ\xb1\x1e\xbd\xd0V)ٮ\xab\xc0\x9bc\x0e\x11)[\x8c\x9c\x1bT\x05:\xfc\xc7!\xa6U\xa2wh\xe7k,ƴt\\\uee0f\xf2\x02E\xad\xba\x8b-\x9d\xaa\x9a\xb0\xf9\xd9Y\xc2\x11\x88\xb3\xb6?\x01\x91\xea\x9d(&\xaf\xe0\xf8@\x9bM\xdc՝\xc2P\x12\tQ\x1b\xbf\x1a\x12\"\xc41$\xf4}\x89.\xe2\xf9a02棜\x88\x8ei'\x06\xb78\rj~\xd3}'h\xe8\xee\x1c\x87\x0e=\b\xfeNJ\xbb\r \x1c\x13\xf9\xe2\xdc\xe9\xb8\xf7ǍX7\xd1ۺ=9v\xfd\xb6\ac\xef\n\xa4\x8db\xbbiB\xbc\xf9\xf7l\x95\x92\x17\xf7\xbf3-9\xfc\xc3\xc1\xafo|\x95qK\x95`b}\x12F~\xf5c\x13\xf1\xbc\a{Έ>\xac\xfc\xd5b\xfa\xa4Y:\xf8\x88\f^\xf6\xf0\xecg\xf2_\xfe/\x00\x00\xff\xffP\a\xb5\x16Cm\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=]s\x1c)\x92\xef\xfa\x15\x84\xeea?B\xdd^\xc7}ą\xde|\xb2gO\xb1\x1e[ai\xf4\xbctU\xb6\x9aQ\x15\xd4\x00\xd5r\xdf\xde\xfe\xf7\x8dL\xa0\xbe\xba\xe8\xa2Z-ygǼت\x86$\xc9L\xf2\x03\x12X,\x16g\xbc\x12\xf7\xa0\x8dP\xf2\x92\xf1J\xc0W\v\x12\xff2\xcb\xc7\xff6K\xa1\xdelߞ=\n\x99_\xb2\xab\xdaXU~\x01\xa3j\x9d\xc1{X\v)\xacP\xf2\xac\x04\xcbsn\xf9\xe5\x19c\\Je9~6\xf8'c\x99\x92V\xab\xa2\x00\xbdx\x00\xb9|\xacW\xb0\xaaE\x91\x83&\xe0\xa1\xebퟖo\xffk\xf9\x9fg\x8cI^\xc2%3\xd9\x06\xf2\xba\x00\xb3\xdcB\x01Z-\x85:3\x15d\b\xf4A\xab\xba\xbad\xed\x0f\xae\x91\xef\xd0!{\xeb\xdbӧB\x18\xfb\x97\xde\xe7\x8f\xc2X\xfa\xa9*j͋N\x7f\xf4\xd5\b\xf9P\x17\\\xb7\xdf\xcf\x183\x99\xaa\xe0\x92}®*\x9eA~Ƙǟ\xba^0\x9e\xe7D\x11^\xdch!-\xe8+U\xd4e\xa0Ă\xe5`2-*K#\xbe\xb5\xdcֆ\xa95\xb3\x1b\xe8\xf6\x83\xe5g\xa3\xe4\r\xb7\x9bK\xb64ToYm\xb8\t\xbf:\x129\x00\xfe\x93\xdd!n\xc6j!\x1f\xc6z{Ǯ\xb4\x92\f\xbeV\x1a\f\xa2\xccrb\xa0|`O\x1b\x90\xcc*\xa6kI\xa8\xfc\x0f\xcf\x1e\xebj\x04\x91\n\xb2\xe5\x00O\x8fI\xff\xe3\x14.w\x1b`\x057\x96YQ\x02\xe3\xbeC\xf6\xc4\r\xe1\xb0V\x9aٍ0\xd34A =l\x1d:\x1f\x87\x9f\x1dB9\xb7\xe0\xd1\xe9\x80\n»\xcc4\x90\xdcމ\x12\x8c\xe5e\x1f\xe6\xbb\aH\x00F$\xaaxmH8\xda\xd67\xddO\x0e\xc0J\xa9\x02\xb8\x80vX4\xb6\nu%\xa0\x80\xe6\f\xddN\x8d\x16FH\xb6\xae\xd1#]2\xd4\x12Q\x19\x11\xd2X\xe0\x11a>\x01\xef\xe0kV\xd49\xe4WEm,\xe8\xdbLU\x90\x87E\xa6Q͜\xca\xc3\x0f\a!\xfb\xf8\xa5\x10\x19 \x1f2WiA\x8b<1\xd1nC\x99]\x05n\xcd\tY\xed\x87\xd0\xc6(\x93\xbaŀņ\xe7\x7f<\xbf \t\xe8\xf7\xde\xef\xc70\xae\xa1!\xd3,\xddL\x16\x7f\xbc\x85\xb0PF\xa8;\xa9\xa3f\xf0\x9dk\xcdw\a\xb8\xde,\xa6\xbd\x00\xdfc\xb0\a\x9c\x97\xa1\xda7\xe2\xfd\xb0\xff\xdf\"\xf7O\xcboC\x8b\xce\\H\xe4s!\x8c\xed\xb1ٸU,$\xebX\b\xe9\t$\x1dLT\x93S\\\xfd'!\xe6I\xe7Nl\xb24\xb2\xe9'\xc0\xbf\x14%7J=\xa6P\xef\x7f\xb1^\xbb\x84\xc52\xda\x18a+\xd8\xf0\xadP\xda\f\x97I\xe1+d\xb5\x8dj\x16nY.\xd6k\xd0\b\x8b\x96\xf9\x9b]\x81C\xc4:\x1c\xbe\xb0\x8eʊV\x18\x8c\xabe:\xb2\x94\xa8\x11\x1b\n\x05\xa8Q\xa8\xce\xc1\xc1Ђ\x1c\x88\\lE^\xf3\x82|\t.37>\xde\xe0\x17\xd3j\x13\x02\xb1\x87\x7fT\xaa]q\x0eM\x18$2\xb1\xb7\xea\xa5$\xa0\x8f_bl\xb4_5N\x89\xb0\x94p\xb0od\xa6\xae\v0\xbe\xbb\x9c\xdc\xe4V']\xb4\xccrk\f\x05_A\xc1\f\x14\x90Y\xa5\xe3\x14J\x91\x03WR\x95n\x84\xb8#Z\xb6\x1fm\xb5\x83\x99\x00\xcb(\xc4݈l\xe3\xdcW\x144\x82\xc5r\x05\x86VExU\x15\x11\xd3ՖI\xe1\xf0\x9dM鍶$h\x90!ܘ.iK\xa2~n\xcb(\xd9۹٧\xfa\xf8:\xff(\xbe\xbf%\xa2\a\xabs\xa4\xb0Oh\x12F\xfb\x05\xc9\xf3!Jz\xa4\xb8\x00\xb3\xec\xac\xce\t\x1b\xbe\xa60\xb4\xe7?\xeem\xa5\xec\x11\xe5\xd7Ż\xe3&\xcc\f\xd6MΩ\x97e\\\xd3Ϳ\b\xdf\xc8d\xddz\x8b5\x8bg\x1f\xbb-/hW\xc03$\xbf`kQX \xa7j\nQ6\x83s\xa7$P\xaa\x05f\xb4Il\xb3͇f\xef(\xa1ŀVC\x00\xceA\x0fQ\x0e\xf1 \x01$k\\\v\xda4\x15\x1aJڌ\xa5H\xb2\xfb\x85\\\xc1w\x9f\xde\xc7c\xcfnI\x94ԽA%LZW\xde\r\x1c\xa3.\xae>T\t\xbf\x90\xbf\xd6\x04\x82n\x13\xfe\x82q\xf6\b;\xe7bqɐo)\x8b\xff|\xf8*\fv,s\xf6^\x81\xf9\xa4,}yQ*\xbbA\xbc\x06\x8d]O4A\xa5\xb3$H\xc4n\U00088ce5(\xa8\r?\x84a\xd7\x12C2G\xa2\x19\xddQ\xae\x90\xeb\xd2uVֆ\xb6Z\xa5\x92\v\xb7,6֛\xe7\x81\xd2=\x16\x9c\xa4c\xdf\xe9\x1d\x1a#\xf7\x8b\xcbZ*x\x06yآ\xa3t\x1an\xe1Ad3\xfa,A?\x00\xab\xd0,\xa4K\xcb\fE\xedG6_\xbc\xd2=\x87n\xf9\xbax\xacW\xa0%X0\v4k\v\x0fŪ2\x91.\xde&\x8c䜌\x95\x05\xce\xf5ĚAZ\x92\xaaG2r\x0eWO%\xd63\xc9D^\x04\xb9]IR\xd0Ml\x9dg\xbdf\xca\xcd1*\xa63\x16\xe7\x02\x94\x9c\xb6\xd6\xfe\x86\x96\x9ef\xe3\xdfYŅ6K\xf6\x8e2{\v\xe8\xfd\xe6\x17&;`\x12\xbb\xadh\x95\xfd\x97Zly\x81\xfe\a\x1a\bɠpވZ\xef\xf9j\x17\xeci\xa3\x8cs\x1b\x9aM\xbb\xf3Gع\x1d\xe5\xa4n\xbb\n\xeb\xfcZ\x9e;_fO\xf14\x8e\x8f\x92Ŏ\x9d\xd3o\xe7\xcfu\xeffH\xf4\x8c\xaa=Q.y\x95.ɔ7;'\xd0\xc0`=8DظI \xc5\x00a\x8a\x02ɢ\\)\x13I\x16\x89\xa0\x95 \xe87\xcaX\xb7\x0e\xd9\xf3\xf7G\x17*UX\x9cd|mA3c\x95\x0e)\x99\xa8\xf8S\x96\xe2\xbb\xe5n\x03\x06\xfc>\x94_\xf4t\x801\x8a=ou\x83\xb3*\xe7n/\x8c:\xe2\x19yOԶ\xd2*\x03\x13͋hK\xa2m\xeaQp\x9f\x0eͺ.w\xd1\xdf:Ik\xa7,J\x872ϑG\xd2\x1d\x11\x19}\xf8\xdaY\xa2F\xed\x82\x7f\xa7H\xeb182:\xafQ\x96|\x98\x0e\x9c\x8c\xee\x95k\x1d\xe6\x98\a\xe6\xc2-\xfdP\x93Ι\xe3u4\xa2\xfc\xcf\xe6ڔB^SG\xec\xed\v\xbaC^\x8b\xc7ң\xc6\xca\xf1N\xfaU\xe8\xac\xe5^\xf3\xc1\xe7\xd4)\xda\xf8\xd1\xd0c\xee\xfe\x9e\by\xd7R\xd9\xce2\xceL'\xbaR\xf9\xef\f[\vml\x17\rs \xb1j\x14\xd4\x11\xa1\xa7\xfc\xa0\xf5ё\xe7g\u05fa\xb3\xa0\xb8QO>qzN\xbc\x1dH\xba\xe1[\xf0\x99\xab 3UKZ\nC=\x80\xdd̀\xe8X\xe3\xac@\xa2\xbd\xeb4\x96u\x99N\x90\x05I\x92\x90\x93\xebf\xdd&?p\x91\xb6nŎc\xab=\x94\xc39V\x8e\x9fG!\xc1\xb3\x9bN_\U000af8acK\xc6K\xe4!\xb9\x1d\xa2\x84&\xa3ޱ\xbbI\xfb\xc4\x16d\xb4\xac\xc2YV\x15`\xc1\xa7m\xce\xc0#S҈\x1c\x1a\xd3\xefE@I\xc6ٚ\x8b\xa2\xd63\xb4\xeal\x92\xcf\r¼69}d\x95\x8eȂH\x94\xb8\xce>\xc3\v\x9e\xd6\xf8\x95\x9e\xe7Ǧ8\x8c\x1a\xe6\xfb\x8b\x95\x16\xca\x1d\x068\xbd\xcb\xe8ӎ\xb9\xdc}\xf7\x19\xbf\xfb\x8c\xdf}\xc69\x1d}\xf7\x19'\xcaw\x9f\xf1\xbb\xcfx\xb8|\xf7\x19S\xcaw\x9fq&\"\xdf\xcagL\xc1pAk\x9c\a*$a\x95\x98\n1\x85\xf6D_>\xe9ǟ\xd58I.\xf3\xf58ȑC<\x91\xe3\x171\xaf\xa35^Mr3\xce\xc00w\xdc)\xca\x04\x87\xf9\x04\xa7g\x02\x02\xa7?=s}\x10\xf2\tO\xcf\xf8!\xa4E\x18G\x9d\x9d\tD\x9a\x7fz\xe2\xc2'\x11\x95\xc0\xc3V\x8aK\xff\x88\x8d1&I\tx|\xe3\xe4\xf7\xbd\x8c\xc9\x17\x90\xa5W9\x913K\x9eFY\x7f\xfe\xc7\xf3_\a\x8bN˔(\x1b\xf6i\xeb\xd4xL?b,\xdfM\x8d\xecg\xa9\xfez\xa6\xc2Ie?\xf5DMC\xe4\b\xbc\xbeX\x0f\xa8\xfck\xd27\x16\xcaϕ\xb7\x96'8a\x7f=\x02/\xe9\x8c=7;\x99m\xb4\x92\xaa6~M\ba\xbd\xcbܽ\x03\x01dL\xd8G5\xc8\x7f\xb0\x8d\xaa#\xa76&H\x9b\x90E\x9bF\x90^R\xadO\x8c\x00˷o\x97\xfd_\xac\xf2)\xb6\xecI\xd8M\x04\x18\xddG\xc1\xf3\x1c\xe3\x82\u0381\x1e\xaf\a\xc2UIC\xa1\x8c\x00S\x9aIQ8\x89\r\x10z\xf2\xca>Wnu\xf0h\xbfiz\r+=\x11wn\xfam\x93-9\xed\xbe?#\xe9\xf6\xa4G\xa3\xbeYZ\xedqɴ\xa9+\x94\t\x89\xb3\xe9\xe9\xb2)lu%=I69BNM\x88\x9d\xbb\x02\xf1\xa2ɯ/\x93\xf2\x9aL\xb3\xb4\xf4ֹ\x14{\x95T\xd6WN`}\xbd\xb4\xd5\x19ɪ\xa7?\xf5\x92\xbe\x96~tveڲ\xcc\xe1\x84Ӥ4Ӥ\xa5\x9b\x94\x01\x1f5Ԥ\xf4ѹI\xa3I\x9cL\x9f\xae\xaf\x9a\x16\xfa\xaaɠ\xaf\x9f\x02:)m\x93\x15\xe6&y\x8e_r\x18ʴ\x03P|\v\xe1|.\x99\x94\xee\xb9\xe6ϊ;?\x0f`\xa1\xb0\x047\xf5\x15〲.\xac\xa8\x8a\xf6>\xb6X\xc0\xb9\x81]sY\xd1ϊ\x8e\xc8\xfb\x9b\xba>\x7fi$~9\x88j\xb8aOP\x14\x8c\xc7\xe6\xe6\x1e\x152w\x0fh\xa6\x16\x80\xb6\x11g\xb9\xbf\x8c\xc9_\x1ez\xe1\xa6\v\xdd\x06@\x16\xb6\x8c-\xf5qy\xf8\xa6\xaf\x83\x06,U\x8f\xedy\xe6.ޠo\xbfԠw\x8c\xee\x1dk|\xb3\xf6P\xa9\x9f\xe8\x06\x03Ӡ~\xbc:<\xb4g\xb2\x17\xe0\xb4ꁽ\x93\xce#\x18\xe2DmP\xef\xb4\x01\x1d*U\x8cӢ\xfdD@H\xd5@\x884Mq\xfe眲|\x89\xf0\xee\x14\x01^\x92\a4\xcf{\xfd\x86\xa7'\x8f=5\x99\x9e\x8c\x92tJ\xf2%½9\x01\xdf,\x7f5\xfd\x14\xe4\xfc\x8d\xe7\x17>\xf5\xf8R\xa7\x1dgP/\xf5t\xe3|ڽ\xd2i\xc6W?\xc5\xf8\x9a\xa7\x17g\x9dZLNϚ\x95q0'\xb5\xea\x19\xc7\xed\xd2r\t\xa6O!&\x9e>L\xcc4H\x1b\xfc\x91\xc3N<]8\xffTa\"\x7f\xe7L\xe9W>=\xf8ʧ\x06\xbf\xc5i\xc1\x04\tL\xa82\xffT\u0cf7\xa4\x94\xceAOn\xfb͑\xdaIyM\x8d\xe5\xfa\x88\r\xf6\xb5\xc2m\xb2X\xab\x17\x03\x90Y\xf2\x17\xf9ӣ\r\x87\xb6\xc1Q2;\x1eQo_\xb2u\xd7\xfa\x0e\xb1\x7f\xcd\xc1m]\x1a\xa88\x1a\x00\n\xdc(5+\xea*|\xe0\xd9f\xd0Æ\x1b\xb6V\xba䖝7\x9b\xc5o\\\a\xf8\xf7\xf9\x92\xb1\x1fT\x93\xabӽ/͈\xb2*v\x18\x89\xb1\xf3n\x83\xe7IIT:C\xcf7\xaa\x10Y\xc4\xe7\x1c\xbdW\xcf5ػl\x88n\xfe\xcb:\xd9\"\xb1\xc0\a\x9b\x8bp\xebb\xffJfw\x9f\xfb\x91k%\xbc\x12\x7f\xa6'\x95N\xb0\xea\xf6\xee\xe6\x9a`\x051\xa2\xb7\x9a\x9a\x04ņ\xe5+@\x97\xa1\x1d\xfb!}r\xbd\xeeA\xed\xe7\bw\x1f\xab\x80ܽL\x12\xdc\x16\xaf\x9a3\x85Z\xeb\xe6\xda\xe1r\xa8'\x94/.wL\xf9\xa7'\x84\xce\x17\x15\xd7v璉.zx\x04\xbb>\xb5jv\xd0Z\xed\xbf\xbc\xd2-=\xb2\x87GWh'{W\xf5\x93\a\x86\xf4|\x0eN\x87OUO\x9e\xa7~\x01\x9c\x0e\xbbP\v\xa2b\xe4\xa7h\x06\xe4\xc9W,\x8d\xbf\xa1\xffG\xb5\x85\xf7ѕ\xcb\xfe\xeb+\x83&#\xa9\x89\x01*]2\x1f\xa1`\x9b\x8fHw|?O\xed\xc5s\r\x03*\xfe\x8e\xf0\xe7,N\xde\xf6A\x8d?HB7\xa8\x87Nc^\x15=\xf5\xb4c7\xf7\x14\xb76\xaa\xd4O}\x1f\xb7\x86\xe5ɐ`\x10\x81%\xe4\xc17ZNEF\xab4\x7f\x80\x8fʽ\xad\x93\"&\xfd\x16\xbd\x97\x97\xbc\xe7\x16\xf2\xb5\xfd$\x8c)z?\xb6!\xc0\xf6|\xc6\xdeE\xff\x88\xed\x91O\x19X[\xba\x91ғ&\xef\xfd\xeb$\xa8\x8f\r \v\x02\x05\x1c\xb4\x15\xfew\xa3\x9e\xe8\x02\xfc\xf8\x1asx@\xa4\xf3\x86\x19\xd0A\x11J\xe1=j\x98uU(\x9e\x83\xbe\xa2GT\x12F\xfcS\xaf\xc1\xc0\x1d\xe8?\xc5\xe2\xedfd<\xa1\xe7\x17̒A\x8f\xae(\xa0\xf8A\x14`\x1c≦\xe1f\xbfec)\xear\xe5<\xd55\xfe\xd8tr\xc02\xbb\xa1\xd2\x06C\x05\x1a\xfdD\xb7\x15Q\x9b \xf9\x87\x89\xc1\x1a>\ni\xe1\x01\xc6c\xe8\t\x9b\xe0\xdeh \a (0\x8a\xf8\xfe\x12[y\xec\x11\xe4>\xdez \x03\xcdbdL\x8e\x95w\xabn\xee\xaf\f\xabeN\x1b\x00\xf7\x7f\xbe=J~\xb7\xbd\xf7e\x82NHQ\xef\xf7\xe3-;!BG;\x91O\x1fW\xe21X\xdc\x18\x95\t\x8a*\x9e\x84\xf5\xd79\xbe\xdc\x1d\xe2\x87\x02\xc4\x03\xd2Q\x1b\xf8\xfc$A\x7f\t\x16\xc8\\\xcbػ-\xd3\xda\xef\xa7=h\xd1\xf7Z\xac¾G`\f\x000\x15\xf6\xb9\x8c{\t(l\xaf\t\xd3H\x1c\xc4>\x01\xdcyG\xc8ik\x85\xb4\xe3\x9c!n\x9bVt\xd8tDCN\x8b\xed\xfd\x00\xc6 \x93\x9d\x1e}j\xaa\xb8Ӧ\x86\xfd^\x8cy\xa3\xb4c\x96\xe1@\xff\xb0\xf7kT\x83\x1f\xd4\xde1\xcd=\xaaF\xf6>\xd2CxyGr\xbc\x97\xde\xfdR\xaf\xda\a\x15\xd8\xdf\xfe~\xf6\x8f\x00\x00\x00\xff\xff)\x00\x87w>{\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=Ks\xdc8sw\xff\n\x94sؤJ#ǕG\xa5ts\xb4v\xac|\xdf\xca*\xc9\xf1\x9e1d\xcf\x10\x9f@\x80\v\x80\x1a\xcf&\xf9\xef)4\x1e|\fHbF\x1a\xednjqQ\x89$\x1a@\xbf\xbb\xd1\xc0\xacV\xab7\xb4a\xdf@i&\xc5\x15\xa1\r\x83\xef\x06\x84\xfdO_>\xfe\x9b\xbed\xf2\xdd\xd3\xfb7\x8fL\x94W\xe4\xba\xd5F\xd6\xf7\xa0e\xab\n\xf8\x116L0äxS\x83\xa1%5\xf4\xea\r!T\bi\xa8}\xac\xed\xbf\x84\x14R\x18%9\a\xb5ڂ\xb8|lװn\x19/A!\xf00\xf4\xd3?^\xbe\xff\xd7\xcb\x7fyC\x88\xa05\\\x11\x05\xdaH\x05\xfa\xf2\t8(y\xc9\xe4\x1b\xdd@aan\x95l\x9b+ҽp}\xfcxn\xae\xf7\xae;>\xe1L\x9b\xbf\xf4\x9f\xfe\x95i\x83o\x1a\xde*ʻ\xc1\xf0\xa1fb\xdbr\xaa\xe2\xe37\x84\xe8B6pEn\xed0\r-\xa0|C\x88\x9f:\x0e\xbb\xf2\xb3~z\xef@\x14\x15\xd4\xd4͇\x10ـ\xf8pw\xf3\xed\x9f\x1e\x06\x8f\t)A\x17\x8a5\x06\x11\xf0?\xab\xf8\x9c\x84\x89\x12\xa6\t%\xdfp\xa1v6\x88xb*j\x88\x82F\x81\x06a41\x15\x10\xda4\x9c\x15\x88w\"7=H\xa1\x97&\x1b%\xeb\x0eښ\x16\x8fmC\x8c$\x94\x18\xaa\xb6`\xc8_\xda5(\x01\x064)x\xab\r\xa8\xcb\b\xa8Q\xb2\x01eX\xc0\xb2k=\xde\xe9=\x9d[\x98m\x16\x17\xae\x17)-\x13\x81[\x82\xc7'\x94\x1e}Dn\x88\xa9\x98\xee\x96\x1a\x96G\xa8 r\xfd7(\xcc\xe5\b\xf4\x03(\v\x86\xe8J\xb6\xbc\xb4\xbc\xf7\x04\xca\"\xab\x90[\xc1~\x8d\xb0\xb5]\xb8\x1d\x94S\x03\xda\x10&\f(A9y\xa2\xbc\x85\vBE9\x82\\\xd3=Q`\xc7$\xad\xe8\xc1\xc3\x0ez<\x8f\x9f\x90xb#\xafHeL\xa3\xaf\u07bd\xdb2\x13$\xaa\x90u\xdd\nf\xf6\xefP8غ5R\xe9w%<\x01\x7f\xa7\xd9vEUQ1\x03\x85i\x15\xbc\xa3\r[\xe1B\x04J\xd5e]\xfe]$\xea`X\xb3\xb7<\xaa\x8dbb\xdb{\x81\x02q\x04y\xac\xa88\xc6s\xa0\xdc\x12;*\xd8G\x16u\xf7\x1f\x1f\xbe\xf6\x99\x92iO\x94\x1eoN\xd1\xc7b\x93\x89\r(\xd7\x0fY\xd3\xc2\x04Q6\x92\t\x83\xff\x14\x9c\x810D\xb7\xeb\x9a\x19\xcb\x06\xbf\xb4\xa0-\xbf\xcb1\xd8k\xd4:d\r\xa4mJj\xa0\x1c\x7fp#\xc85\xad\x81_S\r\xafL+K\x15\xbd\xb2DȢV_\x97\x8e?v\xe8\xed\xbd\b\x1aq\x82\xb4^\x8b<4P\f$\xcdvc\x9b\xa0.6R\r\x94\x8c\xed2\xc4QZ\xf8msZĪ\xc5\xf1\x9b%.\xb3\xed\xdfco\xcbovf\xad`\xbf\xb4\x80\xcaԉ?\x1c\xea+\xd5S\xed\xc3f\xd9hL\xddID\xdb\x06\xdf\vޖPF\xbd~\xb0\xc0\x9ce|<\x80\x82F\x8f2a\x85\xc8Z\x1f\xbb\x16ѽE\x05N\x15\x10!M\x02\x1e\x13\x0e\x1ea\x021\x90\xa4\t~h\xa0N\xccxvɄ\x88\x96s\xba\xe6pE\x8cj\x0f\xd1\xe8\xfaR\xa5\xe8~\x02[\xc1\x03x\x16\xb2\"\x10\xafj8+\x90\xe4Q\xa1 \xbe\xfe\xb8\xa8b\xda*ʰ\xca;\xc9Y\xb1_\xc0\xd7\xc7d\xa7 \xad^v\xfd\n\xc9\x1a*\xfaĤJ\x89\x81T\xf8iϞwjZZ-遌m\\悓Ȫ\xa4|\\b\x88\xcf\xf6\x9b\xce:\x90\x02\x1dʸ\x14Omo\xbb\xd7@\xe0;\x14\xadIL\x93\x90\xb2E\xd3$\x15i\xa46\xd3t\x9fV]\xa4\xef\x1c\xa5^\xce0\xcd\xc1ʒ\xac\xee\x9aW\u0081\xa8\x16\a\x03\x85,\x05\xd8eԖ\xa8ݷJ\xb6\xee\xdbI\xa4\x905\xd5P\x12)&GFvi9h?V\x89\x9c\xd1顋n\xfd\xe8\xf1\x10N\xd7\xc0\x89\x06\x0e\x85\x91\xea\x10\x999(u-G\xb1N\xa02\xa1M\x87\x12\xd0-`\x06$\xb1\x9c\xbe\xabXQ9\x0fò'\xc2!\xa5\x04m\xb5\t\xba\xcc\xfb\xa9E\x92%\xf2\xfbA\xe6\xb4G\xd7\x16\xc4j\f/\xa5Q\xba\x96\xa1\x86\xbb\x96Dm\xa7{\x0ft\x8b\x7fn\xe4\xec\xb2\xff\x7f\"6\x18\x93\x13\x98vF\xfe\t\xba\x9f\xd9<=ɷ\x18ၾ$7\x1b\x02uc\xf6\x17\x84\x99\xf0tI\x12(\xe7\xbd1\xfe\xc0\xb49\x9e\xe93I\x93#\x13g\"L\x1c\xe2\x0fH\x174\x19\x0f\xdebd\xd3\xe4\xaf\xfd^\x17\x84m\"\xd2\xcb\v\xb2a܀\x1aa\xff$U\x1f(\xf3\x12\xc8ȱz\x04\xf3\x04\xa6\xa8>~\xb7.\x8e\xee\x92`\x99x\x19wv\xbeq\x88 \x86\xe6y\x01.\xc1x\x99)\xa81\x0e'_\x11\x9b\xdd\x13t\xaa?\xdc\xfex\x18+\x8f[\x06\xe7\x1d,dA\xe8\\\xfb0ZQ\x7f~>*\bo\xd0\a\x8aA\x95˹\\\x10J\x1ea\xef\\\x17*\x88\xa5\x0f\r\x1fg\f\xaf\x00\x93?\xc8g\x8f\xb0G0\xe9l\xcea\xcb\xe5\x06\xd7\x1e!\xe1\xfa\xa7\xda\x00\x87vN>,vx\xb2\x0f\x10\x11\x18\xc3粁k^\x14\x12\xb9\x93t\xcb\xd4%\xa1\x05ܟ\xb0\xcc,V\xe9\x8f\xd1O}\"\a\xfc\xa0\x1d-\xad\xc4T\xcc\xe745\xa0\xcc\xe4\x12Եo\x94\xb32\x0e\xe4d\xe4F\\\x90[i\xec\x1f\f\xd042ʏ\x12\xf4\xad4\xf8\xe4,\x18u\x13?'>\xdd\b(h\xc2iy\x8b\xb0~\xce\xcf\xd94\xcbm\x11\xf7L\x93\x1ba\xe3\x15\x87\x92̡0\xbd\xeb\x86s\x03խ\xc6t\x9d\x90b\x85639\x92ǷT\x03t?{P?\xe0Wk,\xdc\x1b\x97d洀2D\x96\x98\xfd\xa4\x06\xb6\xac\xc8\x1c\xaf\x06\xb5\x05\xd2X\x15\x9e\xc7\x11\x99\x8aկ\xe68\xf6ɳ\xde\xfd\xf6}\xf5\x18\xf3\x05+krV\x1e\x82\x91u\x06\x0e\xbc\xee.\x97׳\xb22\x9b\xf1U\xe0\x84\xc5O'\x92\xa3ӟ\xe6 \xe5\x19\xe8@+\x8e.\xce\"uiY\xe2\x16\x1a\xe5wGX\x94#x\xe1X\xd5Л\xbb3\xc15m\xacZ\xf8okiQ\x9a\xfe\x974\x94)}I>\xe0N\x19\x87\xc1;\x9f\x87\xeb\x81\xc9\x18\xb2\xb1CY\xfey\xa2\xdc\xda~\xab\xc0\x05\x01\xee<\x01\xb99\xf0\x8b.Ȯ\x92ڙ\xed\r\x03\x8e\xfb\x15o\x1fa\xff\xf6\xc2\x0e\xbf8d_ɼ\xbd\x11o\x9d\x0fq\xa00\xa2\xc3!\x05ߓ\xb7\xf8\xee\xeds\\\xa9LN\xcd\xfcl\xc0\xa25m\xf28T$\x93\xf5]\x1bpL?7\xdf%当=\xb7\xda,\x16m\xa46\x9f\xd3yÉ\xf9܅\x1eC\xcf8\x91c[\x8c\x18|\x1e-\xea{\xebDn\f(\x9fKt6 \xc4\x1fό\xccR\xbb2\xfd\xc9\xc6d \x8d\xf9]\x8b\xe0\x05nr\x1b79S<\xc6a\xb5x9\xd2\xdb\xff\xf8\xbd\x97ϴ\x92k\xff\xef/\xe4\xa5\x1d\xeaB\xd65\x1d\xefjfM\xf5\xda\xf5\f<\xed\x019\xea\xabm\x8b\xf2\x9ck\x91;\x1e\xc2\xfd\xcb\x1d3\x15\x13\x84\x06\xb5\x01\xca3\x14%\x8dL\xe5\xb0S\xad\xa2\x9a\xac\x01DL\xd1\xff\x1e\\\x89\x9a\x89\x1b\x1c\x80\xbc?\x83\xeb\x11\xd1uNg\xf7:\xd2$R>>p&\xab\x91%\xd9U\xa0`\xc0\x18\x87yw\xf4T\x854\xbd\x94\xc5\x11\x0ei#\xcb\x1f4\xd90\xa5M\x7f\n\x9a\xb4:\x97\xd6G\x92\xcf\xce\xfb+\xabA\xb6\xe6\x9c\b\xfe\xd8\r3\xd8k\xae\xe9wV\xb75\xa1\xb5l\x9d17\xac\x8e\xbb\xba\x1e\xbd;\xcaLܶ\xc2\xfc\x8d\x91\x96\x04\r\a\x03d\r\x9b\xf4~o\xaa\x15RhV\x82\nU\n\x8elLZ\xc1\xdcP\xc6\xdb\xd4.Q\xaa\x1d\x1b\x01\x8b\x8fJ\x9d\x14\x00\x7fq={y\xc7J\xee\x86\b\xca\\;n\xa4\x01a\x1b\xc2\f\x01QX\x8c\x83r*\x19\x87\xf0\xc8@\u0530\\=\x97\xa7\xc0m\x03\xd1\xd6y\bX\xa1@21\x9br\xeb\x7f\xfe\x892~\x0e\xb2Y\xce\xfb$\xd5=\xd0\xf2\x94\x1c\xcdϽ\xee\x04\x84n\x15n\xfe;ݱc,q\x85o\x13\xc5\r\xc9\xd5\x1d\xef\tfQ6\xb4AЉyX\xf5\x04\xabV<\n\xb9\x13+\f\xc6\xf5\xd1:\xe4\xc4,\xd5s\x877'+\xa3e\xfd\x92\xaf\xa6\x97\xb4А_\xf3y*\xf8Og\xd02\xd9|sT\xc2c\x8e\v\x96\xf4\x9a+\xc0\x9ex\xb98\x8b\xb9\xf1g:\xfbM\xe9kW,\xfd\xac\xb2\xb8\x9b4\xa8\x9eS\xb8\xab\xc0T\xa0Bi\xf6\nK\xd2\xcb\xd9\x1d\xd2.x\x89ur\x96\xa9\x82\x8b\xec\xca?G\x95s\x18ݴ\x9c_Xަ-O\x86\xc3F\xa2\x88\x1drVV\xfdX\xdacȩ\xbe\xc8\xc6c\xbf\xd2bX_\x18\xab B\x81\xa1\f#{\x1a\xa7\u058b\x85\xa5\xbd\xfd\xfda9\x05\xe6\xff\xc2\xf4\x7f\xf3\xd2ÌJ\x89|4\xe6ViF$&`%\x18\xac\x87Ʈ\xbe\xc2\x7f\xe7\v}\x7f_85P\x7fi\xbc\xc4L\xba\xb0\x19hM\xc0\x19՛\xa05h\xb5s\x05\xa2\x1d\xf09C\xdb\xffC\xe1NA\x040)~\xfdZA\x10__\xbd\xcf4\xf9gR\xc96Q\xd57\x83\xb2\x85\xea\x8e\xe5\x05\x0f\n=\xfc\x86\x02\x18\xfa\xf4\xfer\xf8\xc6H_\xf6\x81Y\xb4\x04 \f\x8a\xba\xcc,\x13%{beKy\x90\xda\xee\f\x81c\xa0\x8e\xcf\x12Ф\"\x82qǀ\xa1\xff\x80\xe1ȗ\xc6m\xcb\x1c\xad\xe2\xe6}ѼꐓkB\x865\x1f\x13\xd6\xf0\xd8\xed\x8b\x17\xa9\x82\xfdMj=\x8e\xaf\xf0ȉ$\x16\xaa9N\xa8\xe1\xc8,\x16{\xf6~KN\x95\xc611\xf7\xd9*2^\xbe\x0e#\v?\xcb5\x17\xc7`\xe7\xec\xf5\x15\xafXU\xf1:\xb5\x14\x99\x15\x14/W\n\x99\x17}\x9eT\n\xb0\x1c\xb0LWA,\xd6><+\xa09iI\x8b5\r\xc7T2,R'O\xcc^\xadV\xe1\xd5*\x14^\xb7.a\x96\x8bf_\x1eSy\x10㤟h\xd30\xb1=d\x8a\\֙e\x9be\x96\xb9\x1dMd\xc03\xfdp\xa6\x8b\x0e'B_w\\:\x11I\x86\xb4%\x13F^\x92\x0fb\xef\xe1&\xe0\xf4\xc2G!\xcd\xc1A6;\xad\x1d\xe3\xbc\x7fZ\v\xc1\u0383\xf2g&5\xadݬ\xa6\xbc\xfd$]\xa5\x1a8\xe5'\x05\x8e_F0\xfa\xd9\xd1\xd7\xf4\xfc\xeb\x96\x1b\xd6p\xb0\x1e\xdd\x13+\x93g\xc8L\x05\xfb\x88\xe4\xbfI\xfb\xa5\x05\xb5'\xf2\tK\x18\xbc\xf7֝U\xf0\xeaF\xdb\x183(@\xaf\x8c\xa76\x15\x0eB\x99NA\x91\x0f\xc2\xf9\x12\xe3\xf9`\x1f\xab\xf9\xbaPͪs\x1b\x85%ǘ\xe8.d\xec\x9d\xe8\xb6\xe4\xf6\xe7\x16\xf5\x9f7p;>t[\xf4\x95\xf2\xfd\xd9ߨX\xff\x94\"\xfd\xbc\xed\xa0Ţ\xfcs\x05rK\xa1\\\xb6\xf7\x9aWt\x7f\xdc&\xea\x19\x8b\xec\xcfQ\\\x9f\x89\xa9\x9cb\xfa\xe3\xf0\xf4\n\xc5\xf3\xafZ4\xffZ\xc5\xf2\xd9E\xf2Y\xfb\x98ٛV\xb9ی'V}/\xef\xba\xcf\x17\xbdg\x14\xbbg\xec\xa4-/\xf2\x84\xe5e\x14\xb3\x1fWĞA\xb3\\Q|\xc5b\xf5W,R\x7f\xed\xe2\xf4\x05\xceZx}\\\x11\xfa\xc9;0a\xab\xffV\x96p'\x95Y\nN\xee\xc6\xdf'vR{\x01\x9b\xe4%\x11\xe1\xd3\xc4*1\xc4\xf0\xe1\xc5i\x8bJoz\x06w\xfa'Yڹ-\xed\xb1\u070f>?8\xab\xbc\x01\x05\xc2]\xf3\xf1\x9f\x0f_n#\xfc\x94\xcf\xeb=\xe3\xd1\xf5\x12\u0383)=r\xfc֜/fr\xd8B\x1f\xe0\x85\xf7Eh\xc3\xfe\x03ou{F:\xe8\xc3\xdd\r\xc2\b~\x1a^\x13\x17\xab(\xe2\x8e\xe5\x1a\xacŊ\xa8\x9a\x14\x8b\x9b\xcd\x00\xe2\xb0\xe2\xb7\x7f\x8d\x12\x94\xeeʬ`1Y\xa8\xf1\xb2\x82ww\xe3\xe615\xca'\xeb4\x8a=\x91\x8e#+\xa6\xcaUC\x95\xd9#\xdb\xe8\x8b\xc1\x1c\x82\x99\x99K\xe7L*\xd6\xc3k\xc0\x92\xe8\r\xb7\x7f\xe1^\xe4\xbe\x19\xee\xf6\x8eqw\xca<\xa6ϟ,\x9eg\x91\x0e\xc0`\x9d\xac\xa8z\x1e\xe4\x0e\x82\x8f\x19\x96\x8dҊݒ\x1a\x1c\xb8?i\xc5\xf8E/{\xfb:e:\x99Wl\x9d|\xb9\x96Cτ\xfa\xc1\x1d\t\xab\xda\x0e1uB\x81\xceb\xb8\x9dq\xf0c>\xb1\x90y5S\x9e\xc18\xe1:&\xc4W.\xaeH\xf2\x96\xa6̛\x98~SD\xcfh5]TP\xb6\x1cN\xbd\x87\xf5\xa1\xd7\x7f\xf9&\xd60Z\xc6]\xac\x16\xd9=\x03m=\xacᝯ\x9e\x12\x1er\x9f\x92SA8&lܕ\x8f\x85\xbb\x1d\xb8(@\xebM\xcbC\xe5h\xa1\x80\x1a(\xc3\xe7L\xc7\x19\x1fU\xfb\xd86\\\xd2\x12\x94s\xc9\x16\xd0\xfa_\x83\x8fG<[\xe0\xc3Vu\xd7\xed\xce^U\xfa,\xcd\xd5PE9\a\xfe\x89q\xd0?ʝ\xb0\xf3\xca\x10ȻT\xbf\xdeY٢U֬\xef\x89h\xeb5(\xa2\xc1\x98\xe9\x04\xdeF\xaa\xf9S+\x0e\xefL\x18\xd8B*\xe7\xb9S\xcc\xc0CC\x95\x06\x9cQ\xc6\n~\x1euq\x19\xc1\r\xa7[W\x9e\\\xb2\x82\x1a\x88\x06\x18G\x98\x9a>\xf6\xd7\b\x8b\xef\xb1ZTNlDd\v\xf5\xd41\xb9I\xb1\x9e\xba\xf29a\xaa\x93\x97>;\x8b\\\xd0\xc6\xe0\xa1D\xa4#\x12\xd1x\x18x\x91\xfa\xe8\xde\xe7\x01\xd8iN\xf3GK|\x11\xb36\xb4ND\t\xcbz\xe7\xfa\x10\f^ծ\xca^-t\xff\xd2\xdbX\xf4LvT\xc7\x03.I\u07fb\x83\xed\xc0\xa0\xabnACI\xe0\t\x04\xb1\xa2H\x19\x87r\x8eS\xbf\xe2\xe6\x9ez\x02\xf5\x83\x8ep\xb0:۲\xf8\x83\xa1\xcaĩ\x1f\xfa1.\x86\xbb\"%5\xb0\xb2\xbdOs\xdd\xd2WW+ub\x89\x06\x9e6\xf6\xe2Q\x84\xa3\x90\xd6\xfa\xb93\xc25hM\xb7!1\xb8\x03\x05d\v\xc2\xe2=\xee\xf7$=\xa6p\xcc\xda\x1b\x8bAb\x80\x16\xa6\xa5~\x00\xe7\xc2Ŋ\x96pg\x0f \xc5\xf0V\x1aʃ\x91\xb1|\x19?\xa8f.uy\b\x17\xdas\xbe\xbf\x18C\x1e\xfdRF\a\xbb\xea\xaeW\xf6\x9a\xa0\xbb\xd2cb\xa0\xb0\x13\x93\x04\x12of\xee|\x92\xa9{p\x97\xec\x1fB\xfd\x84\x93\xca\xc0\xf1\xe7\xee\xeb)<\xbai:\x87\x19D:\xd2$\x18|\x98*J\xc6\tS\x9f\xf1R\x9b\x8a\xea%\xf7\xf4\xce~\x13ݎ\x9e\xb9\x8aN\xe8\xfd\x84T\xa6\xef\x1eX\x91[\xd8%\x9e:daE\x02JU\xe2\x93\x1bq\xa7\xe4V\x81>d\xba\x15\x9e1gb\xfbI\xaa;\xden\x99\xf82}\x1ag\xee\xe3;\xaa\f\xb3L\xeb\xe6\x93\xe8{\x1dl\\\xe2\xddr\xef\xe9\x17LP\xce~M\xe9\xf2\xfe˥\x11f\xf4]\xe3\x91w\x8a\x85\n\x88_R\x80^C\xff\xa0{\xe6'\x8c{IneR\x8c}\xd1\x0e\x1b\x02e\x9a\xacA\x9b\x15l6R\x19\xb7\xa7\xbaZ\x11\xb6\t\x0e\x92\xd5\x10\x18'\xba_\x18!,\xb5\x19\x1a\xcb!\x82ò\xf1\xa9D\x85V\aCΚ\xee]F\x92\x16\x85\x8d\t\xe0\x9d64\x15\x9b\x10\x9cC\x1d^1\xe2\f:\x9f\xaa3\x18\xdc`D\xb4\xc5\xde)ʄ85v3\x1dv癚\xaf\x11ʔz\xf4\xeb\x1b\xfc8\x82/z\xf1\x1fY\xb2\x15\x15\x15\xdb\xc9Cƕ\x92\xed\xb6\n\xbc9\xe5\x10\x91\xb2\xc5ȹAU\xa0Ï9\x99V\x89^!\x85\xaf{\x9b\xd2\xd2q\xba\xd3>\xca3\x14\xb5\xea\x0e\x1bv\xaaj\xc6\xe6gg\t' .\xda\xfe\x04D\xaa\xf7\xa2\x98=\x16y\xb8Gu\x94k\x99DB\xd4\xc6/\x86\x84\bq\n\t}_\xa2\x8bx~7\x18\x99\xf2QNDǼ\x13\x83K\x9c\a\xb5\xbc\xe8\xbe\x134tw\x8eC\x87\x1e\x04\x7f'\xa5\xdd\x06\x10\x8e\x89|q\xect\xdc\xfb\xfb\x8dX\x9f\xa2\xb7\xf5\xf1\xe4\xd8\xf5\xdb\b\xc6\xe8X\xba\x8db\xbbaB\xbc\xf9\xf7l\x93\x92\x17\xf7\x8byk\x0e\xffp\xf0\xf6\x95\x8f\x97\xef\xa8\x12LlO\xc2\xc8Ͼo\"\x9e\xf7`\xcf\x19ч\x99\xbfXL\x9f4K\a\x0f\x91\xc1\xcb\x1e\x9e\xfdH\xfe\xc9\xff\x05\x00\x00\xff\xff\xbc\x9a$\xa6\xd7r\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=\xdbr\x1c+\x92\xef\xfa\nB\xfb0\x97P\xb7DZ\x97\xd8ЛG\xb6g;\xc6c+,\x1d=\x0f]\x95\xad\xe6\xa8\n\xea\x00%\xb9wv\xfe}#\x13\xa8[\x17]T\xab%ϙ1/\xb6\xaa!I2\x93\xbc@\x02\x8b\xc5\xe2\x8cW\xe2\x0e\xb4\x11J^2^\t\xf8fA\xe2_f\xf9\xf0\xdff)ԛǷg\x0fB\xe6\x97\xec\xaa6V\x95_\xc1\xa8Zg\xf0\x1e6B\n+\x94<+\xc1\xf2\x9c[~y\xc6\x18\x97RY\x8e\x9f\r\xfe\xc9X\xa6\xa4ժ(@/\xeeA.\x1f\xea5\xackQ\xe4\xa0\tx\xe8\xfa\xf1\x0f˷\xff\xb5\xfc\xcf3\xc6$/ᒙl\vy]\x80Y>B\x01Z-\x85:3\x15d\b\xf4^\xab\xba\xbad\xed\x0f\xae\x91\xef\xd0!{\xe3\xdbӧB\x18\xfb\xe7\xde\xe7O\xc2X\xfa\xa9*j͋N\x7f\xf4\xd5\by_\x17\\\xb7\xdf\xcf\x183\x99\xaa\xe0\x92}Ʈ*\x9eA~Ƙǟ\xba^0\x9e\xe7D\x11^\\k!-\xe8+U\xd4e\xa0Ă\xe5`2-*K#\xbe\xb1\xdcֆ\xa9\r\xb3[\xe8\xf6\x83\xe5g\xa3\xe45\xb7\xdbK\xb64ToYm\xb9\t\xbf:\x129\x00\xfe\x93\xdd!n\xc6j!\xef\xc7z{Ǯ\xb4\x92\f\xbeU\x1a\f\xa2\xccrb\xa0\xbcgO[\x90\xcc*\xa6kI\xa8\xfc\x91g\x0fu5\x82H\x05\xd9r\x80\xa7Ǥ\xffq\n\x97\xdb-\xb0\x82\x1bˬ(\x81q\xdf!{\xe2\x86p\xd8(\xcd\xecV\x98i\x9a \x90\x1e\xb6\x0e\x9dO\xc3\xcf\x0e\xa1\x9c[\xf0\xe8t@\x05\xe1]f\x1aHnoE\t\xc6\xf2\xb2\x0f\xf3\xdd=$\x00#\x12U\xbc6$\x1cm\xeb\xeb\xee'\a`\xadT\x01\\\x9e\xb5\x95\x1e\xdf:\xd9˶P\xf2K_YU \xdf]\xaf\xee\xfe\xfd\xa6\xf7\x99\xf5)\xfa\x7f\x8b\xe6;k\xb8\xc1\x84a\x9c\xdd\xd1,a\xdaO[f\xb7\xdc2\r(\x06 -֨4,\x02\xa9s\xa6t\aT\x05Z\xa8\\d\x81E\xd4\xd8lU]\xe4l\rȭeS\xbbҪ\x02mE\x98\x87\xaet\xd4K\xe7\xeb!\xf4\xb1\xe0\x88]+'\xa6`H2\xfdl\x83\xdc\x13\xc9M\x1ea\xda\xf1\x10\a\xf13\x97L\xad\x7f\x86\xcc.\a\xa0o@#\x980\x8aL\xc9G\xd0H\x91L\xddK\xf1\xbf\rl\x83S\u0092\xa4Z0\x96\xd1|\x96\xbc`\x8f\xbc\xa8\xe1\x82q\x99\x0f \x97|\xc74`\x9f\xac\x96\x1dx\xd4\xc0\f\xf1\xf8\x8b\xd2\xc0\x84ܨK\xb6\xb5\xb62\x97o\xde\xdc\v\x1b\x94n\xa6ʲ\x96\xc2\xeeސ\xfe\x14\xeb\xda*m\xde\xe4\xf0\b\xc5\x1b#\xee\x17\\g[a!\xb3\xb5\x867\xbc\x12\v\x1a\x88$Ż,\xf3\x7f\v\xfc6\xbdn\xf7f\xa6+\xa42g\xb0\au\xa9\x93.\a\xca\r\xb1\xe5\x02~B\xd2}\xfdpsە\xa0\x10PP\x10#\x15+\x94\xbc\a\xed\xb0h\f<\x1a\x18@\x01\xcd\x19\xfa\xea\x1aͲ\x90lS\xa3\x1b\xbfd\xa8%\xa22\"\xa4\xb1\xc0#\xc2|\x02\xde\xc17\xb4\b\x90_\x15\xb5\xb1\xa0o2UA\x1eV\xe6F\xcdY*\x0f?\x1c\x84샾Bd\x80|\xc8\\\xa5\x05\xad\x8c\xc5D\xbb\x8d\xff\xd0<\xd2B\x1d\xb2\xda\x0f\xa1\r\xec&u\x8b\x01\x8b\r\xcf\x7f\x7f~A\x12\xd0\xef\xbdߏa\\CC\xa6Y\xba\x99ܤ\xf1\x16\xc2B\x19\xa1\ue90e\x9a\xc1w\xae5\xdf\x1d\xe0z\xb3\x02\xf9\x02|\x8f\xc1\x1ep^\x86j߉\xf7\xc3\xfe\xff\x15\xb9\x7fZ~\x1bZ\xa9\xe7B\"\x9f\val\x8f\xcd\xc6-\xfd!Y\xc7\xe2nO \xe9`\xa2\x9a\x9c\xe2\xea?\b1O:wb\x93\xa5\x91M?\x01\xfe\xa9(\xb9U\xea!\x85z\xff\x83\xf5\xdau?\x96\xd1n\x12[Ö?\n\xa5\xcdpm\x19\xbeAVۨf\xe1\x96\xe5b\xb3\x01\x8d\xb0ho\xa4\xd9J9D\xac\xc31\x1f먬h\x85\xc1\xb8Z\xa6#K\x89\x1a\xb1\xa1PT\x1f\x85\xea\x1c\x1c\f-ȁ\xc8ţ\xc8k^\x90/\xc1e\xe6\xc6\xc7\x1b\xfcbZmB \xf6\xf0\x8fJ\xb5+Ρ\t\x83D&\xf6\x96\n\x95\x04\xf4\xf1K\x8c\x8d\xf6\xab\xc6)\x11\xd6_\x0e\xf6\x8d\xcc\xd4u\x01\xc6w\x97\x93\x9b\xdcꤋ\x96Yna\xa6\xe0k(\x98\x81\x022\xabt\x9cB)r\xe0J\xaaҍ\x10wD\xcb\xf6\xa3\xadv0\x13`\x19\x85\xb8[\x91m\x9d\xfb\x8a\x82F\xb0X\xae\xc0\xd0R\x12\xaf\xaa\"b\xba\xda2)\x1c\xbe\xb3)\xbdі\x04\r2\x84\x1b\xd3%mI\xd4\xcfm\x19%{;7\xfbT\x1f\xdf\x1c\x19\xc5\xf7_\x89\xe8\xc1\xea\x1c)\xec\x13\x9a\x84\xd1&K\xf2|\x88\x92\x1e).\xc0,;K\x9a\u0086\xaf)\f\xed\xf9\x8f{\xfbO{D\xf9u\xf1\xee\xb8\t3\x83u\x93s\xeae\x19\xd7t\xf3O\xc272Y7\xdeb\xcd\xe2٧n\xcb\v\xdaJ\xf1\f\xc9/\xd8F\x14\x16ȩ\x9aB\x94\xcd\xe0\xdc)\t\x94j\x81\x19\xed\xac\xdbl\xfb\xa1\xd9pKh1\xa0\xd5\x10\x80s\xd0C\x94C\xf8DDE\x1b?\x1az\xcc\xdd\xdf\x13!\xefZ*\xdbYƙ\xe9DW*\xff\x8da\x1b\xa1\x8d\xed\xa2a\x0e$V\x8d\x82:\"\xf4\x94\x1f\xb4>:\xf2\xfc\xe2Z\x0fR']\xb6\xf9\x9cx;\x90t\xcb\x1f\xc1\xa7\xfb\x82\xccT-i)\f\xf5\x00v3\x03\xa2c\x8d\xb3\x02\x89\xf6\xae\xd38\x9a\x819V\x16$IBN\xae\x9bu\x9b|\xe4\"m݊\x1d\xc7V{(\x87s\xac\x1c?\x8fB\x82g\xf7\fBɿ\x89\xb2.\x19/\x91\x87\xe4v\x88\x12\x9ac\b\x8e\xddM\xda'\xb6 \xa3e\x15β\xaa\x00\v>ms\x06\x1e\x99\x92F\xe4И~/\x02J2\xce6\\\x14\xb5\x9e\xa1Ug\x93|n\x10\xe6\xb5\xc9\xe9#\xabtD\x16D\xa2\xc4u\xf6\x19^\xf0\xb4Ư\xf4c\n\x86\vZ\xe3\x97LJ\xf7\\\xf3gŝ_\x06\xb0PX\x82\x9b\xfa\x8aq@Y\x17VTE{\x89],\xe0\xdc®\xb9\xac\xe8gEG\xe4\xfdM]_\xbe6\x12\xbf\x1cD5ܰ'(\n\xc6css\x8f\n\x99\xbb<5S\v@ۈ\xb3\xdc_\xc6\xe4o\\\xbdpӅn\x03 \v[Ɩ\xfa\xb8<|\xd3\xd7A\x03\x96\xaa\xc7\xf6\x85\x98x\xfa01\xd3 m\xf0G\x0e;\xf1t\xe1\xfcS\x85\x89\xfc\x9d3\xa5_\xf9\xf4\xe0+\x9f\x1a\xfc\x1e\xa7\x05\x13$0\xa1\xca\xfcS\x81\xcfޒR:\a=\xb9\xed7Gj'\xe555\x96\xeb#6\xd8\xd7\n\xb7\xc9b\xad^\f@fɿ~@/]\x1c\xda\x06G\xc9\xecxD\xbd}\xc9\xd6]\xeb;\xc4\xfe\t\f\xb7ui\xa0\xe2h\x00(p\xa3Ԭ\xa8\xab\xf0\x81g\xdbA\x0f[n\xd8F\xe9\x92[v\xdel\x16\xbfq\x1d\xe0\xdf\xe7K\xc6>\xaa&W\xa7{_\x9a\x11eU\xec0\x12c\xe7\xdd\x06ϓ\x92\xa8t\x86\x9e\xafU!\xb2\x88\xcf9z\xaf\x9ek\xb0w\xd9\x10\xdd\xfc\x97u\xb2Eb\x81\x0f6\x17\xe1\xd6\xc5\xfe\x95\xcc\xee\x12\xfc#\xd7Jx%\xfeD\xefP\x9d`\xd5\xed\xdd\xf5\x8a`\x051\xa2\a\xae\x9a\x04ņ\xe5k@\x97\xa1\x1d\xfb!}\xb2\xda\xf4\xa0\xf6s\x84\xbb/|@\xee\x9es\tn\x8bW͙B\xadu\xbdr\xb8\x1c\xea\t\xe5\x8b\xcb\x1dS\xfe\xbd\x0e\xa1\xf3Eŵݹd\xa2\x8b\x1e\x1e\xc1\xaeO\xad\x9a\x1d\xb4V\xfb\xcf\xd5tK\x8f\xec\xe1\xa5\x1a\xda\xc9\xdeU\xfd\xe4\x81!=\x9f\x83\xd3\xe1SՓ\xe7\xa9_\x00\xa7\xc3.Ԃ\xa8\x18\xf9)\x9a\x01y\xf2\x15K\xe3o\xe8\xff\x8bz\x84\xf7ѕ\xcb\xfe\x935\x83&#\xa9\x89\x01*]2\x1f\xa1`\x9b\x8fHw|?O\xed\xc5s\r\x03*\xfe\x8e\xf0\xe7,N\xde\xf4A\x8d\xbf\xe2B7\xa8\x87Nc^\x15\xbd\x8f\xb5c\xd7w\x14\xb76\xaa\xd4O}\x1f\xb7\x86\xe5ɐ`\x10\x81%\xe4\xc1\x87mNEF\xab4\xbf\x87O\xca=H\x94\"&\xfd\x16\xbd窼\xe7\x16\xf2\xb5\xfd$\x8c)z?\xb6!\xc0\xf6|\xc6\xdeE\xff\x88\xed\x91O\x19X[\xb55\xc7\x06\xdc\f\x03\x87쟐{ՑХ\xfb\x13c\xb8\xc6:\xcd)W/G\xd40\\\xd6\x7f\x13c\xc2\xf8Q\xc8\x05\xfb\f\xfb\x11\xfb\x82}\x908\x88}\x02\xb8\xf3\x8e\x90\xd3\xd6\ni\xc79C|lZ\xd1a\xd3\x11\r9-\xb6w\x03\x18\x83Lvz\xf4\xa9\xa9\xe2N\x9b\x1a\xf6[1\xe6\x8dҎY\x86\x03\xfd\xddޯQ\r~P{\xc74\xf7\xa8\x1a\xd9\xfbH\xaf\a\xe6\x1d\xc9\xf1^z\xf7K\xbdn\x1fT`\x7f\xfb\xfb\xd9\xff\a\x00\x00\xff\xff\f/o%s|\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcV\xcfo+5\x10\xbe\xe7\xaf\x18\x89+\xbb\xa1B \x94\x1b*\x1c*\xe0\xa9j\x9ezw\xbc\x93d\xa8\xd7^f\xc6)A\xfc\xf1\xc8\xf6n\x9b\xee:\xb4\x8f\x03\xbe\xad\xed\xf9\xe6\x9bo~x\x9b\xa6Y\x99\x81\x1e\x91\x85\x82߀\x19\b\xffT\xf4\xe9Kڧ\x1f\xa4\xa5\xb0>ݬ\x9e\xc8w\x1b\xb8\x8d\xa2\xa1\x7f@\t\x91-\xfe\x84{\xf2\xa4\x14\xfc\xaaG5\x9dQ\xb3Y\x01\x18\uf0da\xb4-\xe9\x13\xc0\x06\xaf\x1c\x9cCn\x0e\xe8ۧ\xb8\xc3]$\xd7!g\xf0\xc9\xf5\xe9\x9b\xf6\xe6\xfb\xf6\xbb\x15\x807=n@\x90ә\x1a\x8d\xc2\xf8GDQiO\xe8\x90CKa%\x03ڄ\x7f\xe0\x10\x87\r\xbc\x1e\x14\xfb\xd1w\xe1\xbd\xcdP\xdb\f\xf5P\xa0\xf2\xa9#\xd1_\xae\xdd\xf8\x95\xc6[\x83\x8bl\\\x9dP\xbe \xc7\xc0\xfa\xe9\xd5i\x03\"\\N\xc8\x1f\xa23\\5^\x01\x88\r\x03n \xdb\x0e\xc6b\xb7\x02\x18\x05\xc9Xͨ\xc5\xe9\xa6\xc0\xd9#\xf6\xa68\x01\b\x03\xfa\x1f\xef\xef\x1e\xbfݾ\xd9\x06\xe8P,ӠYֿ\x9b\x97}\xa8\x85\t$``\xa4\x04\x1a\xc0X\x8b\"`#3z\x85B\x19\xc8\xef\x03\xf79\xad`v!\xea\x05\xaa\x1e\x11\x1e\xb3\xfec\x98\xed\xcb\xe1\xc0a@V\x9a\xa4)\xeb\xa2\xe2.v\xff\x8dxZ)\xd6b\x05]*=\x94\xecy\xd4\v\xbbQ\x1e\b{\xd0#\t0\x0e\x8c\x82\xbe\x14c\xda6\x1e\xc2\xeew\xb4\xdaΠ\x8b.\x922\x19]\x97*\xf6\x84\xac\xc0h\xc3\xc1\xd3_/ؒ\x04JN\x9dѬ\x9dWdo\x1c\x9c\x8c\x8b\xf85\x18\xdf͐{s\x06\xc6\xe4\x13\xa2\xbf\xc0\xcb\x062\xe7\xf1[`\xccRo\xe0\xa8:\xc8f\xbd>\x90N}hC\xdfGOz^疢]\xd4\xc0\xb2\xee\xf0\x84n-th\f\xdb#)Z\x8d\x8ck3P\x93\x03\xf1\xb9\x17۾\xfb\x8a\xc7Ε7n\xf5\x9cjP\x94\xc9\x1f.\x0er\xeb|AzR#\x95b*P%\xc4\xd7,\xa4\xad$\xdd\xc3\xcf\xdb\xcf01)\x99*Iy\xbd\xba\xd0e\xcaOR\x93\xfc\x1e\xb9\xd8\xed9\xf4\x19\x13}7\x04\xf2\x9a?\xac\xa3\\\xb8qד\xcaT\xda)us\xd8\xdb<\xab`\x87\x10\x87\xce(v\xf3\vw\x1enM\x8f\xee\xd6\b\xfeϹJY\x91&%\xe1Cٺ\x9c\xc0\xf3\xcbEދ\x83iv^ImeJl\a\xb4)\xb9I\xdfdM{\xb2\xa5\xad\xf6\x81\xc1\xd4L\xda\x0f1\xc9\x16_\xc8e\x9cH\x85\xcdlN\xa5.\x7f\x9fM},哣\x11\x9co\xce8ݧ;s\xff\x8e\xf6h\xcf\xd6a\x81(S\bߧ\x92\x16\xfa\xd8/}6\xf0\t\x9f+\xbb\xf7\x1c҄\xc6\xf9\xa8\xb9Z\x1bP\x1e\xb1\x03\xf9E\xb8\xf3\xc8ʭ\xfc0.G~\x0eh\x04\x02\x8eާ\x96\x0e~\x01Yy\x11\x16wH\xb1\xaf\xb0\xa9\xf2\xb9\xf3\xfb\x90\xff\"Lrl\xb4\xb4\x13\x8e\xc9\x1e\xfd\x14^\x15\xc0\xeb\xb9.k9\xe7>$hY\xf9y\xfeo\xc6i.\x11c\xd5w\x93YU\x0f\x92ǚ\xe2\xf5\xfe\x1aYF\xe7\xcc\xce\xe1\x06\x94\xe3Һ\xd8\x1afs\x9eW\xcdTj\x9f\xa9GQ\xd3\x0f\xef\x14\xd0\xe2UH\xeb~\x81\x92\x9a\xe7\xf9\x88\xfeZ\x8b\xc0\xb3\x91W\xe7\x15\xc8\xdd\xf9\x9a\xe9\xed\xcb\xdf\xe6\xb2\xcfJ=o \xcd\xfaF\xa9\"䇔\xaa\xa6\xb4\xd4y\xf5\xb7f\xa1\xd2\xf6\xf2\xee4H\xde\xf4\xcb\xf4W\xb3\x8c\xe1*\x85j\x05,63|w\x11\x9eh`s\x98\x02\xfe'\x00\x00\xff\xff\xef\xf8\xa6>\x10\f\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcVM\x8f\xdb6\x10\xbd\xfbW\f\x92kd7(Z\x14\xbe\x05\xdb\x1e\x82&\xc5\"N\xf7N\x93#{j\x8ad\x87C9.\xfa\xe3\v\x92Ү-\xcb\xc9nQT\x17\xc3\xe4\xf0q>\u07bca\xd34\v\x15\xe8\x019\x92wkP\x81\xf0\x8b\xa0\xcb\xff\xe2\xf2\xf0S\\\x92_\xf5o\x17\arf\rw)\x8a\xef>a\xf4\x895\xfe\x8c-9\x12\xf2nѡ(\xa3D\xad\x17\x00\xca9/*/\xc7\xfc\x17@{'\xec\xadEnv薇\xb4\xc5m\"k\x90\v\xf8xu\xff\xdd\xf2\xed\x8f\xcb\x1f\x16\x00Nu\xb8\x86\xde\xdb\xd4at*Ľ\x17\xebu\xc5\\\xf6h\x91\xfd\x92\xfc\"\x06\xd4\xf9\x8a\x1d\xfb\x14\xd6\xf0\xb4Q!\x86\xeb\xab\xeb\x0f\x05m3\xa0}\x18Њ\x81\xa5(\xbf~\xc5\xe8\x03E)\x86\xc1&V\xf6\xa6g\xc5&\xee=\xcboO\xb77\xd0G[w\xc8\xed\x92U|\xeb\xfc\x02 j\x1fp\r\xe5xP\x1a\xcd\x02`\xc8O\x81k\xc6Լ\xad\x88z\x8f\x9d\xaa\xf7\x00\xf8\x80\xee\xdd\xfd\xfb\x87\xef7\x17\xcb\x00\x06\xa3f\nR\xb2<\x1f\"P\x04\x05\xa3'p\xdc##<\x94|B\x14\xcf\x18\a\xa7\x1fA\x01F\xff\xe3\xf2q1\xb0\x0f\xc8Bc\xf0\xf5;\xe3\xd7\xd9\xeaį\xbf\x9b\x8b=\x80\x1cJ=\x05&\x13\r#\xc8\x1e\xc7t\xa0\x19\xa2\a߂\xec)\x02c`\x8c\xe8*\xf5\xf2\xb2r\xe0\xb7\x7f\xa0\x96\xe5\x04z\x83\x9car\xad\x925\x99\x9f=\xb2\x00\xa3\xf6;G\x7f=bG\x10_.\xb5J0\n\x90\x13d\xa7,\xf4\xca&|\x03ʙ\tr\xa7N\xc0\x98\xef\x84\xe4\xce\xf0ʁ8\xf5\xe3\xa3g\x04r\xad_\xc3^$\xc4\xf5j\xb5#\x19\xbbN\xfb\xaeK\x8e\xe4\xb4*\rD\xdb$\x9e\xe3\xca`\x8fv\x15i\xd7(\xd6{\x12Ԓ\x18W*PS\x02q\xb5K:\xf3\x9a\x87>\x8d\x17\xd7\xca)S,\n\x93\u06ddm\x94.yAyr\xc3T\xd6T\xa8\x1a\xe2S\x15\xf2RNݧ_6\x9fa\xf4\xa4V\xaa\x16\xe5\xc9\xf4*/c}r6ɵ\xc8\xf5\\˾+\x98\xe8L\xf0\xe4\xa4\xfcі\xd0\tĴ\xedH2\r\xfeL\x18%\x97n\n{W\x94\t\xb6\b)\x18%h\xa6\x06\xef\x1dܩ\x0e흊\xf8?\xd7*W%6\xb9\bϪֹ\xdeN\x8dkz\xcf\x1bu\x90\xc9\x1b\xa5\x9dW\x84M@}\xd1x\x19\x85Z\x1a\x14\xa2\xf5i\x8b\x15\x10|;ý\x17\xb9\x9c?t\xa9\x9b#\xe2\xbb^\x91U[{-\t\r\xfc\xee\xd4\xcdݛş\xad\xe7\xd5b̏=\xb3\x06\xe1T\xb1\a\x96\r+\xff\x04\x00\x00\xff\xffNy\xc1Q\xa1\x0e\x00\x00"), } diff --git a/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml b/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml index 2f24f7e81..7a8b9441a 100644 --- a/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml +++ b/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml @@ -121,6 +121,9 @@ spec: description: TargetVolume is the information of the target PVC and PV. properties: + fsType: + description: FSType is the file system type of the target volume. + type: string namespace: description: Namespace is the target namespace type: string diff --git a/config/crd/v2alpha1/bases/velero.io_datauploads.yaml b/config/crd/v2alpha1/bases/velero.io_datauploads.yaml index c4c25cce6..556272aac 100644 --- a/config/crd/v2alpha1/bases/velero.io_datauploads.yaml +++ b/config/crd/v2alpha1/bases/velero.io_datauploads.yaml @@ -133,6 +133,9 @@ spec: description: SnapshotType is the type of the snapshot to be backed up. type: string + sourceFSType: + description: SourceFSType is the file system type of the source volume. + type: string sourceNamespace: description: |- SourceNamespace is the original namespace where the volume is backed up from. diff --git a/config/crd/v2alpha1/crds/crds.go b/config/crd/v2alpha1/crds/crds.go index 53e1958e8..4c62c3c08 100644 --- a/config/crd/v2alpha1/crds/crds.go +++ b/config/crd/v2alpha1/crds/crds.go @@ -29,8 +29,8 @@ import ( ) var rawCRDs = [][]byte{ - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcYK\x93\xe3\xb8\r\xbe\xf7\xaf@M\x0es\x19\xbb3yl\xa5|\x9bq'U]\xd9\xe9q\xad;}\xa7$X\xe6\x0eE2|\xd8\xebM\xf2\xdfS %\x99\x92\xe8\xe7>|3\t\x82\x1f\x01\x10\xf8@\xcdf\xb3\a\xa6\xf9\x1b\x1a˕\\\x00\xd3\x1c\x7fr(韝\x7f\xfb\x9b\x9ds\xf5\xb8\xfb\xf8\xf0\x8d\xcbj\x01Ko\x9dj~@\xab\xbc)\xf1\t7\\rǕ|hб\x8a9\xb6x\x00`R*\xc7h\xd8\xd2_\x80RIg\x94\x10hf5\xca\xf97_`Ṩ\xd0\x04\xe5\xddֻ?\xce?~7\xff\xeb\x03\x80d\r.\x80\xf4Uj/\x85b\x95\x9d\xefP\xa0Qs\xae\x1e\xacƒ\x14\xd7Fy\xbd\x80\xe3D\\\xd8n\x1a\x01?1ǞZ\x1daXp\xeb\xfe9\x99\xfa\x9e[\x17\xa6\xb5\xf0\x86\x89\xd1\xdea\xc6rY{\xc1\xccp\xee\x01\xc0\x96J\xe3\x02^hk\xcdJ\xa4\xb1\xf6L\x01\xca\fXU\x05+1\xb12\\:4K%|\xd3Yg\x06\x15\xda\xd2p\xed\x82\x15RX`\x1dsނ\xf5\xe5\x16\x98\x85\x17\xdc?>˕Q\xb5A\x1ba\x01\xfch\x95\\1\xb7]\xc0<\x8a\xcf\xf5\x96Ylg\xa3)\xd7a\xa2\x1dr\a\xc2k\x9d\xe1\xb2\xce!x\xe5\rB\xe5Mp!\x9d\xbbDp[n\x87\xd0\xf6\xcc\x12<\xe3\xb0:\t$̓:\xebX\xa3Lj\x92\xa5\x11R\xc5\x1c\xe6\x00-U\xa3\x05:\xac\xa088쎱Q\xa6an\x01\\\xba\xef\xfer\xda\x16\xad\xb1\xe6a铒C\xc3|\xa6QH\x86#\x12\xf2R\x8d&k\x1d\xe5\x98\xf8%@\x1c)\xf8\x9c\xac\x8fH\xa2\xdet\xfc\"\x14\n9P\x1bp[\x84Ϭ\xfc\xe65\xac\x9d2\xacF\xf8^\x95\xd1}\xfb-\x1a\f\x12E\x94\xa0\xe8\x05N\xbeS&\xeb:\x8d\xe5<ʶ\xca:]#\xff\r7\xfa\xd5c\xab4Ȳ\xb1ե\x9ay\x90\xe0J\xe6\x03\xecS\x8dW\x05WjD\xa9*L,6\xc0\xc4-h\xa3J\xb4\xf6L\xc0\x93\x82\x01\x8a\x97\xe3\xc0\xc44Qb\xf7'&\xf4\x96}\x8cI\xa6\xdcb\xc3\x16\xed\n\xa5Q~Z=\xbf\xfdy=\x18\x863\t\x83\x95\xceR\xa6 \xf8\xda(\xa7J%\xa0@\xb7G\x94\xd1\xf5\x8dڡ\xa1\x9a\x8d\x93\x06C\xf4\x10@\x93z\x1fhO\x8d\xc6\xf1.\v\xb7\xba\x8f\x05&\x19\x1d\x9d㿳\xc1\x1c\x00\x1d=\xae\x82\x8a*\r\xc6c\xb5\xb9\x15\xab\xd6Z\xd1y܂AmТ\x8c\xb5\x87\x86\x99\x04U\xfc\x88\xa5\x9b\x8fT\xafѐ\x1a\xb0[\xe5EE\x87ݡq`\xb0T\xb5\xe4?\xf7\xba-8\x156\x15̡u\xe12\x1a\xc9\x04\xec\x98\xf0\xf8\x81\x8c6\xd2ܰ\x03\x18\xa4=\xc1\xcbD_X`\xc78\xbe\x90\x15\xb9ܨ\x05l\x9d\xd3v\xf1\xf8Xsו\xddR5\x8d\x97\xdc\x1d\x1e\x837x\xe1\x9d2\xf6\xb1\xc2\x1d\x8aG\xcb\xeb\x193\xe5\x96;,\x9d7\xf8\xc84\x9f\x85\x83\xc8Pz\xe7M\xf5\a\xd3\x16j;\xd8v\x12\x88\xf1\x17\n\xe6\r\xee\xa1*J\xb7\x82\xb5\xaa\xe2\x11\x8f^\xa0!2\xdd\x0f\x7f_\xbfB\x87$z*:\xe5(:\xb1K\xe7\x1f\xb2&\x97\x1b4q\xddƨ&\xe8DYiť\v\x7fJ\xc1Q:\xb0\xbeh\xb8\xa30\xf8\xb7G\xeb\xc8uc\xb5\xcb@M\xa0@\xf0\x9a\xf2A5\x16x\x96\xb0d\r\x8a%\xb3\xf8;\xfb\x8a\xbcbg䄫\xbc\x95\x12\xae\xb1p4o2\xd11\xa6\x13\xaeM3\xc8ZcI^%\xc3\xd22\xbe\xe1m%\xa14\xc0\x06\xb2C\v\xe5\xaf>\xfd\xb2\xd5d,t)\xdc\xe8\xf79\xa7\xa8C+\x93D\xde\xd6:\xdb\x16)1,R\xe9oR\x1f\rje\xb9S\xe6p\xac\x92\xe3P8\xe9\x15\xfa\x95L\x96(\xee9\xde2\xac\x04.+\xb29\xf6\xa1LI(j\r@\x95\xac\x15]\xae\x81+\xe0ّ\fŶE\x97?\xa8\xccV5.\xe1\xc8)!\xe5\x8e\xe3\xe3\x16J\tdc+R\x14~\xa1\xb2\xb0Tr\xc3\xeb\xe9\xc1S\xfa{*D.\xd84\x13\xb0ɖt\n\x8aNB2\v\x15jօ.\xa5\xf6\r\xaf\xbd9\xe5\xff\rGQM\xf2\xcfɛ\xd4\x1d8\xecr\x8f\x8f{\xe8\xdd\xedj\xabZRz\x9d\n\x19\xca\x06\xbe\x9b\x84\xe6\x14$\xc0\xf3&\xd1\xc8-\xbc{\a\xca\xc0\xbb\xd8\x13\xbd\xfb\x10W{.܌\x0f\xea\xff\x9e\v\xd1\xedrSt\x13\xc3\xf9\xba\xbep\xf2\x97 Dx\xbe\xaeo\xe5VS4(}3\xddp\x06\xcc;\x95\x19\x16\\\xfa\x9f2\xe3{.+\xb5\xb7\xb7\x1c\xb6\xe77D1\x95w\xf78\xfc\xebH\xc7\xc8\xef\x8e\bq\xf0\xb5S\xb0g<\xe1\x18\xfd\xee\xf6CFo\x81\x1b*H\x06\x9d7\x92\xd2\x01\x1aC\x19\xda\x06\x95\xcaO8\xcfٓZɴ\xdd*\xf7\xfct\xe1\x8c\xeb^\xb0˻\xcfO\x9d\x8b\xdfB\xd4\xf5ɷ\x95\x84\x8c\x97\b~\xc7\"\xabP\xd6\xefB\xbb\xe6?\xe3\x95xI\xb4C,T\xcdK&\xc0\x861\xd96\x81\xed!:\xddS@\xb9>o\f7\xed\xd6\x12\xbc\x81\xfb\xf4/\x04\xf7\x84\xd1z\xa8\xa2;\x8a2\xbc\xe6\x14,\xb2\x9f9ޱ\x9d\x12\xbe\t\xa2\xe4\x12\xac\xc0\xeb\x13\xb6\x06*\x1fD\xb6\n\x84\x8ao6h\x88Q\x05\xba\x157^\xbd-\xdf\xdbd\x13\xbeI\xffP\xa5j\x98\xd6XQoG\xc1\xd8\xfa\xf6&\xaf:fjto\x01\xf4\x05\x13\xbd&\xa2\x9d)\x88\x9a\x91\x83Z\xee\x1f.W\x10\x83\xd5\xdb2\xc3\xd4\xe9\xb7z\x9b\"<\xcdc\xa0m\xdaN8q\x82r\xe2\xad\x16O\xaf#\xab\xe2l\x19\x04л+v^\xbd\xe5XQo\x0ep[\xe6H\xa2m\xb2\xa18duBw\xa5[wއ\xb7\xbc\n\xf0\xf2,\xe2\xe5\x18\xf2\t\xbc\xc5\xe1\x17C&\xd2\xc5\rV\xb9\x92s\xdas3л\xec`y=\xb5\xc8\xef<\xcb\xf3\xe7\x91̸T\x8d\xa6\x8f\xf9}<1\xcc+\xa3\xd9\xf4J^\xd5h\x84g\x90k[\x8d\xf8\xb8ٺ\xbd\xf4&$\x9d\xf6ɓ\xba\xf7\xbb\x9a\rV\x96\xa8\x1dV\x9f\x0f\xc4B\xae *\x04@\x9e\x7f\x04\xfa\x97>\xd2\x14\xd4\xec֎\xa0\x83\xd4?T\xddS\x00>\x8d\x95\x84\xd7\nS%4b\n7R\xc9Ӡ\x01^\xa9\xe4\x85n\xfb}d\x0e\xb4,\xf0\x11bԓMO\x16Ej\xa7g\xb4~\"!\xbd\x10\xac\x10\xb8\x00g\xfc\xa9\xd6\"\xdfI\xc5w\xdf\xf4\x89﮶j\xaafj;\xd6?j\x85\xc7\xc7\xee\xc59g\xb2\xa3\xbe\xde`Q\x1dV\x80;\x94@\xcd2\xe3\x02\xabNg\xa6\xbf\xb8d\xf9\f\xe8)u\xfd-\x8dߠ\xb5\xac\xbet\x81\xbeD\xa9\xf8\x0e\xd4.\x01V\x10\xcf\x1d\xb3\xfc\xf7\xb6\xbd\xdb7\xf7\x1b\xbf\xce%\xbe\xb2\xdb8\x83%\xf4\xc6\x17\xc0\xacH&\x97\xd3zh\xa7\x93\x1a\x9civ^p\x9f\x19\xed\xeegfj\xd5^\xfa\xcc\xd4\xe4\x13R:\x19\x1f!r\x85\xb1\x9b\xcb\xea\xec\xbf\xd1d\xe6\xfe\x11.\xc3M\x96n\xf1\xdds\xdd\xfb\xa7\x8c\xad\x12\xdd\r\x0f\xdfV\xa4o\n4\xe4\x86\"G\xf8\xc3\vx\xe2\xb5\x1c\xf9\xeb5\xf4\xbdKP5\x87\xd7-Q\x93\xf8\xfe\xd2us\x15\xb7Z\xb0C\x7f\x98\x94\xa1f\x94\x1fo\xcd\xe4y\xfdV\x92\xda\x7f\xeb\xca3\xaf\xf3\x8d\f\\hf\xc2|\xff\r\xeb\xb7\xd9\xe1\xcc\xeb\xcb\xf0\x9b\xe2]\xad\xd4@åR\xd0~\xe3\xbc=\x83\x0f\xb7\xf9=\x93w\xd6z\x93\xc1\x80\xbcJt\xb7\xaf\xa5\xe9\x88/\xfaO\b\v\xf8\xcf\xff\x1e\xfe\x1f\x00\x00\xff\xff73Hq. \x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcZIs\xe3\xb8\x15\xbe\xfbW\xbc\xea\x1c\xe6Ғ\xa7\xb3L\xa5tk\xcbI\x95*3nW\xcb\xf1\x1d\"\x9fD\x8cA\x80\xc1\"\x8d\xb3\xfc\xf7\xd4\x03\b\n$!Q\xd2\xf4\f\x0f]-,\x0fo\xc3\xf7\x16x6\x9bݱ\x86\xbf\xa26\\\xc9\x05\xb0\x86\xe3/\x16%\xfd2\U000f7fda9W\xf7\xfbOwo\\\x96\vX:cU\xfd\x15\x8dr\xba\xc0G\xdcr\xc9-W\xf2\xaeF\xcbJf\xd9\xe2\x0e\x80I\xa9,\xa3aC?\x01\n%\xadVB\xa0\x9e\xedP\xce\xdf\xdc\x067\x8e\x8b\x12\xb5'\x1e\x8f\xde\x7f?\xff\xf4\xc3\xfc/w\x00\x92ո\x00\xa2\xe7\x1a\xa1Xi\xe6{\x14\xa8՜\xab;\xd3`AdwZ\xb9f\x01lj\xb0\xad=2\xb0\xfb\xc8,\xfb\xa7\xa7\xe0\a\x057\xf6\x1f\x83\x89\x1f\xb9\xb1~\xb2\x11N3\xd1;Տ\x1b.wN0\x9d\xce\xdc\x01\x98B5\xb8\x80':\xb2a\x05\xd2X+\x89ga\x06\xac,\xbdn\x98x\xd6\\Z\xd4K%\\\x1du2\x83\x12M\xa1yc\xbd\xecG\x86\xc0Xf\x9d\x01\xe3\x8a\n\x98\x81'<ܯ\xe4\xb3V;\x8d&\xb0\x04\xf0\xb3Q\xf2\x99\xd9j\x01\xf3\xb0|\xdeT\xcc`;\x1bԷ\xf6\x13\xed\x90}'n\x8d\xd5\\\xeer\xe7\xbf\xf0\x1a\xa1tڛ\x8dd.\x10l\xc5M\xca\u0601\x19bN[,O\xb2\xe1牘\xb1\xacn\x86\xfc$[\x03C%\xb3\x98cg\xa9\xeaF\xa0\xc5\x126\xef\x16\xa3\x10[\xa5kf\x17\xc0\xa5\xfd\xe1ϧ5Ѫj\xee\xb7>*\xd9W\xcb\x03\x8dB2\x1c8!\v\xedPgu\xa3,\x13\xbf\x86\x11K\x04\x1e\x92\xfd\x81\x93@7\x1d\x9fde%\v\x8d5\xca\xdb\x18\xe2\xc7\xddcnR\xd2\xe9l\xa3\xb9\xd2ܾ/\xe0\xd3\xf7\x97\xb2I\xb7\x02\xd4\x16l\x85\xf0\xc0\x8a7\xd7\xc0\xda*\xcdv\b?\xaa\"\xf8ءB\xdd\xfa\xd8&,1\x95r\xa2\x84M4\f\x80\xb1Jg\x9d\xad\xc1b\x1ev\xb5t#ف\xc7\xf5\xcf\xfc\xc6w\xa1\xd0Ȳw!\x82\xe1ܯ\xe0J\xe6/\xc4\xe7\x1d^t\x19RmJUb\xa7:L9\xe2\x06\x1a\xad\n4\xe6\xcc\xf5\xa4\xed=\x1e\x9e\x8e\x03#\xb5\x84\x15\xfb?2\xd1T\xecS\x00â\u009a-\xda\x1d\xaaA\xf9\xf9y\xf5\xfa\xa7uo\x18NB\x1b+\xac!L#\xd6\x1b\xad\xac*\x94\x80\r\xda\x03\xa2\xf4\xf0\n\xb5ڣ&,\xdeqi\x80ɲ\xa3\t\xe9\x82cD!\xd7\xf7\xf4h6L\xb6\xee\xa4\x1aԩ\xd9ɕi\xcc\xf2\x18$\u0097D\xbfdt \xc4\x7fg\xbd9\x00\x92;삒\xc2 \x06\xa9\xda\x10\x80e\xab\xaa`7n@c\xa3\xd1\xd0\xf5\xf2^\xa5\xb6\xc0$\xa8\xcd\xcfX\xd8\xf9\x80\xf4\x1a5\x91\x89\xf7\xa1Pr\x8fڂ\xc6B\xed$\xffwGۀU\xfeP\xc1,\x1a\xeb/\xa4\x96L\xc0\x9e\t\x87\x1f\aڣ\xaff\uf811\xce\x04'\x13z~\x83\x19\xf2\xf1\x93\xd2\b\\n\xd5\x02*k\x1b\xb3\xb8\xbf\xdfq\x1bs\x82Bյ\x93ܾ\xdf{c\xf0\x8d\xb3J\x9b\xfb\x12\xf7(\xee\r\xdf͘.*n\xb1\xb0N\xe3=k\xf8\xcc\v\"}^0\xaf\xcb?\xe86\x8b0\xbdcG^\x18>\x1fϯ0\x0f\x85y\xba\x12\xac%\x15D\xa4M.\xb7\xa8þ\xadV\xb5\xa7\x89\xb2l\x14\x97\xd6\xff(\x04Gi\xc1\xb8M\xcd-\xb9\xc1\xbf\x1c\x1aK\xa6\x1b\x92]\xfa\xbc\t6\b\xae!((\x87\vV\x12\x96\xacF\xb1d\x06\x7fg[\x91Ǔ\x8cp\x91\xb5\xd2lp\xb88\xa87\x99\x88\t\xdd\t\xd3\x1e\xe1c\xdd`A6%\xb5\xd2&\xbe\xe5m,!\f`\xc9ʾv\xf2מ\xbel\b\x19.\x9ar5\xfa\x1er\x84\"\xaf2\xc1\xef\x18\xea\xda\xc8$\xfa\x91)\xfd\x8e \xdf\xee\xd1\xd8(í\xd2\xefD8\x84ơ\x1b\x9c\xb4\b}\x05\x93\x05\x8a[\xc4[\xfa\x9d\xc0eI\x1a\xc7\u038d\t\x80\x02UϨ\x92;E\x17+1\x04\xac,\xad \xaf6h\xf3b\xcaL(\xe3\x12\x8eI/\xa4\xc9\xedPԍR\x02\xd9P\x83\x85\xe1k\xc9\x1aS);!\xf0j\vq\xe5\xcb{\x83t\xf8r\xbd\xfaH\xff\xc4q\xf2\xa0=/[\x88\xa7[F\xd9V\xdel\xad\x9d\x97\xeb\x15\x98v\xfb\xd8H\xd2\t\xc16\x02\x17`\xb5\x1b\vv\xdaa=\xf7\x9a\xefQ\xe7f\x867\xc7/\x8c^\x18\xb6\x813>\xa9\xf6C\xafT\x90`\x94r\xa9\xa4E\x99\xb3\xd1Y\xaf\xa2/J\xba\x14\xccdy\x1ep\xb6N\xd7\xe7\xaeI$\b\x85_a+\x96\xe7\vB\xd0\xf5r\x1c7\xf1.7\x83\x03\xb7\xd5M\x12\x85\vz\xb1@\xc9\xf2\xac<\xed}\x0f\xe2\xa8\xed\x19a\x9e_\x97^\xde)\xc9(\xdc\xdc\"پg\xf4\vd\xeb{IN\xba\x01\x97\xa7\x84S\x84\x02\x04fX\x82k\xae\xe7\x9d@\x87k,\xc7<\xcfz\xf6\xcaL\xf7\x85>\x81$\xa3\xc8\x04m\xd2\xf9\x13\xa5\x95K%\xb7|7>;-\xf3\xcf]۳\xa2\x8d\"^r$i\x9c\x02\x1cq2\xf3\x19\xee,F?\xca\r\xb7|\xe7\xf4)4\xdar\x14\xe5(\x81\x99\x04\xa0\t}x&n\x89#\x9dd1~\xb7\x90\x9ad\xf6\xc1KR\x94\n\xe1o,\x03\x10t\x1f)r\x03\x1f>\x80\xd2\xf0!\xb4\x84>|\f\xbb\x1d\x17v\xc6{\xe5Ł\v\x11O\xb9*\x82v%\x05\x15t\xcaM\x85\x96\xac\x0e\xbe\fh\fTa\xa9\xf8\xf4\xe2[\x05\aƓ\xb4\xbe;\xdd|\xcc\xd0\xdd\xe0\x96r@\x8d\xd6iIQ\x18\xb5\xa6\xb4\xc8x\x92\xcae\xc2\xd0\x19IM\x12\x12'\xa4\x1cFO/\x05\xfd\x7f\x88\xe5)\x00d\x04\xc8\xd9\xf8\x1c\x87>e\xef\xfao\xb7\x98b\xdd'\x11\x99W\x9a\xef8)\\v3\xc7d\xacź\xb6k\xe1\x91\xccCq\xd6?;\xb44\x84\x96Grt\x9d\xc3\xe1\x84\xf6L\x96>_\xe8\xe6\xcb\xf6\xeae.\xee\xa4B\x9e_\x97S\xf6\xea\x0e\xce@9\r\x1f*^T}\xd3\xf11\xa8\x02X\xf6\x86>\xf7\xbe\x82\xcd<\x86\xcf\xf2\x99\xf8`\xcd\xf0\xf6\r\xa6S\x97\x1dN\xf5\r\x9d\x9d}~]^T\xad\xf8F\xcae\xf5Jh\xe4\xb6Z.\x9c־\x12\f\xa3j{S\xc5\u008a\x02\x1b\x8b\xe5\xc3\xfb\x93*\xa7\x9c\xfeso11\"/i%eL\xed\x9bKذkK\x8e\xc8n\xd7\x00\xbb\xe5\x9a~\x1e\x12\xf1\xad\x10]&\x809. \x02\u061cf\x1a\xe0\x85\x1cܗ\xf2\xdf\x05\x8c\xa4m\x1ey\xe9z\x8e\x0e\x1dQ\x88=W\xaa\xd5g\xb4\xff\xb6(\x9b/\xd5B\xff;m\x1d\xdeT\xb7\x8dɌu\xc7b\x81\xe9{\x9a\xb1\xf1\x9e\xd3ؑ\\\xa7\xaf@\rK\xc0=J\xa0R\x9cqA\xb1ۓ\xcc\x00\xd8y*m\x10\v\xaf,\xb1G\x13\xfby\xd9fٴ%3J\x18\xa3\xd9oi\xcc.\x85\xfc\x8aƉL\xd2\xf0\x1b\xa6\x90\xe1\xc8\xd0-0\xd9\x14\xf2|9\xcb\f0ЁH\x8b\x1b\xa7@\xebb%e\xf3\xca\xe1\xdb\xc4T\xd5>X\x0e\x95\x12\xadSKWoP\x13\xb7\xfe\x85\x04$\x1e(-,*&w\xd9\xc4#v\xf8\x11\x043\xb6u\xb7\x93\x1e\x92>\xb1\f%K\x9fD\x8e_\x8dư\xdd\x14X\xff\x14V\x85\xa6e\xbb\x05؆2ľֿ3m\f\xb9\n\x89\xe5t\xb8\xb8*H\xf4\xde\x1b\xae\xe6\xe4\xcb\xfa\x02^\xbe\xac\xe9\x90/\xeb_\xcb\vJW\xe7jF\xe6\xac\xca\f\v.\xdd/\x99\xf1\x03\x97\xa5:\x8c\xa1㌨\r\xb3Մ\xa0\xcf\xccV1E\xd8:!\xfc\x9eQ\xea\xdcf\x9d\x1b$L\xfcV\x19\xb4\xef\xaaM\xb1Gkr)\f^\x02\a\xa74\xff\x84\x87\xcch\f\xb9\x99\xa9\xe76\x8eg\xa6Fo\xe3\xe9dh\\\xe6\xe02\xceeiv\xcfϙ\xb9\xbf\xfb\x00w\x95\x9e[\xfen\x89\xe0]\v\xf4\x88o\xfe5y\x84r\xfdV\f\x95\x14\x89\xc52\x84\x93\xfd]\x1d\xe3)\xcd\xe1\xa5\xe2&6mc%Zr\xd3\b\xf6\xde\xc92\x156:\xdc\x1a>ƍ\x9d\xe4|\xb7\xb3{\xc4\xcfw\xaaΣ2L \xb3\x9fW\xa7Cη8\xe1L̋\xd7{\xf5xa\x89\xbdz\x8cW\x91\x97(-\xdf\xf2\xe4\x01\xf4X\xac\xf9\x86zN\x97Ç\x84\xeb\xea\xcbޟv\xdcTo\xf7(Ld\xa2\xed_\x9a\xe4\xf2\xbd5\x81\x01A\x90\x7fr[\x0e\x1f\xd9?v\x11\x9d\xd9\xf6\xdd/\x04\xff\\\x11\xab$\xa57>=\xba>\xb5\xec\v\xf4{f\x95Y\xaf\x1a\rz\xce˄v\xdb&MGܦ{\x88]\xc0\x7f\xfew\xf7\xff\x00\x00\x00\xff\xff\x12=\xc7\xe9\x11&\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcYK\x93\xe3\xb6\x11\xbeϯ\xe8\xda\x1c\xf6\xb2\xd2d\xf3p\xa5t\xdb\xd1\xc4US\xf1Ϊ\xac\xc9\xdcA\xb2E\xc1\v\x02\b\x1e\x92\xe5$\xff\xdd\xd5\x00I\x81$4z\xd8^݄n4\xbe~\xa0\x1f\xe0l6\xbbc\x9a\xbf\xa2\xb1\\\xc9\x050\xcd\xf1g\x87\x92\xfe\xd9\xf9\xd7\x7f\xd89W\xf7\xbb\x8fw_\xb9\xac\x16\xb0\xf4֩\xe6G\xb4ʛ\x12\x1fq\xc3%w\\ɻ\x06\x1d\xab\x98c\x8b;\x00&\xa5r\x8c\x96-\xfd\x05(\x95tF\t\x81fV\xa3\x9c\x7f\xf5\x05\x16\x9e\x8b\nM\x10\xde\x1d\xbd\xfb\xf3\xfc\xe3w\xf3\xbf\xdf\x01H\xd6\xe0\x02H^\xa5\xf6R(V\xd9\xf9\x0e\x05\x1a5\xe7\xea\xcej,Ipm\x94\xd7\v8\x12\xe2\xc6\xf6\xd0\b\xf8\x919\xf6\xd8\xca\b˂[\xf7\xaf\t\xe9\an] k\xe1\r\x13\xa3\xb3\x03\xc5rY{\xc1̐v\a`K\xa5q\x01\xcft\xb4f%\xd2Z\xabS\x802\x03VU\xc1JL\xac\f\x97\x0e\xcdR\t\xdft֙A\x85\xb64\\\xbb`\x85\x14\x16Xǜ\xb7`}\xb9\x05f\xe1\x19\xf7\xf7OreTm\xd0FX\x00?Y%W\xccm\x170\x8f\xecs\xbde\x16[j4\xe5:\x10\xda%w \xbc\xd6\x19.\xeb\x1c\x82\x17\xde T\xde\x04\x17\x92\xde%\x82\xdbr;\x84\xb6g\x96\xe0\x19\x87\xd5I \x81N\xe2\xacc\x8d\x1e#J\xb6FH\x15s\x98\x03\xb4T\x8d\x16谂\xe2\xe0\xb0Sc\xa3L\xc3\xdc\x02\xb8t\xdf\xfd\xed\xb4-Zc\xcd\xc3\xd6G%\x87\x86y\xa0UH\x96#\x12\xf2R\x8d&k\x1d\xe5\x98\xf8-@\x1c\txH\xf6G$Qn\xba~\x16\n\x85\x1c\xa8\r\xb8-\xc2\x03+\xbfz\rk\xa7\f\xab\x11~Pet\xdf~\x8b\x06\x03G\x119(z\x81\x93\xef\x94ɺNc9\x8f\xbc\xad\xb0N\xd6\xc8\x7fÃ~\xf7\xd8*\r\xb2llu\xa9f\x1e8\xb8\x92\xf9\x00\xfbT\xe3E\xc1\x95\x1aQ\xaa\n\x13\x8b\r0q\vڨ\x12\xad}#\xe0I\xc0\x00\xc5\xf3qab\x9aȱ\xfb\v\x13z\xcb>\xc6$Sn\xb1a\x8bv\x87\xd2(?\xad\x9e^\xff\xba\x1e,\xc3\x1b\t\x83\x95\xceR\xa6 \xf8\xda(\xa7J%\xa0@\xb7G\x94\xd1\xf5\x8dڡ\xa1\x12\xbdFCb\xc0n\x95\x17\x15)\xbbC\xe3\xc0`\xa9j\xc9\x7f\xe9e[p*\x1c*\x98C\xeb\xc2e4\x92\t\xd81\xe1\xf1\x03\x19m$\xb9a\a0Hg\x82\x97\x89\xbc\xb0\xc1\x8eq|&+r\xb9Q\v\xd8:\xa7\xed\xe2\xfe\xbe\xe6\xae+\xbb\xa5j\x1a/\xb9;\xdc\ao\xf0\xc2;e\xec}\x85;\x14\xf7\x96\xd73f\xca-wX:o\xf0\x9ei>\v\x8a\xc8Pz\xe7M\xf5'\xd3\x16j;8v\x12\x88\xf1\x17\n\xe6\x15\xee\xa1*J\xb7\x82\xb5\xa2\xa2\x8aG/\xd0\x12\x99\xee\xc7\x7f\xae_\xa0C\x12=\x15\x9drd\x9dإ\xf3\x0fY\x93\xcb\r\x9a\xb8ocT\x13d\xa2\xac\xb4\xe2҅?\xa5\xe0(\x1dX_4\xdcQ\x18\xfcǣu亱\xd8ehM\xa0@\xf0\x9a\xf2A5fx\x92\xb0d\r\x8a%\xb3\xf8\x8d}E^\xb13r\xc2E\xdeJ\x1b\xae1s4oB\xe8:\xa6\x13\xaeM3\xc8ZcI^%\xc3\xd26\xbe\xe1m%\xa14\xc0\x06\xbcC\v\xe5\xaf>\xfd\xb2\xd5d\xcct.\xdc\xe8\xf7\x90\x13ԡ\x95I\"ok\x9dm\x8b\x94\x18\x16\xa9\xf47\xa9\x8f\x06\xb5\xb2\xdc)s8V\xc9q(\x9c\xf4\n\xfdJ&K\x14\xb7\xa8\xb7\f;\x81ˊl\x8e}(S\x12\x8aR\x03P%kE\x97k\xe0\nxr\xc4C\xb1m\xd1\xe5\x15\x95٪\xc6%\x1c{JH{DZ\xba\x85R\x02\xd9؊\x14\x85\x9f\xa9,,\x95\xdc\xf0z\xaax\xda\xfe\x9e\n\x9136\xcd\x04lr$iA\xd1IHf\xa1BͺХԾ\xe1\xb57\xa7\xfc\xbf\xe1(\xaaI\xfe9y\x93:\x85\xc3)\xb7\xf8\xb8\x87\xdeݮ\xb6\xaa%\xa5ש\x90\xa1l\xe8w\x93М\x82\x04x\xda$\x12\xb9\x85w\xef@\x19x\x17g\xa2w\x1f\xe2nυ\x9b\xf1A\xfd\xdfs!\xbaS\xae\x8an\xeap\xbe\xac\xcfh\xfe\x1c\x98\bϗ\xf5\xb5\xbd\xd5\x14\rJ\xdfL\x0f\x9c\x01\xf3Ne\x96\x05\x97\xfe\xe7\xcc\xfa\x9e\xcbJ\xed\xed5\xca\xf6\xfd\r\xb5\x98ʻ[\x1c\xfee$c\xe4wG\rq\xf0\xb5S\xb0g<\xe91\xfa\xd3퇌\xdc\x027T\x90\f:o$\xa5\x034\x862\xb4\r\"\x95\x9f\xf4\xe1]\xe6\xd2\xd9'\xbe\xb6\xb6n/\xbd\tY\xb0}\x83U\x9b\x1b\xa7\x1fV\x96\xa8\x1dV\x0f\aj\x8b.\xe8\x9c\b\x80|\xfbU\xea\xdf\xfa\xd87\xa1f\u05ce(\x1d\xa4\xfe\xe5얊\xf4i,$<\x9f\x98*\xe9k\xa6pco{\x1a4\xc0\v\xd5\xe00\xfe\xbf\x8f\xad\fm\v\r\x12\xb5\xf8\x93COVi\x9a\xefg\xb4\x7f\xc2!\xbd\x10\xac\x10\xb8\x00g\xfc\xa9Y'?\xdaŇ\xe8\xf4\xcd\xf1\xa69o*fj;ֿ\xb2\x85\xd7\xd0\xee\t\x89\x88A\x80\x83E\x8af\xf9\xef\xa9\a\x10\x14HBk:ᡫ\x8d\xe5\xe1m\xf8\xde\x02M&\x93;\xd6\xf07Ԇ+9\x03\xd6p\xfcŢ\xa4\xbf\xcc\xf4\xfd\xeff\xca\xd5\xc3\xf6\xe3\xdd;\x97\xe5\f\xe6\xceXU\x7fE\xa3\x9c.\xf0\t\xd7\\r˕\xbc\xabѲ\x92Y6\xbb\x03`R*\xcbh\xd8П\x00\x85\x92V+!PO6(\xa7\xefn\x85+\xc7E\x89\xda\x13\x8fGo\xbf\x9f~\xfca\xfa\xb7;\x00\xc9j\x9c\x01\xd1s\x8dP\xac4\xd3-\n\xd4j\xca՝i\xb0 \xb2\x1b\xad\\3\x83\xc3D\xd8\xd6\x1e\x19\xd8}b\x96\xfd\xcbS\xf0\x83\x82\x1b\xfb\xcf\xc1\xc4O\xdcX?\xd9\b\xa7\x99\xe8\x9d\xea\xc7\r\x97\x1b'\x98Ng\xee\x00L\xa1\x1a\x9c\xc13\x1dٰ\x02i\xac\x95ij0\x01V\x96^7L\xbch.-\xea\xb9\x12\xae\x8e:\x99@\x89\xa6м\xb1^\xf6\x03C`,\xb3\u0380qE\x05\xcc\xc03\xee\x1e\x16\xf2E\xab\x8dF\x13X\x02\xf8\xd9(\xf9\xc2l5\x83iX>m*f\xb0\x9d\r\xea[\xfa\x89v\xc8\xee\x89[c5\x97\x9b\xdc\xf9\xaf\xbcF(\x9d\xf6f#\x99\v\x04[q\x932\xb6c\x86\x98\xd3\x16ˣl\xf8y\"f,\xab\x9b!?\xc9\xd6\xc0P\xc9,\xe6ؙ\xab\xba\x11h\xb1\x84\xd5\xdeb\x14b\xadt\xcd\xec\f\xb8\xb4?\xfc\xf5\xb8&ZUM\xfd\xd6'%\xfbjy\xa4QH\x86\x03'd\xa1\r\xea\xacn\x94e\xe2\xb70b\x89\xc0c\xb2?p\x12\xe8\xa6\xe3gYY\xc8Bc\x8d\xf26\x86\xf8a\xf7\x98\x9b\x94t:\xdbh\xae4\xb7\xfb\x19|\xfc\xfeR6\xe9V\x80Z\x83\xad\x10\x1eY\xf1\xee\x1aXZ\xa5\xd9\x06\xe1'U\x04\x1f\xdbU\xa8[\x1f[\x85%\xa6RN\x94\xb0\x8a\x86\x010V鬳5XLî\x96n$;\xf0\xb8\xfe\x99\xdf\xf8.\x14\x1aY\xf6.D0\x9c\xfa\x15\\\xc9\xfc\x85\xf8\xb4\xc1\x8b.C\xaaM\xa9J\xecT\x87)G\xdc@\xa3U\x81Ɯ\xb8\x9e\xb4\xbd\xc7\xc3\xf3a`\xa4\x96\xb0b\xfbg&\x9a\x8a}\f`XTX\xb3Y\xbbC5(?\xbd,\xde\xfe\xb2\xec\r\xc3Qhc\x855\x84i\xc4z\xa3\x95U\x85\x12\xb0B\xbbC\x94\x1e^\xa1V[Ԅ\xc5\x1b.\r0Yv4!]p\x88(\xe4\xfa\x9e\x1e͆\xc9֝T\x83:5;\xb92\x8dY\x1e\x83D\xf8\x92藌\x0e\x84\xf8ߤ7\a@r\x87]PR\x18\xc4 U\x1b\x02\xb0lU\x15\xec\xc6\rhl4\x1a\xba^ޫ\xd4\x1a\x98\x04\xb5\xfa\x19\v;\x1d\x90^\xa2&2\xf1>\x14JnQ[\xd0X\xa8\x8d\xe4\xff\xe9h\x1b\xb0\xca\x1f*\x98Ec\xfd\x85Ԓ\t\xd82\xe1\xf0~\xa0=\xfaj\xb6\a\x8dt&8\x99\xd0\xf3\x1b̐\x8f\xcfJ#p\xb9V3\xa8\xacm\xcc\xec\xe1a\xc3m\xcc\t\nU\xd7Nr\xbb\x7f\xf0\xc6\xe0+g\x956\x0f%nQ<\x18\xbe\x990]T\xdcba\x9d\xc6\a\xd6\xf0\x89\x17D\xfa\xbc`Z\x97\x7f\xd2m\x16azǎ\xbc0|>\x9e_a\x1e\n\xf3t%XK*\x88x\xb0\x02\r\x91\xea\xbe\xfec\xf9\n\x91\x93`\xa9`\x94\xc3ґ^\xa2}H\x9b\\\xaeQ\x87}k\xadjO\x13e\xd9(.\xad\xff\xa3\x10\x1c\xa5\x05\xe3V5\xb7\xe4\x06\xffvh,\x99nHv\xee\xf3&X!\xb8\x86\xa0\xa0\x1c.XH\x98\xb3\x1aŜ\x19\xfc\x83mEV1\x132\xc2E\xd6J\xb3\xc1\xe1\xe2\xa0\xded\"&tGL{\x80\x8fe\x83\x05ٔ\xd4J\x9b\xf8\x9a\xb7\xb1\x840\x80%+\xfb\xda\xc9_{\xfa\xb2!d\xb8蜫\xd1\xf7\x98#\x14y\x95\t~\xc7P\xd7F&яL\xe9w\x00\xf9v\x8f\xc6F\x19n\x95\xde\x13\xe1\x10\x1a\x87np\xd4\"\xf4\x15L\x16(n\x11o\xeew\x02\x97%i\x1c;7&\x00\nT=\xa3Jn\x14]\xac\xc4\x10\xb0\xb0\xb4\x82\xbcڠ͋)3\xa1\x8cK8$\xbd\x90&\xb7CQWJ\tdC\r\x16\x86/%kL\xa5\xec\x19\x81\x17k\x88+_\xf7\r\xd2\xe1\xf3\xe5\xe2\x9e\xfe\x89\xe3\xe4A[^\xb6\x10O\xb7\x8c\xb2\xad\xbc\xd9Z;ϗ\v0\xed\xf6\xb1\x91\xa4\x13\x82\xad\x04\xce\xc0j7\x16\xec\xb8\xc3z\xee5ߢ\xce\xcd\fo\x8e_\x18\xbd0l\x03g|R\xed\x87ި \xc1(\xe5\\I\x8b2g\xa3\x93^E_\x94t.\x98\xc9\xf2<\xe0l\x99\xae\xcf]\x93H\x10\n\xbf\xc2V,\xcf\x17\x84\xa0\xeb\xe58l\xe2]n\x06;n\xab\x9b$\n\x17\xf4b\x81\x92\xe5Yy\xda\xfb\x1e\xc4Q\xeb\x13¼\xbcͽ\xbc\xe7$\xa3ps\x8bd۞\xd1/\x90\xad\xef%9\xe9\x06\\\x1e\x13N\x11\n\x10\x98a\t\xae\xb9\x9ew\x02\x1d\xae\xb1\x1c\xf3<\xe9\xd9+3\xdd\x17\xfa\b\x92\x8c\"\x13\xb4I\xe7gJ+\xe7J\xae\xf9f|vZ柺\xb6'E\x1bE\xbc\xe4H\xd28\x058\xe2d\xe23\xdcI\x8c~\x94\x1b\xae\xf9\xc6\xe9ch\xb4\xe6(\xcaQ\x02s\x16\x80\xce\xe8\xc33qK\x1c\xe9$\x8b\xf1\xbb\x85\xd4$\xb3\x0f^\x92\xa2T\b\x7fc\x19\x80\xa0\xfb@\x91\x1b\xf8\xf0\x01\x94\x86\x0f\xa1%\xf4\xe1>\xecv\\\xd8\t\xef\x95\x17;.D<\xe5\xaa\bڕ\x14T\xd0)w.\xb4du\xf0e@c\xa0\nKŧ\x17\xdf*\xd81\x9e\xa4\xf5\xdd\xe9\xe6>Cw\x85k\xca\x015Z\xa7%EaԚ\xd2\"\xe3I*\x97\tC'$5IH<#\xe50zz)\xe8\xffC,O\x01 #@\xceƧ8\xf4)\xfb\x8f\xcbK8L\x96F\x0e\xd7\\ \x98\xbd\xb1X\xf7\xb9\r\x95@\x00\x8c\x1b\x18\xea\x1a\x82\xb7\xf8ƲO\"\xf2\xaa4\xdfp\xf2\x00\xd9\xcd\x1c\xb2\xc3\x16|\xdb6\x8a\x87V\x1f\x1b\xb2\x17\xa6\x83oC\xf0} G\xf8\x12\x0e\xa7\xf0\xc3d\xe9\x13\x98n\xbel\xb1 \x83$g\x15\xf2\xf26\xbf\xc8Fy\x00\xcb\xde\xd1\x17\x03W\xb0\x99\x0f*\x93|i0X3\x84\x83\xc1tz\x87\x86S}Cgg_\xde\xe6\x17\x95O\xbe\xb3sY\x01\x15:˭\x96\v\xa7\xb5/MèZ\xdfTB\xb1\xa2\xc0\xc6b\xf9\xb8\x7fV\xe59\xa7\xff\xd4[L\x8c\xc8Kz[\x19S\xfbn\x176\xec\xda\x1a(\xb2\xdbu\xe4n\xb9\xa6\x9f\x86D|oF\x97\t\x82\x8f+\x9a\x80~Ǚ\x06x%\a\xf7\xbd\x85\xef\x02h\xd36\x1f\n\xe8z\x8e\x0e\x1dQ\x88M\xe0\x92Y\x9c\xd0\xfe\xdb\xc2~\xbev\f\r\xf9\xb4\x97yS!9&3\xd6\x1d\x8b\x15\xafo\xb2Ɨ\x80\x9c\xc6\x0e\xe4:}\x05jX\x02nQ\x82\x92\xb0f\\P2\xe1If\x00\xec4\x956\xaa\x86g\x9f\xd84\x8a\r\xc6l\xf7\xee\xbc%3J\x18\xa3\xd9\xefi\xcc.\xa7\xfd\x8aƉL\x16\xf3;\xe6\xb4\xe1\xc8о0ٜ\xf6t}\xcd\f0ЁH\x8b\x1b\xc7@\xebb%e\x13\xdd\xe1cɹ6\xc2`9TJ\xb4N-]\xbdBM\xdc\xfa'\x1b\x90\xb8\xa3<\xb5\xa8\x98\xdcd3\xa1\xf8\xe4\x80 \x98\xb1\xad\xbb\x1d\xf5\x90\xf4\xcdg(Y\xfaFs\xf8j4\x86m\u0381\xf5\xe7\xb0*tQ\xdb-\xc0V\x94\xb2\xf6\xb5\xfe\x9dic\xc8UH,χ\x8b\xab\x82D\xef\x01\xe4jN\xbe,/\xe0\xe5˒\x0e\xf9\xb2\xfc\xad\xbc\xa0tu\xae\x88eΪ̰\xe0\xd2\xfd\x92\x19\xdfqY\xaa\xdd\x18:N\x88\xda0[\x9d\x11\xf4\x85٪K\x92\x9d\x10~\xcf(\x97o\xb3\xce\x15\x12&~\xab\x94\u07b7\xf9αGkr)\f^\x02\a\xc74\xff\x8c\xbb\xcch\f\xb9\x99\xa9\x976\x8eg\xa6F\x8f\xf5\xe9d\xe8\xa4\xe6\xe02\xceeiv\xefᙹ\x1f}\x80\xbbJ\xcf-\x7f\xb7D\xf0\xae'{\xc07\xff\xbc=B\xb9~o\x88J\x8a\xc4b\x19\xc2\xc9\xfe\xae\x8e\xf1\x94\xa6\xf0Zq\x13\xbbȱ4.\xb9i\x04\xdbw\xb2\x9c\v\x1b\x1dn\r_\a\xc7Nr\xba\xfd\xda\xfd\xaa \xdf:;\x8d\xcap\x06\x99\xfd\xbc:\x1er\xbe\xc5\t'b^\xbcދ\xa7\vk\xfe\xc5S\xbc\x8a\xbcDi\xf9\x9a'/\xb2\x87b\xcdw\xf8s\xba\x1c\xbel\\W_\xf6~krS\xbdݣp&\x13m\x7f\xfa\x92\xcb\xf7\x96\x04\x06\x04A\xfe\rp>|\xf5\xbf\xef\":\xb3\xedCd\b\xfe\xb9\"VIJo|zt}j\xd9\x17\xe8\x8f\xcc*\xb3^5\x1a\xf4\x9c\x97\t\xed\xb6o\x9b\x8e\xb8U\xf72<\x83\xff\xfe\xff\xee\xd7\x00\x00\x00\xff\xff\xf1\x86o_\xa2&\x00\x00"), } var CRDs = crds() diff --git a/design/cli-dynamic-resource-completion_design.md b/design/cli-dynamic-resource-completion_design.md new file mode 100644 index 000000000..67695c275 --- /dev/null +++ b/design/cli-dynamic-resource-completion_design.md @@ -0,0 +1,122 @@ +# Dynamic Resource Autocompletion for Velero CLI + +## Abstract + +Velero CLI has no dynamic shell completion for resource names ([#9782](https://github.com/vmware-tanzu/velero/issues/9782)). +Tab-completing `velero backup describe ` produces no suggestions, even when backups exist on the cluster. +This proposal adds dynamic completion for all commands that take Velero resource names as positional arguments or flag values (using cobra's built-in completion callbacks). + +## Background + +Shell completion is a standard UX feature in Kubernetes CLI tooling. +Tools like `kubectl`, `oc`, and `helm` all provide dynamic completions that query the cluster to suggest resource names. +Velero's `velero completion` command generates completion scripts, but the CLI does not register any completion callbacks, so tab-completing resource names produces no suggestions. +Cobra's completion infrastructure already supports dynamic completion across all shell types (bash, zsh, fish); Velero just needs to register the callbacks. + +## Goals + +- Add dynamic shell completion for all 20 commands that accept existing Velero resource names as positional arguments. +- Add dynamic flag completion for 9 flags that reference existing Velero resources. +- Fail silently when the cluster is unreachable, matching the behavior of `kubectl`. + +## Non Goals + +- Completing positional arguments for commands that take new resource names (e.g., `velero backup create `). +- Completing flags that take non-resource values (e.g., `--include-namespaces`, `--labels`). +- Adding completion for hidden internal commands (`data-mover`, `pod-volume`, `repo-maintenance`). +- Caching cluster state across tab presses. + +## High-Level Design + +A centralized set of completion functions is added to `pkg/cmd/cli/completion_functions.go`. +Each function takes a `client.Factory`, returns a closure matching cobra's completion function signature, and lists resources of a specific type from the cluster. +Each command constructor wires the appropriate completion function onto its `cobra.Command` via `ValidArgsFunction` or `RegisterFlagCompletionFunc`. + +## Detailed Design + +### Completion functions + +A new file `pkg/cmd/cli/completion_functions.go` provides six public functions: + +| Function | Resource listed | +|---|---| +| `CompleteBackupNames(f client.Factory)` | `BackupList` | +| `CompleteRestoreNames(f client.Factory)` | `RestoreList` | +| `CompleteScheduleNames(f client.Factory)` | `ScheduleList` | +| `CompleteBackupStorageLocationNames(f client.Factory)` | `BackupStorageLocationList` | +| `CompleteVolumeSnapshotLocationNames(f client.Factory)` | `VolumeSnapshotLocationList` | +| `CompleteBackupRepositoryNames(f client.Factory)` | `BackupRepositoryList` | + +All six delegate to a single private `completeNames` helper that uses `meta.ExtractList()` and `meta.Accessor()` to extract names from any `ObjectList` type. + +The completion closure: + +- Lists resources in the configured namespace with a **3-second context timeout**. +- Filters by `strings.HasPrefix(name, toComplete)`. +- Removes names already present in `args` to avoid re-suggesting previously typed arguments. +- Returns `cobra.ShellCompDirectiveNoFileComp` in all cases (success or failure). +- Fails silently on any error (client construction, API call, extraction), returning no suggestions. + +### Commands wired with `ValidArgsFunction` + +| Package | Commands | Completion function | +|---|---|---| +| `backup` | get, describe, delete, logs, download | `CompleteBackupNames` | +| `restore` | get, describe, delete, logs | `CompleteRestoreNames` | +| `schedule` | get, describe, delete, pause, unpause | `CompleteScheduleNames` | +| `backuplocation` | get, set, delete | `CompleteBackupStorageLocationNames` | +| `snapshotlocation` | get, set | `CompleteVolumeSnapshotLocationNames` | +| `repo` | get | `CompleteBackupRepositoryNames` | + +### Flags wired with `RegisterFlagCompletionFunc` + +| Command | Flag | Completion function | +|---|---|---| +| `backup create` | `--from-schedule` | `CompleteScheduleNames` | +| `backup create` | `--storage-location` | `CompleteBackupStorageLocationNames` | +| `backup create` | `--volume-snapshot-locations` * | `CompleteVolumeSnapshotLocationNames` | +| `schedule create` | `--storage-location` | `CompleteBackupStorageLocationNames` | +| `schedule create` | `--volume-snapshot-locations` * | `CompleteVolumeSnapshotLocationNames` | +| `restore create` | `--from-backup` | `CompleteBackupNames` | +| `restore create` | `--from-schedule` | `CompleteScheduleNames` | +| `debug` | `--backup` | `CompleteBackupNames` | +| `debug` | `--restore` | `CompleteRestoreNames` | + +\* See Open Issues — comma-separated values. + +## Alternatives Considered + +The approach follows the standard cobra pattern for dynamic completion. No alternative designs were considered. + +## Security Considerations + +Completion functions issue read-only list requests using the user's existing kubeconfig credentials. +No new permissions are required beyond what the user already has. +Users without list permission receive empty completions, consistent with kubectl's behavior. + +## Compatibility + +Existing command behavior is unaffected. +`ValidArgsFunction` is only invoked during shell completion; it has no effect on normal command execution. +Completion respects the `--namespace` flag and `VELERO_NAMESPACE` environment variable. + +## Testing + +Unit tests in `pkg/cmd/cli/completion_functions_test.go` cover: + +- **Core logic:** Table-driven tests across all six resource types: empty cluster, full match, prefix filtering, no match. +- **Error resilience:** Factory errors return nil completions without panicking. +- **Wrapper isolation:** Each `Complete*Names` wrapper returns only its own resource type. + +## Open Issues + +- **Single-argument commands:** Commands like `backup download` and `backup logs` accept exactly one positional argument, but cobra still calls the completion function after one arg is provided. +The completion function should check `len(args)` and return no suggestions when the maximum arg count is reached. +The approach (parameter on the helper vs. per-command wrapper) is TBD. + +- **Comma-separated flag values:** `--volume-snapshot-locations` accepts comma-separated values. +Completion only works for the first value because `toComplete` contains the full string including commas. +Completing subsequent values would require comma-aware splitting, similar to how kubectl handles this. + +- **Bash v1 to v2 migration:** The current bash completion generator already supports dynamic completion through cobra's `__complete` mechanism, so migration to v2 is not required for this feature. +A separate migration could be considered for other benefits (cleaner generated scripts, ActiveHelp support) but would require users to regenerate their completion scripts. diff --git a/design/default-resource-modifier_design.md b/design/default-resource-modifier_design.md new file mode 100644 index 000000000..5f10b3b6b --- /dev/null +++ b/design/default-resource-modifier_design.md @@ -0,0 +1,417 @@ +# Server Default Restore Resource Modifier + +- [Server Default Restore Resource Modifier](#server-default-restore-resource-modifier) + - [Abstract](#abstract) + - [Background](#background) + - [Goals](#goals) + - [Non Goals](#non-goals) + - [High-Level Design](#high-level-design) + - [Detailed Design](#detailed-design) + - [Server Configuration](#server-configuration) + - [Restore API Change](#restore-api-change) + - [Controller Logic](#controller-logic) + - [Restore CLI](#restore-cli) + - [Restore Describe Output](#restore-describe-output) + - [Install Path](#install-path) + - [Curated Default ConfigMap Example](#curated-default-configmap-example) + - [Alternatives Considered](#alternatives-considered) + - [Security Considerations](#security-considerations) + - [Compatibility](#compatibility) + - [Implementation](#implementation) + - [Open Issues](#open-issues) + +## Abstract + +This proposal introduces a server-level default restore resource modifier for Velero. +A new `--default-resource-modifier-configmap` flag on the Velero server references a ConfigMap containing resource modifier rules that apply automatically to every restore, eliminating the need for per-restore configuration for common transformations like stripping stale CNI annotations. + +## Background + +When pods are backed up, CNI-managed annotations may be present that carry pod-specific networking state such as IP addresses, MAC addresses, and routes. +Restoring these stale values can cause networking failures because the CNI expects to inject fresh values and the restored annotations may conflict with the new cluster's network state. + +The following annotations are commonly affected: + +| Annotation | CNI | +|---|---| +| `k8s.ovn.org/pod-networks` | OVN-Kubernetes | +| `k8s.v1.cni.cncf.io/network-status` | Multus | +| `k8s.v1.cni.cncf.io/networks-status` | Multus | + +Today, users can strip these annotations using [Resource Modifiers](https://velero.io/docs/main/restore-resource-modifiers/), but this requires authoring a ConfigMap and referencing it on every restore via `--resource-modifier-configmap`. +This is not discoverable for users unfamiliar with the feature and adds friction for a problem that affects most OpenShift and multi-CNI deployments. + +Velero already strips certain annotations during restore as built-in behavior (e.g., `volume.kubernetes.io/selected-node` from PVCs). +This proposal extends that concept by allowing administrators to configure a default set of resource modifier rules at the server level. + +## Goals + +- Allow Velero administrators to configure a default resource modifier ConfigMap that applies to all restores without per-restore configuration. +- Provide a mechanism for individual restores to opt out of the default modifier. +- Ship a documented example ConfigMap that strips well-known CNI annotations. + +## Non Goals + +- Auto-creating a default ConfigMap during `velero install`. The mechanism is opt-in; administrators create and configure the ConfigMap. +- Merging default and per-restore resource modifier rules. When a per-restore modifier is specified, it takes exclusive precedence over the default. +- Supporting non-ConfigMap sources for default modifiers (e.g., CRDs, inline rules). +- Stripping CNI annotations via a built-in RestoreItemAction plugin. The resource modifier mechanism is the right abstraction for this. + +## High-Level Design + +A new `--default-resource-modifier-configmap` server flag references a ConfigMap name in the Velero namespace. +During restore, if no per-restore resource modifier is specified, the server loads and applies the default ConfigMap's rules. +When a per-restore modifier is specified via `--resource-modifier-configmap`, it takes exclusive precedence and the default is not applied. +A new `--skip-default-resource-modifier` flag on `velero restore create` allows opting out of the default per-restore. + +This follows the existing pattern used by `--backup-repository-configmap` and `--repo-maintenance-job-configmap`. + +## Detailed Design + +### Server Configuration + +Add a new field to the server `Config` struct and bind it as a CLI flag. + +In `pkg/cmd/server/config/config.go`: + +```go +type Config struct { + // ... existing fields ... + DefaultResourceModifierConfigMap string +} +``` + +```go +func (c *Config) BindFlags(flags *pflag.FlagSet) { + // ... existing flags ... + flags.StringVar( + &c.DefaultResourceModifierConfigMap, + "default-resource-modifier-configmap", + c.DefaultResourceModifierConfigMap, + "The name of a ConfigMap in the Velero namespace containing default resource modifier rules applied to all restores. "+ + "Ignored when a per-restore resource modifier is specified.", + ) +} +``` + +The default value is an empty string, meaning no default modifier is configured. +No change to `GetDefaultConfig()` is needed. + +### Restore API Change + +Add a new field to `RestoreSpec` for opting out of the default modifier. + +In `pkg/apis/velero/v1/restore_types.go`: + +```go +type RestoreSpec struct { + // ... existing fields ... + + // SkipDefaultResourceModifier controls whether the server-configured default + // resource modifier is applied to this restore. + // When true, the default modifier is skipped even if configured on the server. + // Has no effect when a per-restore ResourceModifier is specified. + // +optional + // +nullable + SkipDefaultResourceModifier *bool `json:"skipDefaultResourceModifier,omitempty"` +} +``` + +This follows the existing RestoreSpec convention where optional booleans use `*bool` with `+nullable` (e.g., `RestorePVs`, `PreserveNodePorts`, `IncludeClusterResources`). +This preserves the ability to distinguish "unset" from "explicit false" if needed in the future. + + +### Controller Logic + +Thread the new config value through to the restore controller and implement the precedence logic. + +In `pkg/controller/restore_controller.go`, add a field to `restoreReconciler`: + +```go +type restoreReconciler struct { + // ... existing fields ... + defaultResourceModifierConfigMap string +} +``` + +Update `NewRestoreReconciler` to accept and store the new parameter. + +In `pkg/cmd/server/server.go`, pass `s.config.DefaultResourceModifierConfigMap` to `NewRestoreReconciler`. + +Refactor `validateAndComplete` to use a shared helper for ConfigMap loading and implement the precedence logic: + +```go +func (r *restoreReconciler) validateAndComplete(restore *api.Restore) (backupInfo, *resourcemodifiers.ResourceModifiers) { + // ... existing validation logic (unchanged) ... + + // Resource modifier resolution: per-restore takes exclusive precedence over default. + var resourceModifiers *resourcemodifiers.ResourceModifiers + + if restore.Spec.ResourceModifier != nil && + strings.EqualFold(restore.Spec.ResourceModifier.Kind, resourcemodifiers.ConfigmapRefType) { + // Per-restore modifier specified: use it exclusively, ignore default. + resourceModifiers = r.loadResourceModifierConfigMap( + restore, restore.Spec.ResourceModifier.Name, false, + ) + } else if r.defaultResourceModifierConfigMap != "" && !boolptr.IsSetToTrue(restore.Spec.SkipDefaultResourceModifier) { + // No per-restore modifier: apply server default if configured and not skipped. + resourceModifiers = r.loadResourceModifierConfigMap( + restore, r.defaultResourceModifierConfigMap, true, + ) + } + + return info, resourceModifiers +} +``` + +Extract the ConfigMap loading into a helper to avoid code duplication: + +```go +// loadResourceModifierConfigMap loads and validates a resource modifier ConfigMap. +// When isDefault is true, errors are non-fatal (logged as warnings, returns nil). +// When isDefault is false, errors are added to restore.Status.ValidationErrors. +func (r *restoreReconciler) loadResourceModifierConfigMap( + restore *api.Restore, cmName string, isDefault bool, +) *resourcemodifiers.ResourceModifiers { + cm := &corev1api.ConfigMap{} + if err := r.kbClient.Get( + context.Background(), + client.ObjectKey{Namespace: restore.Namespace, Name: cmName}, + cm, + ); err != nil { + if isDefault { + r.logger.WithError(err).Warnf( + "Failed to retrieve default resource modifier configmap %s/%s, skipping", + restore.Namespace, cmName, + ) + return nil + } + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, + fmt.Sprintf("failed to get resource modifiers configmap %s/%s", restore.Namespace, cmName)) + return nil + } + + modifiers, err := resourcemodifiers.GetResourceModifiersFromConfig(cm) + if err != nil { + if isDefault { + r.logger.WithError(err).Warnf( + "Error parsing default resource modifier configmap %s/%s, skipping", + restore.Namespace, cmName, + ) + return nil + } + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, + errors.Wrapf(err, "Error in parsing resource modifiers provided in configmap %s/%s", + restore.Namespace, cmName).Error()) + return nil + } + + if err = modifiers.Validate(); err != nil { + if isDefault { + r.logger.WithError(err).Warnf( + "Validation error in default resource modifier configmap %s/%s, skipping", + restore.Namespace, cmName, + ) + return nil + } + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, + errors.Wrapf(err, "Validation error in resource modifiers provided in configmap %s/%s", + restore.Namespace, cmName).Error()) + return nil + } + + source := "per-restore" + if isDefault { + source = "default" + } + r.logger.Infof("Retrieved %s resource modifiers from configmap %s/%s", source, restore.Namespace, cmName) + return modifiers +} +``` + +Key design decisions in this logic: + +1. **Exclusive precedence**: When a per-restore modifier is specified, the default is not applied at all. +This is the simplest mental model and avoids complex merge semantics. +Users who want both default and custom rules can copy the default rules into their per-restore ConfigMap. + +2. **Non-fatal default errors**: If the default ConfigMap is missing or invalid, log a warning and proceed without it. +A misconfigured default should not break all restores cluster-wide. +Per-restore modifier errors remain fatal (validation errors), preserving current behavior. + +3. **SkipDefaultResourceModifier**: Allows opting out per-restore without specifying a per-restore modifier. +Has no effect when a per-restore modifier is specified (it already takes precedence). + +### Restore CLI + +Add a `--skip-default-resource-modifier` flag to `velero restore create`. + +In `pkg/cmd/cli/restore/create.go`: + +```go +type CreateOptions struct { + // ... existing fields ... + SkipDefaultResourceModifier bool +} +``` + +```go +func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) { + // ... existing flags ... + flags.BoolVar(&o.SkipDefaultResourceModifier, "skip-default-resource-modifier", false, + "Skip applying the server-configured default resource modifier for this restore") +} +``` + +Set the field on the RestoreSpec when building the Restore object. +Only set it when the flag is true (using `boolptr.True()`) to leave it nil otherwise, consistent with how other `*bool` fields are handled: + +```go +if o.SkipDefaultResourceModifier { + restore.Spec.SkipDefaultResourceModifier = boolptr.True() +} +``` + +### Restore Describe Output + +Update the restore describer in `pkg/cmd/util/output/restore_describer.go` to show which resource modifier was applied and its source. +The describe output should reflect the resolved state: + +- When the default resource modifier was applied, display its ConfigMap name and source: + ``` + Default Resource Modifier: default-restore-resource-modifiers + ``` +- When the default was skipped because `SkipDefaultResourceModifier` is true: + ``` + Default Resource Modifier: skipped (SkipDefaultResourceModifier=true) + ``` +- When the default was skipped because a per-restore modifier was specified, no extra output is needed since the per-restore modifier is already displayed under the existing `Resource Modifier` field. +- When the default was ignored due to a validation or retrieval error, the warning is already logged to the restore log. The describe output should not surface transient errors. + +### Install Path + +Add the flag to the install CLI and deployment builder so administrators can configure it during installation. + +In `pkg/install/deployment.go`, add a `defaultResourceModifierConfigMap` field to `podTemplateConfig` with an option function: + +```go +func WithDefaultResourceModifierConfigMap(name string) podTemplateOption { + return func(c *podTemplateConfig) { + c.defaultResourceModifierConfigMap = name + } +} +``` + +In the `Deployment()` function, append the CLI arg: + +```go +if len(c.defaultResourceModifierConfigMap) > 0 { + args = append(args, fmt.Sprintf("--default-resource-modifier-configmap=%s", + c.defaultResourceModifierConfigMap)) +} +``` + +Wire it through `VeleroOptions` in `pkg/install/resources.go` and the install CLI in `pkg/cmd/cli/install/install.go`. + +Add a builder method to `pkg/builder/restore_builder.go`: + +```go +func (b *RestoreBuilder) SkipDefaultResourceModifier(val bool) *RestoreBuilder { + b.object.Spec.SkipDefaultResourceModifier = &val + return b +} +``` + +### Curated Default ConfigMap Example + +Provide a ready-to-use ConfigMap in `examples/default-resource-modifier-cni.yaml` that strips well-known CNI annotations: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: default-restore-resource-modifiers + namespace: velero +data: + resource-modifiers.yaml: | + version: v1 + resourceModifierRules: + - conditions: + groupResource: pods + mergePatches: + - patchData: | + metadata: + annotations: + k8s.ovn.org/pod-networks: null + k8s.v1.cni.cncf.io/network-status: null + k8s.v1.cni.cncf.io/networks-status: null +``` + +This uses JSON Merge Patch to remove annotations by setting them to `null`. +Administrators can extend this ConfigMap with additional CNI-specific annotations (Calico, Cilium, etc.) or other stale metadata as needed. + +Usage: +```bash +# Create the ConfigMap +kubectl apply -f examples/default-resource-modifier-cni.yaml + +# Configure the Velero server to use it +# Option 1: During install +velero install --default-resource-modifier-configmap=default-restore-resource-modifiers ... + +# Option 2: Edit existing deployment +kubectl -n velero edit deploy velero +# Add: --default-resource-modifier-configmap=default-restore-resource-modifiers +``` + +## Alternatives Considered + +**Merge default and per-restore rules**: Instead of exclusive precedence, concatenate default and per-restore rules so both apply. +This avoids users having to copy default rules when specifying per-restore modifiers. +However, it introduces complexity around rule ordering and makes it harder to reason about what transformations will be applied. +It also makes it impossible to fully override the default for a specific restore without the `SkipDefaultResourceModifier` flag. +Exclusive precedence was chosen for simplicity. +Merge semantics can be revisited in a future enhancement if user demand warrants it. + +**Built-in RestoreItemAction plugin**: Implement CNI annotation stripping as a built-in RIA plugin rather than using the resource modifier mechanism. +This would hard-code the logic and make it less configurable. +The resource modifier mechanism already supports this use case and is more flexible. + +**Validate default ConfigMap at server startup**: Validate the ConfigMap when the server starts rather than at restore time. +Rejected because the ConfigMap may be created after the server starts and should not require a server restart to take effect. + +**Auto-create default ConfigMap during install**: Have `velero install` automatically create the CNI-stripping ConfigMap. +Rejected for the initial release to minimize the change surface and let administrators opt in. +Can be added later as a default behavior or install flag. + +## Security Considerations + +No new security surface. +The default ConfigMap resides in the Velero namespace and is subject to the same RBAC controls as existing resource modifier ConfigMaps. +Only users with access to create/edit ConfigMaps in the Velero namespace can modify the default modifier rules. + +## Compatibility + +Fully backward compatible. +When `--default-resource-modifier-configmap` is not set (the default), behavior is identical to current Velero. +No changes to existing per-restore resource modifier behavior. +The new `SkipDefaultResourceModifier` field in RestoreSpec defaults to `false` and has no effect when no default modifier is configured. + +## Implementation + +1. Add `DefaultResourceModifierConfigMap` to `Config` struct and bind the CLI flag. +2. Add `SkipDefaultResourceModifier` to `RestoreSpec` and regenerate deepcopy/CRD. +3. Thread the config to `restoreReconciler` via `NewRestoreReconciler`. +4. Refactor `validateAndComplete` with `loadResourceModifierConfigMap` helper. +5. Add `--skip-default-resource-modifier` to the restore CLI. +6. Wire through the install path (deployment builder, install CLI). +7. Add unit tests for all precedence and error scenarios. +8. Create the example ConfigMap. +9. Update user documentation. +10. Add E2E test for default resource modifier. + + +## Open Issues + +- Should additional CNI annotations (Calico, Cilium) be included in the curated example ConfigMap? +Feedback from the community on which annotations are commonly problematic would be helpful. diff --git a/design/global-backup-volume-policies.md b/design/global-backup-volume-policies.md new file mode 100644 index 000000000..3f0ae3722 --- /dev/null +++ b/design/global-backup-volume-policies.md @@ -0,0 +1,162 @@ +# Global Backup Volume Policies for Velero + +## Background + +Velero supports [resource policies](./Implemented/handle-backup-of-volumes-by-resources-filters.md) (commonly referred to as "volume policies") that let a user control how volumes are handled during a backup — for example, whether a volume is skipped, backed up via file-system backup (`fs-backup`), snapshotted, or handled by a custom plugin. + +Today these policies are defined per-backup: + +1. A user creates a ConfigMap in the Velero install namespace whose single data key holds a `ResourcePolicies` YAML document (`volumePolicies` and the related include/exclude and fine-grained filter policies). +2. The user opts a specific backup into that ConfigMap with the CLI flag `--resource-policies-configmap`, which sets `Backup.Spec.ResourcePolicy` as a reference to the ConfigMap. +3. When the backup is processed, velero loads the referenced ConfigMap, unmarshals the YAML, builds a `Policies` object, and applies it when performing the backup. + +The limitation today is that volume policies are strictly opt-in **per backup**. An administrator, who usually has the best knowledge of the environment, may want a baseline behavior to apply to *every* backup in the cluster (for example, "always skip volumes from the `gp2` storage class", or "always use `fs-backup` for NFS volumes"). However, today they must remember to attach the same ConfigMap to every backup and every schedule. There is no way to express a cluster-wide default volume policy that is enforced regardless of what an individual backup requests. + +## Goals + +- Introduce "global backup volume policies" that an administrator configures once when the Velero server starts. +- Expose it as a Velero server CLI parameter that points to a ConfigMap in the Velero install namespace. +- When a backup runs, merge the global backup volume policies with the backup's own resource policies ConfigMap (if any) and use the merged result as the effective resource policies for that backup. +- Keep the existing per-backup `--resource-policies-configmap` behavior fully backward compatible when no global policy is configured. + +## Non Goals + +- Changing the schema of the `ResourcePolicies`/`volumePolicies` YAML itself. +- Defining global defaults for anything other than resource policies (e.g. it does not introduce new global backup spec defaults). +- Supporting per-namespace or per-schedule global policy overrides. The "global policies" is a single, server-wide configuration. +- Hot-reloading the global policies ConfigMap without a server restart is out of scope for the initial implementation. +- Support setting other filters in "resource policies" (e.g. include/exclude or fine-grained filters) in the global policy is out of scope for the initial implementation. Only `volumePolicies` will be supported in the global policy for now. + +## Design + +A new Velero server flag, `--global-backup-volume-policies-configmap`, accepts the name of a ConfigMap that lives in the Velero install namespace. The ConfigMap has the exact same format as an existing per-backup resource policies ConfigMap (a single data key holding a `ResourcePolicies` YAML document). + +The flag value is plumbed from the server `Config` into the `backupReconciler`. During `prepareBackupRequest`, in addition to loading the backup's own resource policy (referenced by `Backup.Spec.ResourcePolicy`), Velero loads the global policy ConfigMap. The two `ResourcePolicies` documents are then **merged** into a single effective `ResourcePolicies`, which is compiled into a `Policies` object, validated, and stored on `request.ResPolicies` exactly as today. The rest of the backup pipeline is unchanged because it only consumes `request.ResPolicies`. + +``` + server flag --global-backup-volume-policies-configmap + | + v + Backup.Spec.ResourcePolicy global policies ConfigMap (install ns) + | | + v v + backup-level ResourcePolicies global ResourcePolicies + \ / + \ / + v v + merge() -> effective ResourcePolicies + | + v + Policies (compiled + validated) + | + v + request.ResPolicies (unchanged consumers) +``` + +### Volume Policy only + +The resource policies ConfigMap schema includes both volume policies and include/exclude/fine-grained filter policies. The global backup volume policy only applies to the `volumePolicies` section of the schema. If the global ConfigMap includes any include/exclude/fine-grained filter policies, they are ignored and not merged into the effective policy. In this case, a warning message will be printed in the Velero server logs. +This is a design choice because only the volume policies are more tied to the environment where velero runs, and are more likely to be something an administrator would want to enforce globally. The include/exclude/fine-grained filter policies are more tied to the specific backup use case, and it would be less intuitive for an administrator to have those apply globally across all backups. + +### Validation + +Velero will validate the global backup volume policies ConfigMap at server startup. If the ConfigMap is missing or invalid, the server will fail to start and log an error. This ensures any mistakes in configuration will be caught early. +It should also make sure the validation happens for each backup, because the ConfigMap could be updated or removed after the server starts. If the global policies ConfigMap is missing or invalid at backup time, the backup CR will be put into "FailedValidation" phase, with an appropriate error message in the logs. + +### Merge semantics + +The merge combines two `ResourcePolicies` documents: the global policy (`G`) and the backup-level policy (`B`). The guiding principle is that the global policy provides a baseline, and the backup-level policy is layered with it. + +- **`volumePolicies`**: `volumePolicies` is an ordered list where the *first* matching policy wins (per the existing `Policies.match` logic). The merged list is the concatenation of the backup-level policies followed by the global policies: + + ``` + merged.volumePolicies = B.volumePolicies ++ G.volumePolicies + ``` + + This gives a backup the ability to override the global baseline for a specific volume (because its policy is evaluated first), while still inheriting all global rules that the backup does not override. + +When only the global policy is configured (the backup does not reference a resource policy), the effective policy is the global policy alone. When only the backup policy exists (no global policy configured), behavior is identical to today. + +#### Example + +Global policy ConfigMap (set on the server with `--global-backup-volume-policies-configmap=global-volume-policy`): + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: global-volume-policy + namespace: velero +data: + policies.yaml: | + version: v1 + volumePolicies: + - conditions: + storageClass: + - gp2 + action: + type: skip +``` + +Backup-level policy ConfigMap (referenced with `velero backup create --resource-policies-configmap backup01`): + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: backup01 + namespace: velero +data: + policies.yaml: | + version: v1 + volumePolicies: + - conditions: + nfs: {} + action: + type: fs-backup +``` + +Effective (merged) volume policies used for the backup — backup rules first, then global: + +```yaml +version: v1 +volumePolicies: + - conditions: + nfs: {} + action: + type: fs-backup + - conditions: + storageClass: + - gp2 + action: + type: skip +``` + +### Output of `velero backup describe` + +Currently, the `velero backup describe` command shows the backup-level resource policy. We should update the CLI to make sure the global volume policies are also shown in the output, so that user will not need to check the parameter of velero server. + +## Implementation + +- **Server flag and config.** Add a new field (e.g. `GlobalBackupVolumePoliciesConfigMap`) to the server `Config` struct in `pkg/cmd/server/config/config.go`, register the `--global-backup-volume-policies-configmap` flag in `Config.BindFlags`, and leave its default empty in `GetDefaultConfig` so the feature stays opt-in. +- **Plumb the value into the reconciler.** In `pkg/cmd/server/server.go`, pass the configured ConfigMap name (along with the Velero install namespace) into `controller.NewBackupReconciler`. Add a corresponding parameter and store it as a field on the `backupReconciler` struct in `pkg/controller/backup_controller.go`. +- **Load and merge the policies.** In `internal/resourcepolicies/resource_policies.go`, add a new function (e.g. `GetResourcePoliciesFromBackupWithGlobal`) that, in addition to loading the backup-referenced ConfigMap as `GetResourcePoliciesFromBackup` does today, also loads the global ConfigMap from the install namespace via the existing `getResourcePoliciesFromConfig` helper. After that the function merges the two `ResourcePolicies` documents according to the semantics described above. +- **Call site.** Update `prepareBackupRequest` in `pkg/controller/backup_controller.go` (currently calling `GetResourcePoliciesFromBackup`) to apply the merged policies from the new function. The rest of the backup pipeline remains unchanged. +- **CLI describe output.** Update `DescribeResourcePolicies` in `pkg/cmd/util/output/backup_describer.go` and `DescribeResourcePoliciesInSF` in `pkg/cmd/util/output/backup_structured_describer.go` to also surface the global volume policy ConfigMap that contributed to the backup. + +## Security Considerations + +The Global Backup Volume Policy is read from a ConfigMap in the Velero install namespace, the same trust boundary as existing resource policy ConfigMaps and Velero's own configuration. Setting it requires the ability to pass server flags / edit the Velero deployment, which is already an administrative privilege. No new data is exposed and no new external access patterns are introduced. + +## Compatibility + +- The feature is fully opt-in. If `--global-backup-volume-policies-configmap` is not set (the default), behavior is byte-for-byte identical to today. +- Existing per-backup `--resource-policies-configmap` usage is unchanged; it is simply merged with the global baseline when one is configured. +- Backups created before this feature, and backups that reference no resource policy, transparently start honoring the global policy once it is configured. This is the intended behavior of a "global" policy, but operators should be aware that introducing a global policy changes the effective behavior of backups that previously had no resource policy. +- The behavior of scheduled backup may change when a global backup volume policy is introduced, because the scheduled backup will start honoring the global volume policies. This is an expected change, but administrators should be aware of this when introducing a global policy to an existing velero instance with scheduled backups. +- The merged policy is computed at backup time and is reflected wherever `request.ResPolicies` is consumed. `velero backup describe` should be updated to indicate when a global policy contributed to a backup. + +## Alternatives Considered + +- **Global policies applied only when a backup has no policy of its own.** Simpler, but it makes the global policy a fallback default rather than an enforced baseline, and it cannot express "always do X in addition to whatever the backup wants". Merging is more expressive. +- **Global precedence over backup-level policies** (global volume policies evaluated first). Rejected as the default because it would prevent backups from overriding the baseline for specific volumes. diff --git a/design/restore-filter-enhancement/fine-grained-restore-filters-design.md b/design/restore-filter-enhancement/fine-grained-restore-filters-design.md new file mode 100644 index 000000000..0e4107171 --- /dev/null +++ b/design/restore-filter-enhancement/fine-grained-restore-filters-design.md @@ -0,0 +1,833 @@ +# Fine Grained Restore Filters via Resource Policies + +This is a continuation of the work done for backup filters enhancement introduced by [PR 9783](https://github.com/velero-io/velero/pull/9783), referred to as Phase 1 throughout this design. + +## Glossary & Abbreviation + +**Restore Filter**: The mechanism in Velero that determines which resources from a backup archive are restored into the target cluster. Restore filters currently operate on four dimensions: namespace, resource type, label, and cluster scope. +**Global Filter**: A filter that applies uniformly across all namespaces in a restore. All existing Velero restore filters are global filters. +**Namespace-Scoped Filter**: A filter that applies only within specific namespaces, overriding the global filter for those namespaces. This is the capability introduced by this design. +**ClusterScopedFilterPolicy**: A global filter for cluster-scoped resources that allows per-kind label selectors and name patterns, functioning similarly to `NamespacedFilterPolicy` but applied to cluster-scoped resources globally. Mirrors the backup-side concept of the same name. +**Resource Filter**: A filter rule that pairs one or more resource kinds with their own label selector and/or name patterns. Multiple resource filters within a namespace-scoped policy allow different filtering criteria for different resource types. +**Resource Name Filter**: A filter that matches individual resource instances by their metadata.name, using glob patterns. This filter dimension was introduced in Phase 1 (backup-side) and is extended to restore in this design. +**Resource Policy**: An existing Velero mechanism where backup behavior rules are defined in a ConfigMap and referenced from `BackupSpec.ResourcePolicy`. Phase 1 extended this with `namespacedFilterPolicies` and `clusterScopedFilterPolicy` for backup. This design adds an analogous `RestoreSpec.ResourcePolicy` for restore, reusing the same ConfigMap format. + +## Background + +### Why Restore-Side Filters? + +Phase 1 enables selective backup — for example, backing up only Deployments and ConfigMaps from `ns-a` while backing up everything from `ns-b`. However, backup-time filtering alone is insufficient for several real-world restore scenarios: + +**Scenario 1 — Selective restore from a full backup.** An organization performs full-cluster backups (all namespaces, all resource types) for disaster recovery. When a specific application needs recovery, the administrator wants to restore only the application's resources (specific resource types, specific names) from a single namespace — without restoring monitoring, logging, or infrastructure resources that exist in the same namespace. Today, `RestoreSpec.IncludedResources` applies globally, so filtering out ConfigMaps means filtering them out of *every* namespace being restored. + +**Scenario 2 — Cross-environment migration with selective resources.** When migrating workloads between clusters, different namespaces may need different resource types restored. A database namespace needs StatefulSets and PVCs but not Deployments; a frontend namespace needs Deployments and Services but not PVCs. The current global filter cannot express this. + +**Scenario 3 — Restore with name-based selection.** A backup contains many ConfigMaps and Secrets in a namespace (e.g., `app-config`, `app-secret`, `monitoring-config`, `monitoring-secret`). The user wants to restore only the `app-*` resources. Without name-based filtering at restore time, this requires either pre-filtering at backup time (which may not have been done) or post-restore manual cleanup. + +**Scenario 4 — Restore-time override of backup-time filters.** A backup was produced with `namespacedFilterPolicies` that included specific resources per namespace. At restore time, the operator may want to apply *different* per-namespace filters — for example, restoring only a subset of what was backed up, or applying different label selectors to handle environment differences. + +### Existing Restore Filter Mechanisms + +The restore pipeline currently supports: + +| Filter | Scope | Where Applied | +|---|---|---| +| `RestoreSpec.IncludedNamespaces` / `ExcludedNamespaces` | Global | `getOrderedResourceCollection()` | +| `RestoreSpec.IncludedResources` / `ExcludedResources` | Global | `getOrderedResourceCollection()`, `restoreItem()` | +| `RestoreSpec.LabelSelector` / `OrLabelSelectors` | Global | `getSelectedRestoreableItems()` | +| `RestoreSpec.IncludeClusterResources` | Global | `getOrderedResourceCollection()` | +| `RestoreSpec.NamespaceMapping` | Per-namespace | `getSelectedRestoreableItems()` | + +All resource-type, label, and name filters are global. There is no per-namespace override capability. + +### Design Approach: New `RestoreSpec.ResourcePolicy` Field + +Phase 1 avoided CRD changes for backup by reusing the existing `BackupSpec.ResourcePolicy` ConfigMap reference. For restore, no equivalent field exists — `RestoreSpec` has no `ResourcePolicy` field today. + +Two approaches were evaluated: + +**Option A — Reuse the backup's ResourcePolicy ConfigMap.** The restore pipeline could read the `namespacedFilterPolicies` from the backup's ConfigMap. This is rejected because: +- Restore should be able to apply *different* filters than backup +- The backup's ConfigMap may no longer exist at restore time +- The backup's ConfigMap is semantically about backup behavior, not restore +- The ConfigMap may have been updated since the backup was taken +- The ConfigMap may not exist on the target cluster, because it's maybe on a different velero instance. + +**Option B — Add `RestoreSpec.ResourcePolicy` (minimal CRD change).** Add a single `TypedLocalObjectReference` field to `RestoreSpec`, mirroring the existing `BackupSpec.ResourcePolicy` and `RestoreSpec.ResourceModifier` patterns. This is a small, focused CRD change that follows an established pattern in the codebase. + +This design uses **Option B**. The rationale: + +| Consideration | Assessment | +|---|---| +| CRD change size | **Minimal** — one `TypedLocalObjectReference` field, identical pattern to `ResourceModifier` | +| Precedent | `RestoreSpec.ResourceModifier` already uses the exact same pattern (ConfigMap ref loaded in `validateAndComplete()`) | +| Independence from backup | Restore filters are decoupled from backup filters — different ConfigMap, different lifecycle | +| Reuse | The `NamespacedFilterPolicy` and `ClusterScopedFilterPolicy` types from Phase 1 (`internal/resourcepolicies/`) are reused unchanged | + +### Why Not Just Reuse `BackupSpec.ResourcePolicy` Semantics? + +The backup-side `ResourcePolicy` ConfigMap contains multiple policy types (`volumePolicies`, `includeExcludePolicy`, `namespacedFilterPolicies`, `clusterScopedFilterPolicy`). Rather than forcing users to create a ConfigMap with backup-specific sections just to specify restore filters, this design introduces a restore-specific ConfigMap format that contains only `namespacedFilterPolicies` and `clusterScopedFilterPolicy` (and potentially other restore-specific policies in the future). + +The restore-side ConfigMap uses the **same YAML structure** for both sections. The `NamespacedFilterPolicy` and `ClusterScopedFilterPolicy` types are reused without modification. This means: +- Users who already understand the backup-side format can immediately use the restore-side one +- The `internal/resourcepolicies/` validation code is reused +- A single ConfigMap can be used for both backup and restore if the user wants (by specifying it in both `BackupSpec.ResourcePolicy` and `RestoreSpec.ResourcePolicy`) + +## Goals + +- Add a `ResourcePolicy` field to `RestoreSpec` pointing to a ConfigMap with `namespacedFilterPolicies` and/or `clusterScopedFilterPolicy` +- Reuse the `NamespacedFilterPolicy`, `ClusterScopedFilterPolicy`, and `ResourceFilter` types from Phase 1 unchanged +- Apply per-namespace resource type filters, label selectors, and resource name patterns during restore +- Apply per-kind label selectors and name patterns for cluster-scoped resources during restore +- Maintain full backward compatibility — existing restores without `ResourcePolicy` behave exactly as they do today +- Define clear precedence rules for how per-namespace filters interact with global restore filters +- Add corresponding validation in the restore controller +- Update `velero restore describe` output to display per-namespace and cluster-scoped filter information when present +- Ensure restore-side filters work correctly with both filtered and unfiltered backups + +## Non-Goals + +- Modifying the existing `NamespacedFilterPolicy`, `ClusterScopedFilterPolicy`, or `ResourceFilter` types or the `internal/resourcepolicies/` package structure (reused as-is from Phase 1) +- Adding volume policies or include/exclude policies to the restore-side ResourcePolicy ConfigMap +- Supporting regex patterns for resource names (glob patterns only, consistent with Phase 1) +- Modifying the restore plugin `ResourceSelector` system (`AppliesTo()` / `resolvedAction.ShouldUse()`) +- CLI flags for inline specification of namespace-scoped restore filters (configuration is in ConfigMap YAML) + +## Architecture of Restore-Side Filters + +### Configuration Model + +The restore-side filters are defined in a ConfigMap referenced by a new `RestoreSpec.ResourcePolicy` field. The ConfigMap YAML format reuses the `namespacedFilterPolicies` and `clusterScopedFilterPolicy` sections from Phase 1, with the same `resourceFilters` model: + +```yaml +version: v1 +clusterScopedFilterPolicy: + # NEW: global overrides for cluster-scoped resources during restore + resourceFilters: + - kinds: [ClusterRole, ClusterRoleBinding] + names: ["my-app-*"] + - kinds: [CustomResourceDefinition] + labelSelector: + app: my-app +namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [ConfigMap, Secret, Deployment] + labelSelector: + app: my-app + - namespaces: + - ns-b + resourceFilters: + - kinds: [Deployment] + names: [app-1, app-2] + - kinds: [ConfigMap] + labelSelector: + app: my-service +``` + +The restore-side ConfigMap does **not** require `volumePolicies` or `includeExcludePolicy` sections. Those are backup-specific. The YAML parser will ignore unknown fields gracefully, so a user can technically point to the same ConfigMap used for backup — the restore pipeline will only read `namespacedFilterPolicies` and `clusterScopedFilterPolicy`. + +### The `resourceFilters` Model + +Each `namespacedFilterPolicies` entry targets one or more namespaces and contains a `resourceFilters` array. Each entry in `resourceFilters` pairs one or more resource kinds with their own label selector and name patterns: + +```yaml +namespacedFilterPolicies: + - namespaces: [ns-a] + resourceFilters: + - kinds: [ConfigMap, Secret] # these kinds share a selector + labelSelector: {app: my-app} + names: ["app-*"] + - kinds: [Deployment] # this kind has its own selector + names: [workload-1, workload-2] + - kinds: [StatefulSet] # this kind has no extra filtering +``` + +Only resource kinds listed in `resourceFilters` entries are restored for the matched namespaces; unlisted kinds are implicitly excluded (globally excluded kinds cannot be re-included — see precedence model). + +#### Peek-and-Map Fallback for Unresolved Kinds + +The `kinds` field accepts both plural resource names (e.g., `configmaps`, `mycustomkinds.mygroup.io`) and singular `Kind` names (e.g., `ConfigMap`, `MyCustomKind`). + +To ensure consistent case-insensitive behavior across all code paths, Velero normalizes all input `kinds` to lowercase *before* attempting discovery or fallback matching. + +During a restore, Velero attempts to resolve `Kind` names to fully-qualified plural resource names using the cluster's discovery helper. However, for Custom Resources (CRDs), the CRD might not exist in the cluster yet when the restore begins. + +To handle this, Velero implements a **peek-and-map fallback**: +1. If a normalized `Kind` cannot be resolved via the discovery helper at the start of the restore, Velero stores the normalized string as provided in the policy. +2. Later, when iterating through the backup tarball, if Velero encounters a resource type (e.g., `mycustomkinds.mygroup.io`) that doesn't match any resolved filters, it peeks at the `Kind` of the first item in the tarball for that resource type. +3. It then checks if this actual `Kind` (case-insensitively) matches any of the unresolved normalized strings in the user's policy. +4. If a match is found, the filter is applied and cached for subsequent lookups. + +This ensures that users can intuitively write `kinds: [MyCustomKind]` and it will work reliably, even if the CRD hasn't been restored yet. This logic applies to both `namespacedFilterPolicies` and `clusterScopedFilterPolicy`. + +#### Catch-All Resource Filter (Empty `kinds` or `["*"]`) + +A `ResourceFilter` entry with an empty (or omitted) `kinds` field, or a field explicitly set to `["*"]`, acts as a **catch-all**. Its `labelSelector` or `orLabelSelectors` (if provided) is applied to **all resource types in the namespace that are not already matched by a kind-specific filter entry**. If no selectors are provided, all unlisted resources are included. Using `["*"]` is highly recommended as it makes the catch-all intention explicit and self-documenting. + +**Rules for catch-all entries:** +- At most **one** catch-all entry is allowed per `NamespacedFilterPolicy`. +- `names` and `excludedNames` are **not** supported on catch-all entries. Name patterns are kind-specific by nature and cannot be applied across arbitrary kinds; use kind-specific entries for name-based filtering. +- The catch-all applies to kinds that are **not listed in any other `resourceFilters` entry** in the same policy. Kind-specific entries take precedence over the catch-all. +- A catch-all entry **does not inherit or fall back to `RestoreSpec.LabelSelector`**. If a catch-all entry has no `labelSelector`/`orLabelSelectors`, all unlisted resource kinds in the namespace are included with **no label filtering** — the global label selector is not applied. +- **Catch-all is a `namespacedFilterPolicies`-only feature**. `clusterScopedFilterPolicy` does **not** support catch-all entries (empty or `["*"]` kinds). This is because `clusterScopedFilterPolicy` is a refinement overlay — unlisted cluster-scoped kinds already fall back to global filters by default. A catch-all would conflict with that fallback semantics. Validation rejects catch-all entries in `clusterScopedFilterPolicy`. + +**Evaluation order within a namespace filter policy:** +1. For each resource kind encountered during restore, the system first checks whether a kind-specific `resourceFilters` entry exists for that kind. +2. If a kind-specific entry exists, it is used exclusively (label selectors, name patterns from that entry). +3. If no kind-specific entry exists but a catch-all entry is present, the catch-all's `labelSelector`/`orLabelSelectors` is applied to that kind. +4. If neither a kind-specific entry nor a catch-all entry exists, the kind is excluded from the restore for that namespace. + +### Filter Precedence Model + +The restore-side namespace-scoped filter system layers on top of the existing global restore filter system. The evaluation order is: + +1. **Global namespace filter** (`RestoreSpec.IncludedNamespaces`/`ExcludedNamespaces`) is checked first. A namespace must pass this filter to be considered at all. `namespacedFilterPolicies` cannot override namespace exclusion — if a namespace is excluded globally, no filter policy entry can bring it back. + +2. **Global resource type filter** (`RestoreSpec.IncludedResources`/`ExcludedResources`) is checked next. A resource type must pass the global filter to be considered. Per-namespace filters can further narrow the set of resource types within a namespace, but cannot include a resource type that is globally excluded. + +3. **Per-namespace filter lookup.** For each namespace that passes the global filters, the system checks whether any `namespacedFilterPolicies` entry matches (by namespace name or glob pattern). If a match is found, the `resourceFilters` array determines what gets restored for that namespace: + - Only resource kinds listed in `resourceFilters[].kinds` are restored (globally excluded kinds cannot be re-included by a per-namespace policy) + - Each kind uses its own `labelSelector`/`orLabelSelectors` from its `ResourceFilter` entry, **replacing** the global label selector for that kind + - Each kind uses its own `names`/`excludedNames` patterns from its `ResourceFilter` entry + +4. **Namespaces without a matching filter policy** continue to use the global filters (`RestoreSpec.IncludedResources`, `RestoreSpec.LabelSelector`, etc.) exactly as they do today. + +5. **If multiple filter policy entries could match the same namespace** (e.g., `team-*` and `team-frontend-*` both matching `team-frontend-prod`), the **first matching policy in the list** is used. **Important: Place more specific patterns before broader patterns** to achieve the intended filtering behavior. + +6. **Namespace mapping** is applied after filter lookup. If `RestoreSpec.NamespaceMapping` maps `ns-a` to `ns-a-restored`, the filter policy lookup uses the *original* namespace name (`ns-a`), since the ConfigMap was authored against the backup's namespace structure. + +**For Cluster-Scoped Resources:** + +1. If `clusterScopedFilterPolicy` is present, it acts as a **refinement overlay** over the existing global filters for cluster-scoped resources. It is NOT an exclusive allowlist. + - If a cluster-scoped kind is listed in its `resourceFilters`, its specific `labelSelector`/`orLabelSelectors` and `names`/`excludedNames` patterns are applied. + - If a cluster-scoped kind is **not listed**, it falls back to the standard global filters (`RestoreSpec.LabelSelector`, etc.). + +2. If `clusterScopedFilterPolicy` is absent, Velero falls back to the existing global filters (`IncludedResources`, `LabelSelector`, etc.) for cluster-scoped resources. + +3. **The `velero.io/exclude-from-backup=true` label** always takes precedence over all filters. Although named for backup, this label is set on resources at backup time and remains present on items in the archive. The restore pipeline honors it: any item carrying this label is skipped regardless of whether it matches global or per-namespace restore filters. + +```mermaid +flowchart TD + A["RestoreSpec Global
IncludedNamespaces / ExcludedNamespaces"] + B{Namespace passes
global filter?} + C[Namespace excluded
from restore] + D{"Resource type passes
IncludedResources / ExcludedResources?"} + E[Resource type excluded
from restore] + G{namespacedFilterPolicies
lookup by original namespace} + H{"For each resource kind:
is kind in resourceFilters?"} + I["Apply namespace kind-specific filters:
- labelSelector / orLabelSelectors
- names / excludedNames"] + J[Kind skipped for
this namespace] + K["Use global filters:
- RestoreSpec LabelSelector
- RestoreSpec OrLabelSelectors"] + L{"Is resource
cluster-scoped?"} + M{"Is clusterScopedFilterPolicy
present?"} + N{"Is kind in clusterScopedFilterPolicy
resourceFilters?"} + O["Apply cluster kind-specific filters:
- labelSelector / orLabelSelectors
- names / excludedNames"] + + L -- Yes --> M + M -- Yes --> N + N -- Yes --> O + N -- No --> K + M -- No --> K + L -- No --> A + A --> B + B -- No --> C + B -- Yes --> D + D -- No --> E + D -- Yes --> G + G -- Match found --> H + H -- Yes --> I + H -- No --> J + G -- No match found --> K +``` + +### Key Difference from Backup-Side Precedence + +Both sides enforce the same fundamental rule: **a per-namespace filter policy cannot re-include a resource kind that has been globally excluded**. The difference lies in which global gate enforces this constraint and how unlisted kinds are handled for namespaces *without* a matching filter policy: + +- **Backup side**: The global exclusion gate is `includeExcludePolicy` (in the ResourcePolicy ConfigMap). It runs first at the resource-type level before any per-namespace lookup occurs. For a namespace that *has* a matching `namespacedFilterPolicies` entry, the per-namespace kind list acts as an exclusive allowlist — only listed kinds are collected, and no fallback to `BackupSpec.IncludedResources` occurs. However, any kind that `includeExcludePolicy` globally excludes remains excluded even if it appears in the per-namespace `resourceFilters`. For a namespace *without* a matching entry, the standard global filters (`BackupSpec.IncludedResources`, `BackupSpec.LabelSelector`, `includeExcludePolicy`) apply as before. See point 6 in the backup design's Filter Precedence Model (`fine-grained-backup-filters-design.md`) for the full treatment, including the warning log emitted when a per-namespace entry lists a globally excluded kind. +- **Restore side**: The global exclusion gate is `RestoreSpec.IncludedResources`/`ExcludedResources` directly on the RestoreSpec. It runs first, globally. For a namespace that *has* a matching `namespacedFilterPolicies` entry, the per-namespace kind list acts as an exclusive allowlist within what the global gate permits — a kind must pass the global filter and be listed in `resourceFilters` to be restored. No fallback to `RestoreSpec.IncludedResources` for additional kinds occurs. For a namespace *without* a matching entry, the standard global filters apply as before. See the "Interaction with Global `IncludedResources`/`ExcludedResources`" entry in the Edge Cases section below for a detailed example. + +In both cases, per-namespace policies are an **allowlist that operates within globally established bounds** — the label selector for a matched kind is fully replaced by the per-namespace one on both sides. + +For label selectors, **replacement** semantics are used on both sides, because label selectors are typically workload-specific and a per-namespace selector is a complete override of the filtering intent for that namespace. + +| | Backup | Restore | +|---|---|---| +| **Data source** | Live cluster — items are listed from Kubernetes API | Backup archive — items are read from tarball | +| **Operator intent** | "What should go into the archive for this namespace?" | "Of what's in the archive, what should I restore for this namespace?" | +| **Global exclusion gate** | `includeExcludePolicy` in ResourcePolicy ConfigMap | `RestoreSpec.IncludedResources` / `ExcludedResources` | +| **Namespaces without a matching policy** | Fall back to `BackupSpec.IncludedResources` + `includeExcludePolicy` | Fall back to `RestoreSpec.IncludedResources` / `ExcludedResources` | +| **Per-namespace label selector** | Replaces global label selector for that kind | Replaces global label selector for that kind | +| **clusterScopedFilterPolicy behavior** | Refinement overlay (unlisted kinds fall back to global) | Refinement overlay (unlisted kinds fall back to global) | + +### Data Flow in the Restore Pipeline + +The restore pipeline has two phases: resource selection and item restore. Namespace-scoped filters are applied in both: + +**Phase A — Resource Selection (`getOrderedResourceCollection()` + `getSelectedRestoreableItems()`)** + +Resources are enumerated from the backup archive (not from the live cluster — this is a key difference from backup). + +- **Resource type check** in `getOrderedResourceCollection()`: The global resource type check still applies. Within the namespace iteration, a per-namespace resource type check is added. If a filter policy matches the current namespace, only kinds listed in `resourceFilters[].kinds` (or matched by a catch-all) are restored — unlisted kinds are skipped for that namespace. Globally excluded kinds cannot be re-included by a per-namespace policy. +- **Label selector** in `getSelectedRestoreableItems()`: The function looks up the filter policy for the current namespace and retrieves the `ResourceFilter` entry for the current resource kind. If found, it uses that entry's `labelSelector`/`orLabelSelectors` instead of the global ones. If not found, the global selectors are used as before. +- **Name pattern check** in `getSelectedRestoreableItems()`: After the label selector check, the item's name is checked against the `ResourceFilter` entry's `names`/`excludedNames` glob patterns for the current kind. + +**Phase B — Item Restore (`restoreItem()`)** + +The `restoreItem()` function is called for each selected item and also for "additional items" requested by restore plugins. + +**Important:** Like the backup-side Stage 2 which is permissive for unlisted kinds requested by plugins, the restore-side Phase B is permissive for AdditionalItems requested by plugins regarding kind, name, and label selectors. This means if a plugin requests an AdditionalItem, it bypasses the fine-grained `namespacedFilterPolicies` and `clusterScopedFilterPolicy` checks, though it must still pass global resource/namespace exclusions. This is intentional to ensure that semantic dependencies (like a PV needed by a PVC) are successfully restored even if their specific resource kind or name pattern wasn't explicitly allowed in the user's namespace-scoped filter policy. + +### Interaction with NamespaceMapping + +When `RestoreSpec.NamespaceMapping` remaps namespaces (e.g., `ns-a` -> `ns-a-staging`), the filter policy lookup uses the **original** (backup-side) namespace name. This is because: + +- The filter ConfigMap is authored against the backup's namespace structure +- The archive directory structure uses the original namespace names +- The `getSelectedRestoreableItems()` function receives `originalNamespace` and applies mapping afterward + +The `getNamespaceFilter()` method on `restoreContext` takes the original namespace name as input. + +### Interaction with Existing Restore Features + +| Feature | Interaction | +|---|---| +| `RestoreSpec.RestorePVs` | Orthogonal — controls PV snapshot restoration, not resource inclusion | +| `RestoreSpec.ExistingResourcePolicy` | Orthogonal — controls overwrite behavior for resources that pass all filters | +| `RestoreSpec.RestoreStatus` | Orthogonal — controls status field restoration for resources that pass all filters | +| `RestoreSpec.Hooks` | Applied to resources that pass all filters. Hooks run regardless of how the item was selected | +| `RestoreSpec.ResourceModifier` | Applied to resources that pass all filters. Modifiers run on resources after filter selection | +| `RestoreSpec.PreserveNodePorts` | Orthogonal — applies to Services that pass all filters | +| Restore Item Actions (plugins) | Plugins may request "additional items." These go through `restoreItem()` which permits them, bypassing the fine-grained filter checks (similar to backup side Stage 2). | + +### Edge Cases and Behavior Documentation + +**Plugin Additional Items (Restore-Side):** +Like the backup side — which is permissive at Stage 2 to allow CSI plugin-injected resources through — the restore side is permissive for AdditionalItems in `restoreItem()`. If a restore plugin requests an additional item, it is allowed to bypass the fine-grained `namespacedFilterPolicies` and `clusterScopedFilterPolicy` kind, name, and label selector checks. This allows plugins to successfully restore dependencies (like a PV needed by a PVC, or a specific Secret) without the user having to explicitly authorize every single dependent resource type in their configuration. Note that these additional items must still pass global resource/namespace exclusions. + +**Multiple Glob Patterns Matching Same Namespace (Incorrect Order):** +```yaml +namespacedFilterPolicies: + - namespaces: ["team-*"] # Broader pattern listed first + resourceFilters: + - kinds: [Deployment, Service] + - namespaces: ["team-frontend-*"] # More specific pattern listed second + resourceFilters: + - kinds: [ConfigMap, Secret, Deployment, Service] +``` +**Behavior:** For namespace `team-frontend-prod`, the broader `team-*` pattern matches first, so only `Deployment` and `Service` are restored. The more specific `team-frontend-*` rule is never reached. + +**Multiple Glob Patterns Matching Same Namespace (Correct Order):** +```yaml +namespacedFilterPolicies: + - namespaces: ["team-frontend-*"] # More specific pattern listed first + resourceFilters: + - kinds: [ConfigMap, Secret, Deployment, Service] + - namespaces: ["team-*"] # Broader pattern listed second + resourceFilters: + - kinds: [Deployment, Service] +``` +**Behavior:** For namespace `team-frontend-prod`, the specific `team-frontend-*` pattern matches first, restoring all specified resources. For `team-backend-dev`, the broader `team-*` pattern matches, restoring only `Deployment` and `Service`. This achieves the intended behavior. + +**Namespace Included Globally But No Matching Filter Policy:** +```yaml +# RestoreSpec includes "production" namespace +# ResourcePolicy has no namespacedFilterPolicies entry for "production" +``` +**Behavior:** The namespace uses global filters exactly as it does today. This is the backward compatibility behavior. + +**Empty ResourceFilters Array:** +```yaml +namespacedFilterPolicies: + - namespaces: ["test-namespace"] + resourceFilters: [] # empty array +``` +**Behavior:** Validation error during restore creation: +``` +namespacedFilterPolicies[0]: at least one resourceFilter must be specified +``` + +**Namespace Pattern with No Matches:** +```yaml +namespacedFilterPolicies: + - namespaces: ["nonexistent-*"] + resourceFilters: [...] +``` +**Behavior:** No error. The filter policy is loaded but never applied since no namespaces match the pattern. + +**Resource Kind Not Present in Target Namespaces:** +```yaml +resourceFilters: + - kinds: ["StatefulSet"] # namespace has no StatefulSets in the backup archive + names: ["workload-1"] +``` +**Behavior:** No error. The filter is applied but finds no matching resources. Empty result set is valid. + +**Conflicting Name Patterns:** +```yaml +resourceFilters: + - kinds: ["ConfigMap"] + names: ["app-*"] + excludedNames: ["app-config"] # conflicts with names pattern +``` +**Behavior:** The `excludedNames` takes precedence. Resources matching `app-*` are included, then `app-config` is excluded. Net result: includes `app-secret`, `app-data`, etc., but excludes `app-config`. + +**Invalid Label Selector Syntax:** +```yaml +resourceFilters: + - kinds: ["Deployment"] + labelSelector: + "invalid label key!": "value" # invalid key syntax +``` +**Behavior:** Validation error during restore creation when `labels.ValidatedSelectorFromSet()` fails: +``` +namespacedFilterPolicies[0].resourceFilters[0]: invalid label selector: "invalid label key!" is not a valid label key +``` + +**Out-of-Scope Kinds in Filter Entries:** +A user may accidentally list a cluster-scoped kind (e.g., `ClusterRole`) inside a `namespacedFilterPolicies` entry, or a namespace-scoped kind (e.g., `ConfigMap`) inside `clusterScopedFilterPolicy`. The system silently ignores such entries at the archive traversal level: namespace-scoped items are never in the cluster-scope portion of the archive, and vice versa. A warning is logged at restore start so the user can detect the misconfiguration: + +``` +WARN kind "ClusterRole" in namespacedFilterPolicies[0].resourceFilters[1] is a cluster-scoped resource; it will never match in a namespace-scoped filter — did you mean clusterScopedFilterPolicy? +``` + +**Discovery Helper Unavailable:** +If the discovery helper is unavailable during restore initialization, the restore fails with: +``` +failed to resolve namespace filter policies: discovery client unavailable +``` + +**Interaction with Global `IncludedResources`/`ExcludedResources`:** + +`namespacedFilterPolicies` operates within the bounds already established by the global resource type filter — it is a refinement, not a replacement. `RestoreSpec.IncludedResources`/`ExcludedResources` is applied first at the resource-type level, before any per-namespace filter policy is consulted. A namespace-scoped filter policy cannot re-include a resource kind that has been globally excluded. + +Two separate gates are applied in order: +1. **`RestoreSpec.IncludedResources`/`ExcludedResources` runs first**, globally, across all namespaces. It decides which resource types are eligible at all. +2. **`namespacedFilterPolicies` runs second**, within the bounds established by step 1. It can only further restrict kinds that survived the global gate — it cannot widen it. + +```yaml +# RestoreSpec +excludedResources: [secrets] # global — Secrets excluded from all namespaces + +# ResourcePolicy ConfigMap +namespacedFilterPolicies: + - namespaces: [ns-a] + resourceFilters: + - kinds: [ConfigMap, Secret] # Secret listed here is ineffective — globally excluded + labelSelector: + app: my-app + - kinds: [Deployment] +``` + +**What gets restored from `ns-a`:** +- `ConfigMap` with label `app=my-app` — restored (listed in per-namespace policy, not globally excluded) +- `Secret` with label `app=my-app` — **not restored** (globally excluded by `ExcludedResources`, even though listed in the per-namespace policy) +- `Deployment` — restored (listed in per-namespace policy, not globally excluded) + +The "no fallback to `RestoreSpec.IncludedResources`" rule means that for a namespace *with* a matching policy, only the kinds listed in `resourceFilters` are candidates for restore — `RestoreSpec.IncludedResources` is not consulted to add additional kinds. The global `ExcludedResources` exclusions, however, still apply because they are enforced at an earlier, separate stage. + +To restore `Secret` in specific namespaces, users must remove `secrets` from `ExcludedResources` globally, or restructure their policy. + +A warning is logged at restore start when a `namespacedFilterPolicies` entry lists a kind that is globally excluded: +``` +level=warn msg="namespacedFilterPolicies entry lists a kind that is globally excluded by RestoreSpec.ExcludedResources; the per-namespace filter entry has no effect" kind="secrets" namespacePattern="ns-a" +``` + +> **See also:** The backup-side design's "Interaction with `includeExcludePolicy`" (point 6 in the Filter Precedence Model of `fine-grained-backup-filters-design.md`) documents the structurally identical behavior for backup. The only difference is the global gate: on the backup side it is `includeExcludePolicy` (in the ResourcePolicy ConfigMap); on the restore side it is `RestoreSpec.IncludedResources`/`ExcludedResources` (on the RestoreSpec directly). + +# Detailed Design + +## Workflow + +### Restore Workflow + +The restore workflow is preserved with the following additions. The modules in the existing restore path remain unchanged when `ResourcePolicy` is absent from `RestoreSpec`. + +**Step 1 — Load and parse policies (in `restore_controller.go`, `validateAndComplete()`)** + +The restore controller loads the ConfigMap, similar to how `ResourceModifier` is loaded today: + +The loaded policies are passed through to `runValidatedRestore()` and stored on the `restore.Request`. + +**Step 2 — Resolve namespace and cluster-scoped filter maps (in `restore.go`, `RestoreWithResolvers()`)** + +After existing filter setup, the filter policies are resolved into the runtime maps: + +The `resolveRestoreNamespacedFilterPolicies` function: +- For each `NamespacedFilterPolicy`, iterates its `ResourceFilters` entries +- Resolves kind names to fully-qualified group-resource strings using the discovery helper +- Converts `labelSelector` maps into `labels.Selector` objects using `labels.ValidatedSelectorFromSet()` +- Converts `orLabelSelectors` maps into `[]labels.Selector` +- Creates `IncludesExcludes` instances for `names`/`excludedNames` patterns +- Identifies catch-all entries (empty or `["*"]` kinds) and stores them in `catchAllFilter` +- Builds a `resourceFilterMap` keyed by the resolved group-resource string +- Returns both the map and an ordered `namespacedFilterPatterns` slice for first-match traversal + +**Step 3 — Per-namespace resource type check (in `restore.go`, `getOrderedResourceCollection()`)** + +Inside the namespace iteration, after the global namespace check and global resource type check, and before calling `getSelectedRestoreableItems()`: + +**Step 4 — Label selector and name filter (in `restore.go`, `getSelectedRestoreableItems()`)** + +Before the items loop, resolve the effective `ResourceFilter` (hoisted for performance). The function handles three cases in order: + +1. **Namespace-scoped item with a matching `namespacedFilterPolicies` entry** — resolve the effective `ResourceFilter` by checking the kind-specific entry first, then falling back to the catch-all +2. **Cluster-scoped item with the kind listed in `clusterScopedFilterPolicy`** — apply that kind's label/name filters (refinement overlay; unlisted cluster-scoped kinds fall through to global) +3. **All other cases** — fall back to the existing global label selector logic + +**Note on cluster-scoped resources:** There is no separate kind-level skip step in `getOrderedResourceCollection()` for cluster-scoped resources analogous to Step 3. `clusterScopedFilterPolicy` is a refinement overlay — unlisted cluster-scoped kinds are not skipped; they fall through to existing global filter handling. Behavior changes only when the kind is explicitly listed in `clusterScopedFilterMap`, and only in `getSelectedRestoreableItems()` (above). + +### Backup Workflow + +No changes. The backup pipeline is unaffected by this design. + +### Delete Workflow + +No changes. Restore deletion removes the restore metadata. The backup archive is unaffected. + +## Validation + +The following validation is added in `restore_controller.go`'s `validateAndComplete()`: + +1. **ConfigMap existence and format**: Handled by `GetResourcePoliciesFromRestore()`, which returns validation errors if the ConfigMap is missing, malformed, or fails `Policies.Validate()`. + +2. **`ResourcePolicy.Kind` must be `"configmap"`** (case-insensitive): Consistent with `BackupSpec.ResourcePolicy` and `RestoreSpec.ResourceModifier`. + +3. **Namespace filter policy validation** (delegated to `Policies.Validate()`): + - Each filter policy must specify at least one namespace + - Each filter policy must specify at least one resource filter + - Each resource filter without kinds can only be defined once (at most one catch-all), and cannot specify `names`/`excludedNames` + - No duplicate kinds across resource filter entries within the same namespace filter + - `labelSelector` and `orLabelSelectors` cannot co-exist within each resource filter + - No duplicate exact namespace patterns across filter policies (overlapping glob patterns are allowed — first-match semantics handle them at runtime) + - Name/excludedNames patterns must be valid globs + +4. **`clusterScopedFilterPolicy` validation** (delegated to `Policies.Validate()`): + - At least one resourceFilter must be specified + - Each resource filter must specify at least one kind — **catch-all (empty `kinds` or `["*"]`) is NOT permitted in `clusterScopedFilterPolicy`** since it is a refinement overlay rather than an allowlist + - No duplicate kinds across resource filters + - `labelSelector` and `orLabelSelectors` mutual exclusion + - Resource name patterns must be valid globs + +5. **Mutual exclusion with global `OrLabelSelectors`/`LabelSelector`**: If `namespacedFilterPolicies` are present and the `RestoreSpec` also has both `LabelSelector` and `OrLabelSelectors`, the existing validation catches this. No additional validation needed for the interaction — per-namespace selectors simply override the global ones for matching namespaces. + +## ConfigMap Examples + +### Restore-Specific ResourcePolicy ConfigMap + +Restore only Deployments and ConfigMaps (labeled `app=my-app`) from `ns-a`, but everything from `ns-b`: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: restore-filter-policy + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [Deployment, ConfigMap] + labelSelector: + app: my-app + # ns-b has no filter policy entry, so global filters apply (restore everything) +``` + +Restore CR: + +```yaml +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: selective-restore + namespace: velero +spec: + backupName: full-backup + includedNamespaces: + - ns-a + - ns-b + resourcePolicy: + kind: configmap + name: restore-filter-policy +``` + +### Restore with Name Pattern Filtering + +Restore only `app-*` ConfigMaps and Secrets from `production`: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: app-restore-filter + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [ConfigMap, Secret] + names: ["app-*"] + excludedNames: ["*-tmp", "*-debug"] +``` + +### Catch-All with No Label Selector (Override-Only) + +A user may want to use the global configuration for 99% of resources in a namespace, but only apply a specific name filter to a single kind. A catch-all filter without a label selector achieves this: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: override-only-restore-policy + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + - namespaces: + - ns-a + resourceFilters: + - kinds: [Secret] + names: [my-secret] # Specific override for Secrets + - kinds: ["*"] # Catch-all: NO label selector + # Restores all other kinds unconditionally +``` + +**Result:** +- `Secret` resources: only `my-secret` is restored. +- All other resource types: restored unconditionally (acting like a global fallback). + +### Catch-All with Per-Kind Name Overrides + +Use exact names for specific kinds, and fall back to a label selector for all remaining kinds: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: mixed-restore-filter-policy + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [Deployment] + names: [api-server, worker] # these exact Deployments by name + - kinds: [Secret] + names: [db-credentials, tls-cert] # these exact Secrets by name + - kinds: ["*"] # catch-all for all other kinds + labelSelector: + backup: "true" # restore by label +``` + +**Result:** +- `Deployment` resources: only `api-server` and `worker` are restored. +- `Secret` resources: only `db-credentials` and `tls-cert` are restored. +- All other resource types: restored only if they carry `backup=true`. + +### Cluster-Scoped Filter Policy + +Restore only specific ClusterRoles and CRDs matching a label: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-restore-filter + namespace: velero +data: + policy: | + version: v1 + clusterScopedFilterPolicy: + resourceFilters: + - kinds: [ClusterRole, ClusterRoleBinding] + names: ["my-app-*"] + - kinds: [CustomResourceDefinition] + labelSelector: + app: my-app + namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [Deployment, ConfigMap, Secret, StatefulSet, PersistentVolumeClaim] +``` + +### Restore with Glob Namespace Patterns + +Apply the same filter to all namespaces matching a pattern. **Critical: Order patterns from most specific to least specific:** + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: team-restore-filter + namespace: velero +data: + policy: | + version: v1 + namespacedFilterPolicies: + # More specific patterns first + - namespaces: + - "team-frontend-prod" # Most specific (exact match) + resourceFilters: + - kinds: [Deployment, Service, ConfigMap, Secret, PersistentVolumeClaim] + - namespaces: + - "team-frontend-*" # Less specific (pattern match) + resourceFilters: + - kinds: [Deployment, Service, ConfigMap] + - namespaces: + - "team-*" # Least specific (broad pattern) + resourceFilters: + - kinds: [Deployment, Service] +``` + +**Pattern Matching Results:** +- `team-frontend-prod` → Uses exact match policy (restores 5 resource types) +- `team-frontend-dev` → Uses `team-frontend-*` policy (restores 3 resource types) +- `team-backend-test` → Uses `team-*` policy (restores 2 resource types) +- `app-namespace` → No match, uses global filters + +### Same ConfigMap for Backup and Restore + +A single ConfigMap can be referenced by both `BackupSpec.ResourcePolicy` and `RestoreSpec.ResourcePolicy`. The backup pipeline uses `volumePolicies`, `includeExcludePolicy`, `namespacedFilterPolicies`, and `clusterScopedFilterPolicy`. The restore pipeline uses only `namespacedFilterPolicies` and `clusterScopedFilterPolicy`: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: shared-policy + namespace: velero +data: + policy: | + version: v1 + volumePolicies: + - conditions: + capacity: "0,10Gi" + action: + type: fs-backup + clusterScopedFilterPolicy: + resourceFilters: + - kinds: [ClusterRole, ClusterRoleBinding] + names: ["my-app-*"] + namespacedFilterPolicies: + - namespaces: + - production + resourceFilters: + - kinds: [Deployment, ConfigMap, Secret, StatefulSet, PersistentVolumeClaim] +``` + +### Restore CR — No ResourcePolicy (backward compatible) + +Existing restores continue to work exactly as before: + +```yaml +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: full-restore + namespace: velero +spec: + backupName: my-backup + includedNamespaces: + - "*" +``` + +## CLI + +### `velero restore describe` + +The output is extended to display resource policy configmap name when present: + +``` +Name: selective-restore +Namespace: velero +Labels: +Annotations: + +Phase: Completed + +Errors: 0 +Warnings: 0 + +Backup: full-backup + +Namespaces: + Included: ns-a, ns-b + Excluded: + +Resources: + Included: * + Excluded: + Cluster-scoped: auto + +Namespace Mapping: + +Label Selector: + +Resource Policy: restore-filter-policy + +Restore PVs: auto + +... +``` + +### `velero restore create` + +A new `--resource-policies-configmap` flag is added to `velero restore create`, mirroring the existing backup-side flag: + +```bash +velero restore create selective-restore \ + --from-backup full-backup \ + --include-namespaces ns-a,ns-b \ + --resource-policies-configmap restore-filter-policy +``` + +The `--help` output for `velero restore create` is updated to clarify the interaction between global and namespace-scoped filters: + +``` +Restore Filtering Options: + --include-namespaces stringArray namespaces to include in the restore (use '*' for all namespaces) + --exclude-namespaces stringArray namespaces to exclude from the restore + --include-resources stringArray resources to include in the restore, formatted as resource.group + --exclude-resources stringArray resources to exclude from the restore, formatted as resource.group + --include-cluster-resources optionalBool[=true] include cluster-scoped resources + --selector labelSelector only restore resources matching this label selector + --or-selector labelSelector restore resources matching any of the label selectors (can be repeated) + --resource-policies-configmap string reference to a configmap containing resource policies for namespace-scoped and cluster-scoped filtering + +Notes: +- Global filters (--include-resources, --selector, etc.) apply to all included namespaces +- Namespace-scoped filters defined in --resource-policies-configmap refine global filters for matching namespaces (globally excluded kinds cannot be re-included) +- Fine-grained global filter policies defined in --resource-policies-configmap refine global filters for cluster-scoped resources +- Use 'velero restore describe' to view resolved filter policies after restore creation +``` + +## User Perspective + +- **For users not using restore-side filter policies**: Zero changes. All existing restores work identically. +- **For users adopting restore-side filter policies**: Create a ConfigMap with the `namespacedFilterPolicies` and/or `clusterScopedFilterPolicy` sections and reference it via `RestoreSpec.ResourcePolicy` (or `--resource-policies-configmap` CLI flag). The restore will selectively include/exclude resources per namespace. +- **For users already using backup-side filter policies**: Restore-side policies are independent. A backup-side ConfigMap can be reused for restore (both `BackupSpec.ResourcePolicy` and `RestoreSpec.ResourcePolicy` can point to the same ConfigMap), or a different ConfigMap can be used. +- **Interaction with NamespaceMapping**: Filter policies use the original (backup-side) namespace names. If `NamespaceMapping` remaps `ns-a` to `ns-b`, the filter ConfigMap should reference `ns-a`. +- **`velero restore describe`**: Shows per-namespace and cluster-scoped filter details when `ResourcePolicy` is present. +- **Validation errors**: Reported at restore start when the ConfigMap is invalid. + +## Alternatives Considered + +1. **Reuse Backup's ResourcePolicy ConfigMap**: Automatically apply the backup's `namespacedFilterPolicies` during restore without requiring restore-side configuration. Rejected because restore should be independently configurable from backup, and the backup's ConfigMap may not exist at restore time or may have been modified. + +2. **No CRD Change — Annotation-Based Reference**: Use a Velero annotation on the Restore CR to point to the ConfigMap instead of a CRD field. Rejected because annotations are not validated, not documented via `kubectl explain`, and are inconsistent with how the backup side works. + +3. **Embed Filter Policies in RestoreSpec (Full CRD Approach)**: Add `NamespacedFilters []NamespaceFilter` directly to `RestoreSpec`. Rejected because it requires complex nested CRD types, doesn't reuse the Phase 1 ConfigMap infrastructure, and is a drift from backup side design. + +4. **CLI-Only (No CRD Change)**: Express restore filters entirely via CLI flags that get stored as annotations. Rejected because it doesn't support the declarative Restore CR workflow and is not auditable. diff --git a/design/volume-policy-pvc-volume-mode-access-modes.md b/design/volume-policy-pvc-volume-mode-access-modes.md new file mode 100644 index 000000000..1c5abba94 --- /dev/null +++ b/design/volume-policy-pvc-volume-mode-access-modes.md @@ -0,0 +1,359 @@ +# Add PVC VolumeMode and AccessModes as Criteria for Volume Policy + +## Abstract +This proposal extends Velero VolumePolicy conditions with two PVC-based criteria, `pvcVolumeMode` and `pvcAccessModes`. +These conditions allow users to select volumes according to the `volumeMode` and `accessModes` of the associated PersistentVolumeClaim (PVC), enabling backup behavior such as skipping block-mode PVCs or choosing a specific backup method for volumes with selected access modes. + +## Background +Velero VolumePolicy already supports selecting volumes by attributes such as capacity, storage class, volume source, volume type, PVC labels, and PVC phase. +PVC metadata and spec fields are often the most direct way for users to express the intended storage semantics of a workload. + +Kubernetes PVCs include a `spec.volumeMode` field that describes whether the volume is exposed as a filesystem or as a raw block device. +The field supports values such as `Filesystem` and `Block`. + +Kubernetes PVCs also include a `spec.accessModes` field that describes how the volume can be mounted. +Common values are `ReadWriteOnce`, `ReadOnlyMany`, `ReadWriteMany`, and `ReadWriteOncePod`. +For resource policies, `pvcAccessModes` uses an exact set match against the PVC's `spec.accessModes`, so a policy does not match PVCs that have missing or additional access modes. + +## Goals +- Add a `pvcVolumeMode` VolumePolicy condition to match volumes by a single `spec.volumeMode` value of their associated PVC. +- Add a `pvcAccessModes` VolumePolicy condition to match volumes whose associated PVC has exactly the configured `spec.accessModes` values, regardless of order. +- Keep the new conditions consistent with existing VolumePolicy behavior, where all conditions in a policy must match and the first matching policy wins. + +## Non-Goals +- This proposal does not add new VolumePolicy actions. +- This proposal does not change how PVCs are discovered or passed into the resource policy matching code. +- This proposal does not add set-based or negative matching operators such as `NotIn`, `Exists`, or `DoesNotExist`. +- This proposal does not change Kubernetes PVC semantics or validate storage provider capabilities. + +## Use-cases/Scenarios + +### Skip block-mode PVCs +A user wants to skip volumes whose associated PVC is configured with raw block volume mode. + +```yaml +version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: Block + action: + type: skip +``` + +### Snapshot filesystem PVCs +A user wants to use snapshots only for volumes whose associated PVC has filesystem mode. + +```yaml +version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: Filesystem + action: + type: snapshot +``` + +### Match PVCs by access mode +A user wants to apply a policy only to PVCs whose `spec.accessModes` is exactly `ReadWriteOnce`. + +```yaml +version: v1 +volumePolicies: +- conditions: + pvcAccessModes: + - ReadWriteOnce + action: + type: skip +``` + +### Match an exact access mode set +A user wants to match volumes whose associated PVC access modes are exactly `ReadOnlyMany` and `ReadWriteMany`. +A PVC that includes only one of these modes, or includes additional modes, does not match. + +```yaml +version: v1 +volumePolicies: +- conditions: + pvcAccessModes: + - ReadOnlyMany + - ReadWriteMany + action: + type: snapshot +``` + +### Combine PVC spec criteria +A user wants to select block-mode PVCs whose access modes are exactly `ReadWriteOnce`. +Because VolumePolicy conditions are conjunctive, the volume must satisfy both conditions. + +```yaml +version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: Block + pvcAccessModes: + - ReadWriteOnce + action: + type: snapshot +``` + +## High-Level Design +The VolumePolicy condition schema is extended with two optional fields, `pvcVolumeMode` and `pvcAccessModes`. +`pvcVolumeMode` is represented as a single string value in the resource policy YAML. +`pvcAccessModes` is represented as a string list in the resource policy YAML. + +The internal `structuredVolume` representation is extended to store the associated PVC's volume mode and access modes. +The existing PVC parsing path populates these fields when a PVC is available in `VolumeFilterData`. + +The policy builder creates a `pvcVolumeModeCondition` when `pvcVolumeMode` is specified and creates a `pvcAccessModesCondition` when `pvcAccessModes` is specified. +The existing matching flow remains unchanged: each condition implements the `volumeCondition` interface, all conditions in a policy must match, and the first matching policy's action is returned. + +## Detailed Design + +### Resource policy YAML schema +Two new fields are added under `volumePolicies[].conditions`. + +```yaml +version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: Block + pvcAccessModes: + - ReadWriteOnce + - ReadWriteMany + action: + type: snapshot +``` + +`pvcVolumeMode` is a string. +The intended values are Kubernetes PVC volume mode values, including `Filesystem` and `Block`. +The condition matches only when the PVC volume mode value observed by Velero exactly equals the configured value. +Matching is case-sensitive, so `block` does not match `Block`. + +`pvcAccessModes` is a list of strings. +The intended values are Kubernetes PVC access mode values, including `ReadWriteOnce`, `ReadOnlyMany`, `ReadWriteMany`, and `ReadWriteOncePod`. +The condition matches only when the configured access modes exactly equal the PVC's `spec.accessModes`, ignoring order. +Matching is case-sensitive, so `readwriteonce` does not match `ReadWriteOnce`. + +The implementation validates that `pvcVolumeMode`, when present, is a string. +The implementation validates that `pvcAccessModes`, when present, is a list of strings. +The implementation does not strictly reject unknown string values so that the condition format remains tolerant of Kubernetes additions or storage-provider-specific behavior. +Unknown `pvcVolumeMode` values match only when the PVC has the same string value, and unknown `pvcAccessModes` values match only as part of the same exact access-mode set. + +### Volume condition struct +The parsed condition struct is extended as follows. + +```go +type volumeConditions struct { + Capacity string `yaml:"capacity,omitempty"` + StorageClass []string `yaml:"storageClass,omitempty"` + NFS *nFSVolumeSource `yaml:"nfs,omitempty"` + CSI *csiVolumeSource `yaml:"csi,omitempty"` + VolumeTypes []SupportedVolume `yaml:"volumeTypes,omitempty"` + PVCLabels map[string]string `yaml:"pvcLabels,omitempty"` + PVCPhase []string `yaml:"pvcPhase,omitempty"` + PVCVolumeMode string `yaml:"pvcVolumeMode,omitempty"` + PVCAccessModes []string `yaml:"pvcAccessModes,omitempty"` +} +``` + +### Structured volume data +The internal `structuredVolume` is extended with `pvcVolumeMode` and `pvcAccessModes`. + +```go +type structuredVolume struct { + capacity resource.Quantity + storageClass string + nfs *nFSVolumeSource + csi *csiVolumeSource + volumeType SupportedVolume + pvcLabels map[string]string + pvcPhase string + pvcVolumeMode string + pvcAccessModes []string +} +``` + +When a PVC is available, `parsePVC` extracts PVC attributes into `structuredVolume` for later condition evaluation. +This parsing step does not create or imply a `pvcVolumeMode` policy condition; `pvcVolumeMode` only constrains matching when the user explicitly configures `conditions.pvcVolumeMode` in the VolumePolicy. +Velero uses `pvc.Spec.VolumeMode` as-is when it is present. +If `pvc.Spec.VolumeMode` is nil, `pvcVolumeMode` remains empty and does not match any non-empty `pvcVolumeMode` condition. +If `pvc.Spec.AccessModes` is empty, `pvcAccessModes` remains empty and does not match any non-empty `pvcAccessModes` condition. + +```go +func (s *structuredVolume) parsePVC(pvc *corev1api.PersistentVolumeClaim) { + if pvc != nil { + if len(pvc.GetLabels()) > 0 { + s.pvcLabels = pvc.Labels + } + s.pvcPhase = string(pvc.Status.Phase) + if pvc.Spec.VolumeMode != nil { + s.pvcVolumeMode = string(*pvc.Spec.VolumeMode) + } + if len(pvc.Spec.AccessModes) > 0 { + s.pvcAccessModes = make([]string, 0, len(pvc.Spec.AccessModes)) + for _, accessMode := range pvc.Spec.AccessModes { + s.pvcAccessModes = append(s.pvcAccessModes, string(accessMode)) + } + } + } +} +``` + +### PVC volume mode condition +`pvcVolumeModeCondition` matches when the associated PVC's parsed volume mode exactly equals the configured value. +The comparison is case-sensitive and does not normalize values. +An empty configured value is treated as no constraint and always matches, consistent with other VolumePolicy conditions. +A non-empty configured value does not match if no PVC volume mode is available. + +```go +type pvcVolumeModeCondition struct { + volumeMode string +} + +func (c *pvcVolumeModeCondition) match(v *structuredVolume) bool { + if c.volumeMode == "" { + return true + } + if v.pvcVolumeMode == "" { + return false + } + return v.pvcVolumeMode == c.volumeMode +} +``` + +### PVC access modes condition +`pvcAccessModesCondition` matches when the configured access modes exactly equal the associated PVC's access modes, ignoring order. +The comparison is case-sensitive and does not normalize values. +An empty configured list is treated as no constraint and always matches. +A non-empty configured list does not match if the structured volume has no PVC access modes, has a different number of access modes, or has a different access-mode set. + +```go +type pvcAccessModesCondition struct { + accessModes []string +} + +func (c *pvcAccessModesCondition) match(v *structuredVolume) bool { + if len(c.accessModes) == 0 { + return true + } + if len(v.pvcAccessModes) == 0 || len(v.pvcAccessModes) != len(c.accessModes) { + return false + } + + return sets.New(c.accessModes...).Equal(sets.New(v.pvcAccessModes...)) +} +``` + +### Condition validation +Both `pvcVolumeModeCondition` and `pvcAccessModesCondition` implement the `validate()` method required by the `volumeCondition` interface. +The `validate()` method returns nil for both conditions. + +```go +func (c *pvcVolumeModeCondition) validate() error { + return nil +} + +func (c *pvcAccessModesCondition) validate() error { + return nil +} +``` + +YAML shape validation is handled when resource policy conditions are unmarshaled. +`pvcVolumeMode` must be a string, and `pvcAccessModes` must be a list of strings. +Condition-level validation intentionally does not reject unknown string values. +This keeps the policy format forward-compatible with future Kubernetes values and consistent with other string-based VolumePolicy conditions. +Unknown values simply do not match normal PVCs unless the evaluated PVC has the same exact value or access-mode set. + +### Policy builder integration +The policy builder appends the new conditions only when the corresponding YAML fields are present. + +```go +func (p *Policies) BuildPolicy(resPolicies *ResourcePolicies) error { + for _, vp := range resPolicies.VolumePolicies { + con, err := unmarshalVolConditions(vp.Conditions) + if err != nil { + return errors.WithStack(err) + } + + // Existing conditions are appended here. + + if con.PVCVolumeMode != "" { + volP.conditions = append(volP.conditions, &pvcVolumeModeCondition{volumeMode: con.PVCVolumeMode}) + } + if len(con.PVCAccessModes) > 0 { + volP.conditions = append(volP.conditions, &pvcAccessModesCondition{accessModes: con.PVCAccessModes}) + } + } + return nil +} +``` + +### Matching behavior with other conditions +The new conditions follow the existing VolumePolicy matching behavior. +Within a single policy, every configured condition must match. +If `pvcVolumeMode` is omitted from a policy, Velero does not add a volume mode condition and the policy does not restrict volume mode. +`pvcVolumeMode` and `pvcAccessModes` are PVC-specific conditions and only match when the volume policy evaluation has associated PVC data. +For non-PVC volumes such as `emptyDir`, `configMap`, or inline volumes without an associated PVC, the parsed PVC fields are empty and policies requiring `pvcVolumeMode` or `pvcAccessModes` do not match. +Across multiple policies, the first matching policy wins. + +For example, this policy matches only PVC-backed volumes that are both `Block` mode and have exactly `ReadWriteOnce` as their access modes. + +```yaml +version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: Block + pvcAccessModes: + - ReadWriteOnce + action: + type: snapshot +``` + +## Alternatives Considered + +### A single `pvcSpec` condition object +One alternative is to add a nested object such as `pvcSpec.volumeMode` and `pvcSpec.accessModes`. +This was not chosen because existing PVC-based VolumePolicy conditions use flat field names such as `pvcLabels` and `pvcPhase`. +Flat names keep the YAML concise and consistent with existing conditions. + +### List-based `pvcVolumeMode` +One alternative is to make `pvcVolumeMode` a list, similar to `pvcPhase`. +This was not chosen because Kubernetes PVC `spec.volumeMode` is a single value and the policy condition is intended to describe an exact match against that value. +Using a string avoids implying that multiple volume modes can apply to one PVC. + +### Contains-based access mode matching +Another alternative is to make `pvcAccessModes` match when any or all configured access modes are present on the PVC. +This was not chosen because contains-based matching would also select PVCs with additional access modes. +Using exact set matching keeps `pvcAccessModes` consistent with `pvcVolumeMode`'s exact-match behavior and avoids matching PVCs whose access mode set differs from the policy. + +### Strict validation of allowed Kubernetes values +Another alternative is to reject `pvcVolumeMode` or `pvcAccessModes` values that are not currently known Kubernetes constants. +This was not chosen because accepting strings is more forward-compatible and keeps behavior consistent with other string-based resource policy conditions. +Invalid or unknown values naturally fail to match unless a PVC has the same value. + +## Security Considerations +This proposal does not introduce new privileges or access to additional Kubernetes resources. +It only uses PVC data already available to the volume policy matching path. + +The new conditions can cause Velero to skip or choose different backup actions for matched volumes. +Users should review policy configuration carefully because an overly broad policy can exclude data from backup or select an unintended backup method. + +## Compatibility +The new fields are optional and do not affect existing resource policy files. +Existing VolumePolicy behavior remains unchanged when `pvcVolumeMode` and `pvcAccessModes` are not configured. + +PVCs without a parsed `spec.volumeMode` value do not match non-empty `pvcVolumeMode` conditions. +PVCs without `spec.accessModes` do not match non-empty `pvcAccessModes` conditions. + +Unknown `pvcVolumeMode` or `pvcAccessModes` string values in a policy are accepted as strings but will not match normal Kubernetes PVCs unless the evaluated PVC has the same exact value or access-mode set. + +## Implementation +Implementation requires changes in the resource policies package and documentation. + +- Extend `volumeConditions` with `PVCVolumeMode string` and `PVCAccessModes []string`. +- Extend `structuredVolume` with `pvcVolumeMode string` and `pvcAccessModes []string`. +- Update `parsePVC` to populate the new fields from the PVC spec. +- Add `pvcVolumeModeCondition` and `pvcAccessModesCondition` implementations. +- Update `Policies.BuildPolicy` to append the new conditions. +- Add YAML type validation to ensure `pvcVolumeMode` is a string and `pvcAccessModes` is a string list. +- Add unit tests for parsing, validation, condition matching, and end-to-end `GetMatchAction` behavior. +- Update user documentation in `site/content/docs/main/resource-filtering.md`. diff --git a/go.mod b/go.mod index 0cb694712..a2c41faf6 100644 --- a/go.mod +++ b/go.mod @@ -3,21 +3,22 @@ module github.com/vmware-tanzu/velero go 1.26.0 require ( - cloud.google.com/go/storage v1.62.1 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1 + cloud.google.com/go/storage v1.62.3 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.6.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0 github.com/RoaringBitmap/roaring v1.9.4 - github.com/aws/aws-sdk-go-v2 v1.24.1 - github.com/aws/aws-sdk-go-v2/config v1.26.3 - github.com/aws/aws-sdk-go-v2/credentials v1.16.14 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.11 + github.com/aws/aws-sdk-go-v2 v1.41.12 + github.com/aws/aws-sdk-go-v2/config v1.32.17 + github.com/aws/aws-sdk-go-v2/credentials v1.19.16 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.18 github.com/aws/aws-sdk-go-v2/service/ec2 v1.143.0 - github.com/aws/aws-sdk-go-v2/service/s3 v1.48.0 - github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 - github.com/bombsimon/logrusr/v3 v3.0.0 + github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 + github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 + github.com/bombsimon/logrusr/v3 v3.1.0 + github.com/cockroachdb/errors v1.13.0 github.com/evanphx/json-patch/v5 v5.9.11 github.com/fatih/color v1.19.0 github.com/gobwas/glob v0.2.3 @@ -25,32 +26,31 @@ require ( github.com/google/uuid v1.6.0 github.com/hashicorp/go-hclog v1.6.3 github.com/hashicorp/go-plugin v1.7.0 - github.com/joho/godotenv v1.3.0 github.com/kopia/kopia v0.16.0 github.com/kubernetes-csi/external-snapshot-metadata v1.0.0 github.com/kubernetes-csi/external-snapshotter/client/v8 v8.4.0 + github.com/netresearch/go-cron v0.15.0 github.com/onsi/ginkgo/v2 v2.28.3 github.com/onsi/gomega v1.40.0 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 - github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_model v0.6.2 - github.com/robfig/cron/v3 v3.0.1 - github.com/sirupsen/logrus v1.9.3 - github.com/spf13/afero v1.10.0 + github.com/sirupsen/logrus v1.9.4 + github.com/spf13/afero v1.15.0 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 - github.com/vmware-tanzu/crash-diagnostics v0.3.7 + github.com/vmware-tanzu/crash-diagnostics v0.4.3 + github.com/vmware-tanzu/velero/pkg/apis v0.0.0 go.uber.org/zap v1.28.0 - golang.org/x/mod v0.35.0 + go.yaml.in/yaml/v3 v3.0.4 + golang.org/x/mod v0.36.0 golang.org/x/oauth2 v0.36.0 - golang.org/x/sys v0.43.0 - golang.org/x/text v0.36.0 - google.golang.org/api v0.277.0 - google.golang.org/grpc v1.80.0 + golang.org/x/sys v0.46.0 + golang.org/x/text v0.37.0 + google.golang.org/api v0.283.0 + google.golang.org/grpc v1.81.1 google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af - gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.36.0 k8s.io/apiextensions-apiserver v0.36.0 k8s.io/apimachinery v0.36.0 @@ -75,39 +75,41 @@ require ( cloud.google.com/go/monitoring v1.24.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect - github.com/Masterminds/semver/v3 v3.4.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 // indirect - github.com/aws/smithy-go v1.19.0 // indirect + github.com/Masterminds/semver/v3 v3.5.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect + github.com/aws/smithy-go v1.27.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.12.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chmduquesne/rollinghash v4.0.0+incompatible // indirect github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/redact v1.1.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/edsrzf/mmap-go v1.2.0 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect - github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect - github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect - github.com/go-ini/ini v1.67.0 // indirect + github.com/getsentry/sentry-go v0.46.0 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -128,12 +130,13 @@ require ( github.com/go-openapi/swag/yamlutils v0.25.5 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gofrs/flock v0.13.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.7.1 // indirect github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect github.com/google/s2a-go v0.1.9 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.16 // indirect github.com/googleapis/gax-go/v2 v2.22.0 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/hashicorp/cronexpr v1.1.3 // indirect @@ -141,19 +144,25 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kcp-dev/apimachinery/v2 v2.0.1-0.20250223115924-431177b024f3 // indirect + github.com/kcp-dev/kcp/cli v0.27.1 // indirect + github.com/kcp-dev/kcp/sdk v0.27.1 // indirect + github.com/kcp-dev/logicalcluster/v3 v3.0.5 // indirect github.com/klauspost/compress v1.18.6 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/klauspost/crc32 v1.3.0 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/klauspost/reedsolomon v1.14.0 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect github.com/kubernetes-csi/external-snapshot-metadata/client v1.0.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect - github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-colorable v0.1.15 // indirect github.com/mattn/go-isatty v0.0.21 // indirect github.com/minio/crc64nvme v1.1.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/minio-go/v7 v7.1.0 // indirect + github.com/minio/minio-go/v7 v7.2.0 // indirect github.com/moby/spdystream v0.5.1 // indirect github.com/moby/term v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -162,52 +171,57 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-vss v1.2.1 // indirect github.com/natefinch/atomic v1.0.1 // indirect - github.com/nxadm/tail v1.4.8 // indirect github.com/oklog/run v1.1.0 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/common v0.67.5 // indirect + github.com/prometheus/common v0.68.1 // indirect github.com/prometheus/procfs v0.20.1 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/rs/xid v1.6.0 // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/tinylib/msgp v1.6.1 // indirect - github.com/vladimirvivien/gexe v0.1.1 // indirect + github.com/vladimirvivien/gexe v0.4.0 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/zeebo/blake3 v0.2.4 // indirect github.com/zeebo/xxh3 v1.1.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.42.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect - go.opentelemetry.io/otel v1.43.0 // indirect - go.opentelemetry.io/otel/metric v1.43.0 // indirect - go.opentelemetry.io/otel/sdk v1.43.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect - go.opentelemetry.io/otel/trace v1.43.0 // indirect - go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/sdk v1.44.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.starlark.net v0.0.0-20241226192728-8dfa5b98479f // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.50.0 // indirect + golang.org/x/crypto v0.52.0 // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect - golang.org/x/net v0.53.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/term v0.42.0 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/term v0.43.0 // indirect golang.org/x/time v0.15.0 // indirect golang.org/x/tools v0.44.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect k8s.io/streaming v0.36.0 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect ) -replace github.com/kopia/kopia => github.com/project-velero/kopia v0.0.0-20260512025144-908c5c098101 +replace ( + github.com/kopia/kopia => github.com/project-velero/kopia v0.0.0-20260616052725-d83462d382c9 + github.com/vmware-tanzu/velero/pkg/apis => ./pkg/apis +) diff --git a/go.sum b/go.sum index 1a0945cd3..ed0070272 100644 --- a/go.sum +++ b/go.sum @@ -1,44 +1,13 @@ cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA= @@ -47,23 +16,12 @@ cloud.google.com/go/longrunning v0.9.0 h1:0EzbDEGsAvOZNbqXopgniY0w0a1phvu5IdUFq8 cloud.google.com/go/longrunning v0.9.0/go.mod h1:pkTz846W7bF4o2SzdWJ40Hu0Re+UoNT6Q5t+igIcb8E= cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -cloud.google.com/go/storage v1.62.1 h1:Os0G3XbUbjZumkpDUf2Y0rLoXJTCF1kU2kWUujKYXD8= -cloud.google.com/go/storage v1.62.1/go.mod h1:cpYz/kRVZ+UQAF1uHeea10/9ewcRbxGoGNKsS9daSXA= +cloud.google.com/go/storage v1.62.3 h1:SZq1t23NCI+e96dH77Dg3PEfsNNEjqO8zE5AnD8gVD0= +cloud.google.com/go/storage v1.62.3/go.mod h1:cpYz/kRVZ+UQAF1uHeea10/9ewcRbxGoGNKsS9daSXA= cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U= cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1 h1:jHb/wfvRikGdxMXYV3QG/SzUOPYN9KEUUuC0Yd0/vC0= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1/go.mod h1:pzBXCYn05zvYIrwLgtK8Ap8QcjRg+0i76tMQdWN6wOk= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 h1:aokoqcHvaGjiM3VpjKDfMMnF/8epJ+Q1HLJ7CudztqE= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0/go.mod h1:/WYEx9pcM9Y+Dd/APJaNlSvVSvzl54rrMdZT5+Oi2LM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= @@ -80,23 +38,14 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1. github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0 h1:BM85pSYlVYQHdq00nxyPoOkyLF5NArJG3bOsrmbwr4k= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0/go.mod h1:QYjP2cB7ZYtS/8jAbE0VSBZde/tjExqGjp+8JY6/+ts= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= -github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= -github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 h1:IEjq88XO4PuBDcvmjQJcQGg+w+UaafSy8G5Kcb5tBhI= github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5/go.mod h1:exZ0C/1emQJAw5tHOaUDyY1ycttqBAPcxuzf7QbY6ec= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ= @@ -107,105 +56,78 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= -github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= -github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= +github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv2QzDdQ= github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU= -github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo= -github.com/aws/aws-sdk-go-v2/config v1.26.3 h1:dKuc2jdp10y13dEEvPqWxqLoc0vF3Z9FC45MvuQSxOA= -github.com/aws/aws-sdk-go-v2/config v1.26.3/go.mod h1:Bxgi+DeeswYofcYO0XyGClwlrq3DZEXli0kLf4hkGA0= -github.com/aws/aws-sdk-go-v2/credentials v1.16.14 h1:mMDTwwYO9A0/JbOCOG7EOZHtYM+o7OfGWfu0toa23VE= -github.com/aws/aws-sdk-go-v2/credentials v1.16.14/go.mod h1:cniAUh3ErQPHtCQGPT5ouvSAQ0od8caTO9OOuufZOAE= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.11 h1:I6lAa3wBWfCz/cKkOpAcumsETRkFAl70sWi8ItcMEsM= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.11/go.mod h1:be1NIO30kJA23ORBLqPo1LttEM6tPNSEcjkd1eKzNW0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM= -github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10 h1:5oE2WzJE56/mVveuDZPJESKlg/00AaS2pY2QZcnxg4M= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10/go.mod h1:FHbKWQtRBYUz4vO5WBWjzMD2by126ny5y/1EoaWoLfI= +github.com/aws/aws-sdk-go-v2 v1.41.12 h1:DIKX2c31ekm9RA2D9FBj1EWXx++9AdAqRw+e78Tq2Ck= +github.com/aws/aws-sdk-go-v2 v1.41.12/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= +github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU= +github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.18 h1:9XFUd2lkr7VrbE4Qtrhm7AtNhGgZeGFI5QLZtQIflj8= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.18/go.mod h1:trImuKdWelQIJALvyGj6sKolJ1W8t628JOoTdDGVL9Q= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= github.com/aws/aws-sdk-go-v2/service/ec2 v1.143.0 h1:ZAO4y7MSRqU74ZFCA+HC6Ek5fI7dsTdwJg88s72I/gE= github.com/aws/aws-sdk-go-v2/service/ec2 v1.143.0/go.mod h1:hIsHE0PaWAQakLCshKS7VKWMGXaqrAFp4m95s2W9E6c= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10 h1:L0ai8WICYHozIKK+OtPzVJBugL7culcuM4E4JOpIEm8= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10/go.mod h1:byqfyxJBshFk0fF9YmK0M0ugIO8OWjzH2T3bPG4eGuA= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10 h1:KOxnQeWy5sXyS37fdKEvAsGHOr9fa/qvwxfJurR/BzE= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10/go.mod h1:jMx5INQFYFYB3lQD9W0D8Ohgq6Wnl7NYOJ2TQndbulI= -github.com/aws/aws-sdk-go-v2/service/s3 v1.48.0 h1:PJTdBMsyvra6FtED7JZtDpQrIAflYDHFoZAu/sKYkwU= -github.com/aws/aws-sdk-go-v2/service/s3 v1.48.0/go.mod h1:4qXHrG1Ne3VGIMZPCB8OjH/pLFO94sKABIusjh0KWPU= -github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 h1:dGrs+Q/WzhsiUKh82SfTVN66QzyulXuMDTV/G8ZxOac= -github.com/aws/aws-sdk-go-v2/service/sso v1.18.6/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 h1:Yf2MIo9x+0tyv76GljxzqA3WtC5mw7NmazD2chwjxE4= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8= -github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0= -github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U= -github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= -github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA= github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/bombsimon/logrusr/v3 v3.0.0 h1:tcAoLfuAhKP9npBxWzSdpsvKPQt1XV02nSf2lZA82TQ= -github.com/bombsimon/logrusr/v3 v3.0.0/go.mod h1:PksPPgSFEL2I52pla2glgCyyd2OqOHAnFF5E+g8Ixco= +github.com/bombsimon/logrusr/v3 v3.1.0 h1:zORbLM943D+hDMGgyjMhSAz/iDz86ZV72qaak/CA0zQ= +github.com/bombsimon/logrusr/v3 v3.1.0/go.mod h1:PksPPgSFEL2I52pla2glgCyyd2OqOHAnFF5E+g8Ixco= github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chmduquesne/rollinghash v4.0.0+incompatible h1:hnREQO+DXjqIw3rUTzWN7/+Dpw+N5Um8zpKV0JOEgbo= github.com/chmduquesne/rollinghash v4.0.0+incompatible/go.mod h1:Uc2I36RRfTAf7Dge82bi3RU0OQUmXT9iweIcPqvr8A0= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/cockroachdb/errors v1.13.0 h1:BoCcJeiP9hpBJDETkX19qi8Tb8So37srSsp3stTaDMQ= +github.com/cockroachdb/errors v1.13.0/go.mod h1:bjxt/4E5+OyuAnacpTIU9rn2mzPu1VlthvHP+xpROq0= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/container-storage-interface/spec v1.12.0 h1:zrFOEqpR5AghNaaDG4qyedwPBqU2fU0dWjLQMP/azK0= github.com/container-storage-interface/spec v1.12.0/go.mod h1:txsm+MA2B2WDa5kW69jNbqPnvTtfvZma7T/zsAZ9qX8= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= @@ -216,72 +138,45 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/edsrzf/mmap-go v1.2.0 h1:hXLYlkbaPzt1SaQk+anYwKSRNhufIDCchSPkUD6dD84= github.com/edsrzf/mmap-go v1.2.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= -github.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3reDNEnfOboJ0QySbH736g= -github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v1.3.0 h1:TvGH1wof4H33rezVKWSpqKz5NXWg5VPuZ0uONDT6eb4= -github.com/envoyproxy/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA= -github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/getsentry/sentry-go v0.46.0 h1:mbdDaarbUdOt9X+dx6kDdntkShLEX3/+KyOsVDTPDj0= +github.com/getsentry/sentry-go v0.46.0/go.mod h1:evVbw2qotNUdYG8KxXbAdjOQWWvWIwKxpjdZZIvcIPw= github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= -github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= -github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -291,16 +186,10 @@ github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.22.5 h1:8on/0Yp4uTb9f4XvTrM2+1CPrV05QPZXu+rvu2o9jcA= github.com/go-openapi/jsonpointer v0.22.5/go.mod h1:gyUR3sCvGSWchA2sUBJGluYMbe1zazrYWIkWPjjMUY0= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.21.5 h1:6uCGVXU/aNF13AQNggxfysJ+5ZcU4nEAe+pJyVWRdiE= github.com/go-openapi/jsonreference v0.21.5/go.mod h1:u25Bw85sX4E2jzFodh1FOKMTZLcfifd1Q+iKKOUxExw= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.25.5 h1:pNkwbUEeGwMtcgxDr+2GBPAk4kT+kJ+AaB+TMKAg+TU= github.com/go-openapi/swag v0.25.5/go.mod h1:B3RT6l8q7X803JRxa2e59tHOiZlX1t8viplOcs9CwTA= github.com/go-openapi/swag/cmdutils v0.25.5 h1:yh5hHrpgsw4NwM9KAEtaDTXILYzdXh/I8Whhx9hKj7c= @@ -331,166 +220,59 @@ github.com/go-openapi/testify/enable/yaml/v2 v2.4.0 h1:7SgOMTvJkM8yWrQlU8Jm18VeD github.com/go-openapi/testify/enable/yaml/v2 v2.4.0/go.mod h1:14iV8jyyQlinc9StD7w1xVPW3CO3q1Gj04Jy//Kw4VM= github.com/go-openapi/testify/v2 v2.4.0 h1:8nsPrHVCWkQ4p8h1EsRVymA2XABB4OT40gcvAu+voFM= github.com/go-openapi/testify/v2 v2.4.0/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.2.2 h1:TUR3TgtSVDmjiXOgAAyaZbYmIeP3DPkld3jgKGV8mXQ= github.com/godbus/dbus/v5 v5.2.2/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= +github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg= github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.15 h1:xolVQTEXusUcAA5UgtyRLjelpFFHWlPQ4XfWGc7MBas= -github.com/googleapis/enterprise-certificate-proxy v0.3.15/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/enterprise-certificate-proxy v0.3.16 h1:F/VPrx0YPBdksZJQdCAp0WUsqnNmZpUZszzfYt0M5Dw= +github.com/googleapis/enterprise-certificate-proxy v0.3.16/go.mod h1:9Yb0eAkH/Xqhvv3zbeKf/+wMJqCeocWc6KIhDvEAuYE= github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4= github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= -github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hanwen/go-fuse/v2 v2.10.1 h1:QAqZuc9+aBtTou+OPruU/hkYQYCkgPtQd2QaepHkTTs= github.com/hanwen/go-fuse/v2 v2.10.1/go.mod h1:aU7NkGYZUmuJrZapoI3mEcNve7PZTySUOLBuch/vR6U= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/cronexpr v1.1.3 h1:rl5IkxXN2m681EfivTlccqIryzYJSXRGRNa0xeG7NA4= github.com/hashicorp/cronexpr v1.1.3/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-plugin v1.7.0 h1:YghfQH/0QmPNc/AZMTFE3ac8fipZyZECHdDPshfk+mA= github.com/hashicorp/go-plugin v1.7.0/go.mod h1:BExt6KEaIYx804z8k4gRzRLEvxKVb+kn0NMcihqOqb8= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/yamux v0.1.2 h1:XtB8kyFOyHXYVFnwT5C3+Bdo8gArse7j2AQ0DA0Uey8= github.com/hashicorp/yamux v0.1.2/go.mod h1:C+zze2n6e/7wshOZep2A70/aQU6QBRWJO/G6FT1wIns= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jhump/protoreflect v1.17.0 h1:qOEr613fac2lOuTgWN4tPAtLL7fUSbuJL5X5XumQh94= @@ -499,23 +281,20 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kcp-dev/apimachinery/v2 v2.0.1-0.20250223115924-431177b024f3 h1:YwNX7ZIpQXg9u5vav/fobmf4nnO0WhbELWaL3X74Oe4= +github.com/kcp-dev/apimachinery/v2 v2.0.1-0.20250223115924-431177b024f3/go.mod h1:n0+EV+LGKl1MXXqGbGcn0AaBv7hdKsdazSYuq8nM8Us= +github.com/kcp-dev/kcp/cli v0.27.1 h1:ogPTtNk1A/4uVY/az3ncDfandpRTK+O4zDnQlMppHfI= +github.com/kcp-dev/kcp/cli v0.27.1/go.mod h1:CNb21eZd2V3CBNSQPcZrhrXM6RrJ94xLZTCgEccSsLU= +github.com/kcp-dev/kcp/sdk v0.27.1 h1:jBVdrZoJd5hy2RqaBnmCCzldimwOqDkf8FXtNq5HaWA= +github.com/kcp-dev/kcp/sdk v0.27.1/go.mod h1:3eRgW42d81Ng60DbG1xbne0FSS2znpcN/GUx4rqJgUo= +github.com/kcp-dev/logicalcluster/v3 v3.0.5 h1:JbYakokb+5Uinz09oTXomSUJVQsqfxEvU4RyHUYxHOU= +github.com/kcp-dev/logicalcluster/v3 v3.0.5/go.mod h1:EWBUBxdr49fUB1cLMO4nOdBWmYifLbP1LfoL20KkXYY= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= @@ -529,17 +308,10 @@ github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/reedsolomon v1.14.0 h1:5YSZeclzSYg5nl349+GDG/agDtQ6MZiwUYXvVKN1Jx0= github.com/klauspost/reedsolomon v1.14.0/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kopia/htmluibuild v0.0.1-0.20260502040510-a4505d4145ae h1:igSzPZDDs3icBsXWC/2zRFBRlzelXcBSODpxpORf6s8= -github.com/kopia/htmluibuild v0.0.1-0.20260502040510-a4505d4145ae/go.mod h1:h53A5JM3t2qiwxqxusBe+PFgGcgZdS+DWCQvG5PTlto= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kopia/htmluibuild v0.0.1-0.20260608231842-7bf9fcc0831d h1:2nHZuoDenhCDeIdDnQXLaaMziiHfIjf+INorIIUVNk8= +github.com/kopia/htmluibuild v0.0.1-0.20260608231842-7bf9fcc0831d/go.mod h1:h53A5JM3t2qiwxqxusBe+PFgGcgZdS+DWCQvG5PTlto= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kubernetes-csi/csi-lib-utils v0.23.2 h1:+x9W4RRyuRnJcTiSHKEFpl0cYUeLUqshM5ioPeYPRXw= @@ -556,45 +328,24 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= -github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs= github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.1.0 h1:QEt5IStDpxgGjEdtOgpiZ5QhmSl3ax7qy61vi2SwHO8= -github.com/minio/minio-go/v7 v7.1.0/go.mod h1:Dm7WS1AgLmBa0NcQD6SeJnJf+K/EUW3GR7Ks6olB3OA= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/minio/minio-go/v7 v7.2.0 h1:RCJM0R1XOsRs+A3x3UCaf3ZYbByDaLjFeAi+YCQEPhs= +github.com/minio/minio-go/v7 v7.2.0/go.mod h1:EU9hENAStx/xXduNdrGO5e4X5vk19NtgB+RIPjZO8o0= github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= @@ -602,148 +353,87 @@ github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/mxk/go-vss v1.2.1 h1:shspH0qgqZ9l5sfIRsXS5BgZXz25/BY+ZQsW0HlD0fM= github.com/mxk/go-vss v1.2.1/go.mod h1:ZQ4yFxCG54vqPnCd+p2IxAe5jwZdz56wSjbwzBXiFd8= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/netresearch/go-cron v0.15.0 h1:pu+dhMZjBao9m5IpYe0o+zcNFlP94Z7TDHrORQk+/t0= +github.com/netresearch/go-cron v0.15.0/go.mod h1:79iktHfV90py3jcaFUtWcGSKbZXRev+WwoLMyV5eMvo= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= -github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.28.3 h1:4JvMdwtFU0imd8fHx25OJXoDMRexnf8v5NHKYSTTji4= github.com/onsi/ginkgo/v2 v2.28.3/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.40.0 h1:Vtol0e1MghCD2ZVIilPDIg44XSL9l2QAn8ZNaljWcJc= github.com/onsi/gomega v1.40.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/project-velero/kopia v0.0.0-20260512025144-908c5c098101 h1:bTzkHkWqMM2Zp942BqDQ3TMrfAKZ6tRTG6vcRBlObps= -github.com/project-velero/kopia v0.0.0-20260512025144-908c5c098101/go.mod h1:VxeLQ3AfxPMOraxoEqzbsOrHPSohTc6CWGs5PgMvtDs= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/project-velero/kopia v0.0.0-20260616052725-d83462d382c9 h1:sz/aL2lfvYvnbEmfbmVgjoT5NeGHSdDXQspuUnwScEU= +github.com/project-velero/kopia v0.0.0-20260616052725-d83462d382c9/go.mod h1:iXcxxMES+wBh3cEDMli2gIO10Rkj3bMRrJGK4bQ7hsg= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= -github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/common v0.68.1 h1:omjRRl4QP4komogpXuhfeOiisQg7xdy8VM1UY+pStaY= +github.com/prometheus/common v0.68.1/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= -github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= -github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tg123/go-htpasswd v1.2.4 h1:HgH8KKCjdmo7jjXWN9k1nefPBd7Be3tFCTjc2jPraPU= github.com/tg123/go-htpasswd v1.2.4/go.mod h1:EKThQok9xHkun6NBMynNv6Jmu24A33XdZzzl4Q7H1+0= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= @@ -756,22 +446,14 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/vladimirvivien/gexe v0.1.1 h1:2A0SBaOSKH+cwLVdt6H+KkHZotZWRNLlWygANGw5DxE= -github.com/vladimirvivien/gexe v0.1.1/go.mod h1:LHQL00w/7gDUKIak24n801ABp8C+ni6eBht9vGVst8w= -github.com/vmware-tanzu/crash-diagnostics v0.3.7 h1:6gbv/3o1FzyRLS7Dz/+yVg1Lk1oRBQLyI3d1YTtlTT8= -github.com/vmware-tanzu/crash-diagnostics v0.3.7/go.mod h1:gO8670rd+qdjnJVol674snT/A46GQ27u085kKhZznlM= +github.com/vladimirvivien/gexe v0.4.0 h1:yk51bQu4HRlkt+MzXGQbSucvg7VIyOU4U0fR+awNN6c= +github.com/vladimirvivien/gexe v0.4.0/go.mod h1:fp7cy60ON1xjhtEI/+bfSEIXX35qgmI+iRYlGOqbBFM= +github.com/vmware-tanzu/crash-diagnostics v0.4.3 h1:bl3JmgTD/64DIwdaiCLJHmj2TIwqzRk1tOFsGXJ7+rw= +github.com/vmware-tanzu/crash-diagnostics v0.4.3/go.mod h1:pIcBvCnsWg4PWrrFsEmNoRj5qvakOTfm7oyOa63zqgc= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zalando/go-keyring v0.2.8 h1:6sD/Ucpl7jNq10rM2pgqTs0sZ9V3qMrqfIIy5YPccHs= github.com/zalando/go-keyring v0.2.8/go.mod h1:tsMo+VpRq5NGyKfxoBVjCuMrG47yj8cmakZDO5QGii0= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= @@ -782,326 +464,88 @@ github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/detectors/gcp v1.39.0 h1:kWRNZMsfBHZ+uHjiH4y7Etn2FK26LAGkNFw7RHv1DhE= -go.opentelemetry.io/contrib/detectors/gcp v1.39.0/go.mod h1:t/OGqzHBa5v6RHZwrDBJ2OirWc+4q/w2fTbLZwAKjTk= +go.opentelemetry.io/contrib/detectors/gcp v1.42.0 h1:kpt2PEJuOuqYkPcktfJqWWDjTEd/FNgrxcniL7kQrXQ= +go.opentelemetry.io/contrib/detectors/gcp v1.42.0/go.mod h1:W9zQ439utxymRrXsUOzZbFX4JhLxXU4+ZnCt8GG7yA8= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0 h1:TC+BewnDpeiAmcscXbGMfxkO+mwYUwE/VySwvw88PfA= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0/go.mod h1:J/ZyF4vfPwsSr9xJSPyQ4LqtcTPULFR64KwTikGLe+A= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= -go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= -go.starlark.net v0.0.0-20201006213952-227f4aabceb5/go.mod h1:f0znQkUKRrkk36XxWbGjMqQM8wGv/xHBVE2qc3B5oFU= -go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY= -go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA= +go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.starlark.net v0.0.0-20241226192728-8dfa5b98479f h1:Zs/py28HDFATSDzPcfIzrBFjVsV7HzDEGNNVZIGsjm0= +go.starlark.net v0.0.0-20241226192728-8dfa5b98479f/go.mod h1:YKMCv9b1WrfWmeqdV5MAuEHWsu5iC+fe6kYl2sQjdI8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= -golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= -golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1112,212 +556,68 @@ gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= -google.golang.org/api v0.277.0 h1:HJfyJUiNeBBUMai7ez8u14wkp/gH/I4wpGbbO9o+cSk= -google.golang.org/api v0.277.0/go.mod h1:B9TqLBwJqVjp1mtt7WeoQwWRwvu/400y5lETOql+giQ= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/api v0.283.0 h1:0lkp8u0MPwJVHqRL+nJlMAoZVVzbmiXmFHXMOTmSPik= +google.golang.org/api v0.283.0/go.mod h1:6Wssta4c5n9qHq5CBhmlai5h/PUa1djdDAIhYEHyvcM= google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= -google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= -google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 h1:tEkOQcXgF6dH1G+MVKZrfpYvozGrzb91k6ha7jireSM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/ini.v1 v1.67.2 h1:JtOSMb9OuaCZKr7h5D/h6iii14sK0hLbplTc6frx4Ss= +gopkg.in/ini.v1 v1.67.2/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8= k8s.io/api v0.36.0 h1:SgqDhZzHdOtMk40xVSvCXkP9ME0H05hPM3p9AB1kL80= k8s.io/api v0.36.0/go.mod h1:m1LVrGPNYax5NBHdO+QuAedXyuzTt4RryI/qnmNvs34= k8s.io/apiextensions-apiserver v0.36.0 h1:Wt7E8J+VBCbj4FjiBfDTK/neXDDjyJVJc7xfuOHImZ0= k8s.io/apiextensions-apiserver v0.36.0/go.mod h1:kGDjH0msuiIB3tgsYRV0kS9GqpMYMUsQ3GHv7TApyug= -k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= k8s.io/apimachinery v0.36.0 h1:jZyPzhd5Z+3h9vJLt0z9XdzW9VzNzWAUw+P1xZ9PXtQ= k8s.io/apimachinery v0.36.0/go.mod h1:FklypaRJt6n5wUIwWXIP6GJlIpUizTgfo1T/As+Tyxc= -k8s.io/cli-runtime v0.22.2/go.mod h1:tkm2YeORFpbgQHEK/igqttvPTRIHFRz5kATlw53zlMI= +k8s.io/apiserver v0.36.0 h1:Jg5OFAENUACByUCg15CmhZAYrr5ZyJ+jodyA1mHl3YE= +k8s.io/apiserver v0.36.0/go.mod h1:mHvwdHf+qKEm+1/hYm756SV+oREOKSPnsjagOpx6Vho= k8s.io/cli-runtime v0.36.0 h1:HNxciQpQMMOKS0/GiUXcKDyA6J2FDILJj9NmP2BZrTg= k8s.io/cli-runtime v0.36.0/go.mod h1:KObkknK9Ro5LYX+1RdiKc7C8CvGg4aX+V/Zv+E8WPHA= -k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U= k8s.io/client-go v0.36.0 h1:pOYi7C4RHChYjMiHpZSpSbIM6ZxVbRXBy7CuiIwqA3c= k8s.io/client-go v0.36.0/go.mod h1:ZKKcpwF0aLYfkHFCjillCKaTK/yBkEDHTDXCFY6AS9Y= k8s.io/component-base v0.36.0 h1:hFjEktssxiJhrK1zfybkH4kJOi8iZuF+mIDCqS5+jRo= k8s.io/component-base v0.36.0/go.mod h1:JZvIfcNHk+uck+8LhJzhSBtydWXaZNQwX2OdL+Mnwsk= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kube-aggregator v0.36.0 h1:yrR+aw41p4/Wur55FCcfozPSBHH70HIs5j+J6ZphExg= k8s.io/kube-aggregator v0.36.0/go.mod h1:2CkdUvPZjEbKnlhn+wxj6z3yity7H4xsTrFX+M/t1UE= -k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= k8s.io/metrics v0.36.0 h1:VF41Mv9ZWKKQ4jEiJ0n3Tp6jdyO+oM6dbKcJn6Y/DVg= k8s.io/metrics v0.36.0/go.mod h1:FY1dgPJZqnSfnOYbVdBEdRNUdy0n1nUCU6yxSMUrVG4= k8s.io/streaming v0.36.0 h1:agnTxU+NFulUrtYzXUGKO3ndEa8jKwht1Kwn9nu9x+4= k8s.io/streaming v0.36.0/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= -k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/controller-runtime v0.24.0 h1:Ck6N2LdS8Lovy1o25BB4r1xjvLEKUl1s2o9kU+KWDE4= sigs.k8s.io/controller-runtime v0.24.0/go.mod h1:vFkfY5fGt5xAC/sKb8IBFKgWPNKG9OUG29dR8Y2wImw= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= -sigs.k8s.io/kustomize/api v0.8.11/go.mod h1:a77Ls36JdfCWojpUqR6m60pdGY1AYFix4AH83nJtY1g= -sigs.k8s.io/kustomize/kyaml v0.11.0/go.mod h1:GNMwjim4Ypgp/MueD3zXHLRJEjz7RvtPae0AwlvEMFM= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/hack/build-image/Dockerfile b/hack/build-image/Dockerfile index eb1d728f5..88dedde95 100644 --- a/hack/build-image/Dockerfile +++ b/hack/build-image/Dockerfile @@ -96,7 +96,9 @@ RUN ARCH=$(go env GOARCH) && \ chmod +x /usr/bin/goreleaser # get golangci-lint -RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.5.0 +# Use "go install" so the download goes through GOPROXY instead of the GitHub +# release API/CDN, which has been returning intermittent/persistent HTTP 504s. +RUN go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.5.0 # install kubectl RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/$(go env GOARCH)/kubectl diff --git a/internal/credentials/file_store.go b/internal/credentials/file_store.go index d1f1fb10a..e21418b56 100644 --- a/internal/credentials/file_store.go +++ b/internal/credentials/file_store.go @@ -21,7 +21,7 @@ import ( "os" "path/filepath" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" kbclient "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/internal/credentials/secret_store.go b/internal/credentials/secret_store.go index f4d2111a5..c03dfe73b 100644 --- a/internal/credentials/secret_store.go +++ b/internal/credentials/secret_store.go @@ -17,7 +17,7 @@ limitations under the License. package credentials import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" kbclient "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/internal/delete/actions/csi/volumesnapshotcontent_action.go b/internal/delete/actions/csi/volumesnapshotcontent_action.go index 9473686e0..c57a0eb1a 100644 --- a/internal/delete/actions/csi/volumesnapshotcontent_action.go +++ b/internal/delete/actions/csi/volumesnapshotcontent_action.go @@ -20,9 +20,9 @@ import ( "context" "time" + "github.com/cockroachdb/errors" "github.com/google/uuid" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/internal/delete/delete_item_action_handler.go b/internal/delete/delete_item_action_handler.go index ba242c0ca..2a16044ee 100644 --- a/internal/delete/delete_item_action_handler.go +++ b/internal/delete/delete_item_action_handler.go @@ -21,10 +21,11 @@ import ( "github.com/vmware-tanzu/velero/pkg/plugin/framework" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" + kubeerrs "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -80,6 +81,15 @@ func InvokeDeleteActions(ctx *Context) error { } processdResources := sets.NewString() + // deleteErrs collects errors returned by DeleteItemAction plugins. We keep + // looping over the remaining items even when a plugin fails, but we must not + // swallow these errors: a DIA failure means the private artifacts it manages + // (e.g. data mover repository snapshots) may not have been deleted. If we + // returned nil here, the caller would proceed to delete the backup and its + // metadata, orphaning those artifacts forever. Returning the aggregated error + // makes the caller fail the deletion so it can be retried. + var deleteErrs []error + for resource := range backupResources { groupResource := schema.ParseGroupResource(resource) @@ -124,15 +134,19 @@ func InvokeDeleteActions(ctx *Context) error { Item: obj, Backup: ctx.Backup, }) - // Since we want to keep looping even on errors, log them instead of just returning. + // Keep looping even on errors so a single failing plugin + // doesn't prevent the remaining items from being cleaned up, + // but record the error so it can be surfaced to the caller. if err != nil { itemLog.WithError(err).Error("plugin error") + deleteErrs = append(deleteErrs, errors.Wrapf(err, + "error executing DeleteItemAction for %s %s", groupResource.String(), obj.GetName())) } } } } } - return nil + return kubeerrs.NewAggregate(deleteErrs) } // getApplicableActions takes resolved DeleteItemActions and filters them for a given group/resource and namespace. diff --git a/internal/delete/delete_item_action_handler_test.go b/internal/delete/delete_item_action_handler_test.go index 6743cd1f9..b7d4e6e6a 100644 --- a/internal/delete/delete_item_action_handler_test.go +++ b/internal/delete/delete_item_action_handler_test.go @@ -17,6 +17,7 @@ limitations under the License. package delete import ( + "errors" "io" "sort" "testing" @@ -276,3 +277,54 @@ func TestInvokeDeleteItemActionsWithNoPlugins(t *testing.T) { err := InvokeDeleteActions(c) require.NoError(t, err) } + +// failingAction is a DeleteItemAction that always returns an error from Execute. +// It is used to verify that InvokeDeleteActions surfaces plugin errors instead +// of swallowing them. +type failingAction struct { + selector velero.ResourceSelector + err error + executed int +} + +func (a *failingAction) AppliesTo() (velero.ResourceSelector, error) { + return a.selector, nil +} + +func (a *failingAction) Execute(input *velero.DeleteItemActionExecuteInput) error { + a.executed++ + return a.err +} + +func TestInvokeDeleteActionsReturnsPluginErrors(t *testing.T) { + fs := test.NewFakeFileSystem() + log := logrus.StandardLogger() + + tarball := test.NewTarWriter(t). + AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result()). + Done() + + action := &failingAction{err: errors.New("could not delete artifact")} + + h := newHarness(t) + h.addResource(t, test.Pods()) + + c := &Context{ + Backup: builder.ForBackup("velero", "velero").Result(), + BackupReader: tarball, + Filesystem: fs, + DiscoveryHelper: h.discoveryHelper, + Actions: []velero.DeleteItemAction{action}, + Log: log, + } + + err := InvokeDeleteActions(c) + + // The plugin error must be surfaced so the caller can fail the deletion + // rather than orphaning the artifacts the plugin failed to delete. + require.Error(t, err) + assert.Contains(t, err.Error(), "could not delete artifact") + // The loop must keep going: the action should run for every matching item, + // not stop at the first failure. + assert.Equal(t, 2, action.executed) +} diff --git a/internal/hook/item_hook_handler.go b/internal/hook/item_hook_handler.go index 52dd815ab..bed48c5ea 100644 --- a/internal/hook/item_hook_handler.go +++ b/internal/hook/item_hook_handler.go @@ -23,8 +23,8 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" "github.com/google/uuid" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" diff --git a/internal/hook/item_hook_handler_test.go b/internal/hook/item_hook_handler_test.go index 37f1500a3..1f2df9469 100644 --- a/internal/hook/item_hook_handler_test.go +++ b/internal/hook/item_hook_handler_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" diff --git a/internal/hook/wait_exec_hook_handler_test.go b/internal/hook/wait_exec_hook_handler_test.go index fb102b16f..0bcfafb61 100644 --- a/internal/hook/wait_exec_hook_handler_test.go +++ b/internal/hook/wait_exec_hook_handler_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -53,13 +53,25 @@ func TestWaitExecHandleHooks(t *testing.T) { // delta to wait since last change applied or pod added wait time.Duration updated *corev1api.Pod + // waitForSignal, if set, blocks the goroutine after applying this change + // until the channel is closed. Use this to ensure the handler processes + // an intermediate pod state before the next change is applied. + waitForSignal chan struct{} } type expectedExecution struct { hook *velerov1api.ExecHook name string error error pod *corev1api.Pod + // onCalled, if set, is invoked by the mock when ExecutePodCommand is called. + // Use this together with change.waitForSignal to synchronize state transitions. + onCalled func() } + // hookFired is used by the test case that has two containers with hooks in + // different containers. It ensures the second pod state change is only sent + // after the first hook has fired, preventing the informer from coalescing + // both updates and skipping the intermediate state. + hookFired := make(chan struct{}) tests := []struct { name string // Used as argument to HandleHooks and first state added to ListerWatcher @@ -622,6 +634,8 @@ func TestWaitExecHandleHooks(t *testing.T) { }, }). Result(), + // Signal after this hook fires so the goroutine can apply the next change. + onCalled: func() { close(hookFired) }, }, { name: "my-hook-1", @@ -678,6 +692,10 @@ func TestWaitExecHandleHooks(t *testing.T) { }, }). Result(), + // Block until the hook for container1 has fired before sending the + // next change. Without this, the informer may coalesce both updates + // and deliver only resourceVersion:3, skipping the intermediate state. + waitForSignal: hookFired, }, // 2nd modification: container2 starts running, resourceVersion 3 { @@ -838,11 +856,15 @@ func TestWaitExecHandleHooks(t *testing.T) { go func() { // This is the state of the pod that will be seen by the AddFunc handler. source.Add(test.initialPod) - // Changes holds the versions of the pod over time. Each of these states - // will be seen by the UpdateFunc handler. + // Changes holds the versions of the pod over time. The informer may + // coalesce rapid updates, so use waitForSignal when a test requires the + // handler to observe a specific intermediate state before the next change. for _, change := range test.changes { time.Sleep(change.wait) source.Modify(change.updated) + if change.waitForSignal != nil { + <-change.waitForSignal + } } }() @@ -857,7 +879,11 @@ func TestWaitExecHandleHooks(t *testing.T) { for _, e := range test.expectedExecutions { obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(e.pod) require.NoError(t, err) - podCommandExecutor.On("ExecutePodCommand", mock.Anything, obj, e.pod.Namespace, e.pod.Name, e.name, e.hook).Return(e.error) + call := podCommandExecutor.On("ExecutePodCommand", mock.Anything, obj, e.pod.Namespace, e.pod.Name, e.name, e.hook).Return(e.error) + if e.onCalled != nil { + onCalled := e.onCalled + call.Run(func(mock.Arguments) { onCalled() }) + } } ctx := t.Context() diff --git a/internal/resourcemodifiers/resource_modifiers.go b/internal/resourcemodifiers/resource_modifiers.go index cc780df03..e04510804 100644 --- a/internal/resourcemodifiers/resource_modifiers.go +++ b/internal/resourcemodifiers/resource_modifiers.go @@ -19,9 +19,9 @@ import ( "fmt" "regexp" + "github.com/cockroachdb/errors" jsonpatch "github.com/evanphx/json-patch/v5" "github.com/gobwas/glob" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/internal/resourcepolicies/resource_policies.go b/internal/resourcepolicies/resource_policies.go index 232633484..235f48ed5 100644 --- a/internal/resourcepolicies/resource_policies.go +++ b/internal/resourcepolicies/resource_policies.go @@ -23,13 +23,14 @@ import ( "k8s.io/apimachinery/pkg/util/sets" + "github.com/cockroachdb/errors" "github.com/gobwas/glob" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" crclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + datamover "github.com/vmware-tanzu/velero/pkg/util/datamover" "github.com/vmware-tanzu/velero/pkg/util/wildcard" ) @@ -48,6 +49,20 @@ const ( Custom VolumeActionType = "custom" ) +const ( + // DataMoverParameter is the key of the action parameter that selects the data + // mover to be used for the matched volumes when the action type is snapshot. + DataMoverParameter = "dataMover" +) + +// validDataMovers is the set of data mover values accepted in the snapshot +// action's dataMover parameter. +var validDataMovers = map[string]struct{}{ + datamover.DataMoverTypeVelero: {}, + datamover.DataMoverTypeVeleroFs: {}, + datamover.DataMoverTypeVeleroBlock: {}, +} + // Action defined as one action for a specific way of backup type Action struct { // Type defined specific type of action, currently only support 'skip' @@ -56,6 +71,35 @@ type Action struct { Parameters map[string]any `yaml:"parameters,omitempty"` } +// GetDataMover returns the data mover configured in the snapshot action's +// dataMover parameter. The dataMover parameter is only meaningful for the +// snapshot action, so it returns an error when the action is nil or its type is +// not snapshot. When the parameter is absent, it returns the default built-in +// data mover. The empty string and "velero" both denote the default built-in +// data mover and are returned unchanged; normalizing them to the concrete +// default mover is the consuming workflow's responsibility (issue #9830). +func (a *Action) GetDataMover() (string, error) { + if a == nil || a.Type != Snapshot { + return "", fmt.Errorf("the %q parameter is only supported for the %q action", DataMoverParameter, Snapshot) + } + if len(a.Parameters) == 0 { + return datamover.GetDefaultBuiltInDataMover(), nil + } + raw, ok := a.Parameters[DataMoverParameter] + if !ok { + return datamover.GetDefaultBuiltInDataMover(), nil + } + dataMover, ok := raw.(string) + if !ok { + return "", fmt.Errorf("parameter %q must be a string, got %T", DataMoverParameter, raw) + } + if _, ok := validDataMovers[dataMover]; !ok { + return "", fmt.Errorf("invalid %q value %q, valid values are %q, %q, %q", + DataMoverParameter, dataMover, datamover.DataMoverTypeVelero, datamover.DataMoverTypeVeleroFs, datamover.DataMoverTypeVeleroBlock) + } + return dataMover, nil +} + // ResourceFilter defines a filter for specific resource kinds. type ResourceFilter struct { Kinds []string `yaml:"kinds"` @@ -155,10 +199,35 @@ func unmarshalResourcePolicies(yamlData *string) (*ResourcePolicies, error) { return nil, fmt.Errorf("pvcLabels must be a map of string to string, got %T", raw) } } + if raw, ok := vp.Conditions["pvcVolumeMode"]; ok { + if _, ok := raw.(string); !ok { + return nil, fmt.Errorf("pvcVolumeMode must be a string, got %T", raw) + } + } + if raw, ok := vp.Conditions["pvcAccessModes"]; ok { + if err := validateStringSliceCondition("pvcAccessModes", raw); err != nil { + return nil, err + } + } } return resPolicies, nil } +func validateStringSliceCondition(name string, raw any) error { + switch values := raw.(type) { + case []any: + for _, value := range values { + if _, ok := value.(string); !ok { + return fmt.Errorf("%s must be a list of strings, got element %T", name, value) + } + } + case []string: + default: + return fmt.Errorf("%s must be a list of strings, got %T", name, raw) + } + return nil +} + func (p *Policies) BuildPolicy(resPolicies *ResourcePolicies) error { for _, vp := range resPolicies.VolumePolicies { con, err := unmarshalVolConditions(vp.Conditions) @@ -182,6 +251,12 @@ func (p *Policies) BuildPolicy(resPolicies *ResourcePolicies) error { if len(con.PVCPhase) > 0 { volP.conditions = append(volP.conditions, &pvcPhaseCondition{phases: con.PVCPhase}) } + if con.PVCVolumeMode != "" { + volP.conditions = append(volP.conditions, &pvcVolumeModeCondition{volumeMode: con.PVCVolumeMode}) + } + if len(con.PVCAccessModes) > 0 { + volP.conditions = append(volP.conditions, &pvcAccessModesCondition{accessModes: con.PVCAccessModes}) + } p.volumePolicies = append(p.volumePolicies, volP) } @@ -261,6 +336,34 @@ func (p *Policies) Validate() error { } } + if err := p.validateClusterScopedFilterPolicy(); err != nil { + return errors.WithStack(err) + } + + if err := p.validateNamespacedFilterPolicies(); err != nil { + return errors.WithStack(err) + } + + return nil +} + +func (p *Policies) ValidateForRestore() error { + if p.version != currentSupportDataVersion { + return fmt.Errorf("incompatible version number %s with supported version %s", p.version, currentSupportDataVersion) + } + + if len(p.volumePolicies) > 0 { + return fmt.Errorf("volumePolicies are not supported for restore") + } + + if p.GetIncludeExcludePolicy() != nil { + return fmt.Errorf("includeExcludePolicy is not supported for restore") + } + + if err := p.validateClusterScopedFilterPolicy(); err != nil { + return errors.WithStack(err) + } + if err := p.validateNamespacedFilterPolicies(); err != nil { return errors.WithStack(err) } @@ -296,26 +399,143 @@ func GetResourcePoliciesFromBackup( if err != nil { logger.Errorf("Fail to get ResourcePolicies %s ConfigMap with error %s.", backup.Namespace+"/"+backup.Spec.ResourcePolicy.Name, err.Error()) - return nil, fmt.Errorf("fail to get ResourcePolicies %s ConfigMap with error %s", - backup.Namespace+"/"+backup.Spec.ResourcePolicy.Name, err.Error()) + return nil, fmt.Errorf("fail to get ResourcePolicies %s ConfigMap: %w", + backup.Namespace+"/"+backup.Spec.ResourcePolicy.Name, err) } resourcePolicies, err = getResourcePoliciesFromConfig(policiesConfigMap) if err != nil { logger.Errorf("Fail to read ResourcePolicies from ConfigMap %s with error %s.", backup.Namespace+"/"+backup.Name, err.Error()) - return nil, fmt.Errorf("fail to read the ResourcePolicies from ConfigMap %s with error %s", - backup.Namespace+"/"+backup.Name, err.Error()) + return nil, fmt.Errorf("fail to read the ResourcePolicies from ConfigMap %s: %w", + backup.Namespace+"/"+backup.Name, err) } else if err = resourcePolicies.Validate(); err != nil { logger.Errorf("Fail to validate ResourcePolicies in ConfigMap %s with error %s.", backup.Namespace+"/"+backup.Name, err.Error()) - return nil, fmt.Errorf("fail to validate ResourcePolicies in ConfigMap %s with error %s", - backup.Namespace+"/"+backup.Name, err.Error()) + return nil, fmt.Errorf("fail to validate ResourcePolicies in ConfigMap %s: %w", + backup.Namespace+"/"+backup.Name, err) } } return resourcePolicies, nil } +// GetGlobalResourcePolicies loads and validates the cluster-wide global backup volume +// policies from a ConfigMap in the Velero install namespace. Only the volumePolicies +// section is honored globally; any include/exclude or fine-grained filter policies are +// ignored (a warning is logged), as those are tied to a specific backup use case. +func GetGlobalResourcePolicies( + client crclient.Client, + namespace string, + configMapName string, + logger logrus.FieldLogger, +) (*Policies, error) { + cm := &corev1api.ConfigMap{} + if err := client.Get(context.Background(), crclient.ObjectKey{Namespace: namespace, Name: configMapName}, cm); err != nil { + return nil, fmt.Errorf("fail to get global backup volume policies ConfigMap %s/%s: %w", namespace, configMapName, err) + } + + policies, err := getResourcePoliciesFromConfig(cm) + if err != nil { + return nil, fmt.Errorf("fail to read global backup volume policies from ConfigMap %s/%s: %w", namespace, configMapName, err) + } + if err := policies.Validate(); err != nil { + return nil, fmt.Errorf("fail to validate global backup volume policies in ConfigMap %s/%s: %w", namespace, configMapName, err) + } + + // Only volumePolicies apply globally; warn about any other filter policies that will be ignored. + if policies.includeExcludePolicy != nil || + policies.clusterScopedFilterPolicy != nil || + len(policies.namespacedFilterPolicies) > 0 { + logger.Warnf("Global backup volume policies ConfigMap %s/%s contains include/exclude or fine-grained "+ + "filter policies; these are ignored, only volumePolicies apply globally.", namespace, configMapName) + } + + // Return a fresh Policies carrying only the globally-applicable fields. Using an allowlist here + // (rather than nil-ing out the ignored fields) means any filter field added to Policies in the + // future is excluded from the global policies by default, without needing to update this code. + return &Policies{ + version: policies.version, + volumePolicies: policies.volumePolicies, + }, nil +} + +// GetResourcePoliciesFromBackupWithGlobal builds the effective resource policies for a backup +// by merging the backup-referenced resource policies with the global backup volume policies +// (when globalConfigMapName is set). The merged volumePolicies list is the backup-level +// policies followed by the global ones, so the first match wins and a backup can override the +// global baseline for a specific volume while still inheriting the rest of the global rules. +func GetResourcePoliciesFromBackupWithGlobal( + backup velerov1api.Backup, + client crclient.Client, + globalConfigMapName string, + installNamespace string, + logger logrus.FieldLogger, +) (*Policies, error) { + backupPolicies, err := GetResourcePoliciesFromBackup(backup, client, logger) + if err != nil { + return nil, err + } + + if globalConfigMapName == "" { + return backupPolicies, nil + } + + globalPolicies, err := GetGlobalResourcePolicies(client, installNamespace, globalConfigMapName, logger) + if err != nil { + return nil, err + } + + if backupPolicies == nil { + return globalPolicies, nil + } + // Backup-level policies first, then global, so backups can override the global baseline. + backupPolicies.volumePolicies = append(backupPolicies.volumePolicies, globalPolicies.volumePolicies...) + return backupPolicies, nil +} + +// GetResourcePoliciesFromRestore retrieves the resource policies from the ConfigMap referenced in the Restore spec. +func GetResourcePoliciesFromRestore( + ctx context.Context, + restore *velerov1api.Restore, + client crclient.Client, + logger logrus.FieldLogger, +) (resourcePolicies *Policies, err error) { + if restore.Spec.ResourcePolicy != nil { + if !strings.EqualFold(restore.Spec.ResourcePolicy.Kind, ConfigmapRefType) { + return nil, fmt.Errorf("invalid ResourcePolicy kind %q, only %q is supported", + restore.Spec.ResourcePolicy.Kind, ConfigmapRefType) + } + policiesConfigMap := &corev1api.ConfigMap{} + err = client.Get( + ctx, + crclient.ObjectKey{ + Namespace: restore.Namespace, + Name: restore.Spec.ResourcePolicy.Name, + }, + policiesConfigMap, + ) + if err != nil { + logger.Errorf("Fail to get ResourcePolicies %s ConfigMap with error %s.", + restore.Namespace+"/"+restore.Spec.ResourcePolicy.Name, err.Error()) + return nil, fmt.Errorf("fail to get ResourcePolicies %s ConfigMap: %w", + restore.Namespace+"/"+restore.Spec.ResourcePolicy.Name, err) + } + resourcePolicies, err = getResourcePoliciesFromConfig(policiesConfigMap) + if err != nil { + logger.Errorf("Fail to read ResourcePolicies from ConfigMap %s with error %s.", + restore.Namespace+"/"+restore.Spec.ResourcePolicy.Name, err.Error()) + return nil, fmt.Errorf("fail to read the ResourcePolicies from ConfigMap %s: %w", + restore.Namespace+"/"+restore.Spec.ResourcePolicy.Name, err) + } else if err = resourcePolicies.ValidateForRestore(); err != nil { + logger.Errorf("Fail to validate ResourcePolicies in ConfigMap %s with error %s.", + restore.Namespace+"/"+restore.Spec.ResourcePolicy.Name, err.Error()) + return nil, fmt.Errorf("fail to validate ResourcePolicies in ConfigMap %s: %w", + restore.Namespace+"/"+restore.Spec.ResourcePolicy.Name, err) + } + } + return resourcePolicies, nil +} + func getResourcePoliciesFromConfig(cm *corev1api.ConfigMap) (*Policies, error) { if cm == nil { return nil, fmt.Errorf("could not parse config from nil configmap") @@ -414,3 +634,44 @@ func (p *Policies) validateNamespacedFilterPolicies() error { return nil } + +func (p *Policies) validateClusterScopedFilterPolicy() error { + if p.clusterScopedFilterPolicy == nil { + return nil + } + + if len(p.clusterScopedFilterPolicy.ResourceFilters) == 0 { + return fmt.Errorf("clusterScopedFilterPolicy: resourceFilters cannot be empty; remove the policy block entirely if it is not needed") + } + + seenKinds := make(map[string]int) + for j, rf := range p.clusterScopedFilterPolicy.ResourceFilters { + if rf.IsCatchAll() { + return fmt.Errorf("clusterScopedFilterPolicy.resourceFilters[%d]: kinds must be specified (catch-all is not supported)", j) + } + + for _, kind := range rf.Kinds { + if prevJ, ok := seenKinds[kind]; ok { + return fmt.Errorf("clusterScopedFilterPolicy: kind %q appears in both resourceFilters[%d] and resourceFilters[%d]", kind, prevJ, j) + } + seenKinds[kind] = j + } + + if len(rf.LabelSelector) > 0 && len(rf.OrLabelSelectors) > 0 { + return fmt.Errorf("clusterScopedFilterPolicy.resourceFilters[%d]: labelSelector and orLabelSelectors cannot co-exist", j) + } + + for k, pattern := range rf.Names { + if _, err := glob.Compile(pattern); err != nil { + return fmt.Errorf("clusterScopedFilterPolicy.resourceFilters[%d].names[%d]: invalid glob pattern %q: %v", j, k, pattern, err) + } + } + for k, pattern := range rf.ExcludedNames { + if _, err := glob.Compile(pattern); err != nil { + return fmt.Errorf("clusterScopedFilterPolicy.resourceFilters[%d].excludedNames[%d]: invalid glob pattern %q: %v", j, k, pattern, err) + } + } + } + + return nil +} diff --git a/internal/resourcepolicies/resource_policies_test.go b/internal/resourcepolicies/resource_policies_test.go index 898c6d1ca..445b479f0 100644 --- a/internal/resourcepolicies/resource_policies_test.go +++ b/internal/resourcepolicies/resource_policies_test.go @@ -16,15 +16,26 @@ limitations under the License. package resourcepolicies import ( + "context" "testing" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerotest "github.com/vmware-tanzu/velero/pkg/test" ) +func pvcVolumeMode(mode corev1api.PersistentVolumeMode) *corev1api.PersistentVolumeMode { + return &mode +} + func TestLoadResourcePolicies(t *testing.T) { testCases := []struct { name string @@ -158,6 +169,64 @@ volumePolicies: `, wantErr: false, }, + { + name: "supported format pvcVolumeMode", + yamlData: `version: v1 +volumePolicies: + - conditions: + pvcVolumeMode: Block + action: + type: skip +`, + wantErr: false, + }, + { + name: "error format of pvcVolumeMode (not a string)", + yamlData: `version: v1 +volumePolicies: + - conditions: + pvcVolumeMode: + - Block + action: + type: skip +`, + wantErr: true, + }, + { + name: "supported format pvcAccessModes", + yamlData: `version: v1 +volumePolicies: + - conditions: + pvcAccessModes: + - ReadWriteOnce + action: + type: skip +`, + wantErr: false, + }, + { + name: "error format of pvcAccessModes (not a list)", + yamlData: `version: v1 +volumePolicies: + - conditions: + pvcAccessModes: ReadWriteOnce + action: + type: skip +`, + wantErr: true, + }, + { + name: "error format of pvcAccessModes (list with non-string)", + yamlData: `version: v1 +volumePolicies: + - conditions: + pvcAccessModes: + - 123 + action: + type: skip +`, + wantErr: true, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -353,14 +422,20 @@ func TestGetResourceMatchedAction(t *testing.T) { } func TestGetResourcePoliciesFromConfig(t *testing.T) { - // Create a test ConfigMap - cm := &corev1api.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-configmap", - Namespace: "test-namespace", - }, - Data: map[string]string{ - "test-data": `version: v1 + testCases := []struct { + name string + cm *corev1api.ConfigMap + expectedErr string + }{ + { + name: "valid configmap", + cm: &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "test-data": `version: v1 volumePolicies: - conditions: capacity: '0,10Gi' @@ -381,63 +456,457 @@ volumePolicies: action: type: skip `, + }, + }, + expectedErr: "", + }, + { + name: "nil configmap", + cm: nil, + expectedErr: "could not parse config from nil configmap", + }, + { + name: "empty data configmap", + cm: &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{}, + }, + expectedErr: "illegal resource policies test-namespace/test-configmap configmap", + }, + { + name: "multiple data configmap", + cm: &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "data1": "value1", + "data2": "value2", + }, + }, + expectedErr: "illegal resource policies test-namespace/test-configmap configmap", + }, + { + name: "invalid yaml data", + cm: &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "test-data": `version: v1 +volumePolicies: + - conditions: + capacity: '0,10Gi' + csi: + driver: disks.csi.driver + action: + type: skip + invalid-key: value +`, + }, + }, + expectedErr: "failed to decode yaml data into resource policies", + }, + { + name: "build policy error", + cm: &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "test-data": `version: v1 +volumePolicies: + - conditions: + capacity: 'invalid-capacity' + csi: + driver: disks.csi.driver + action: + type: skip +`, + }, + }, + expectedErr: "wrong format of Capacity invalid-capacity", }, } - // Call the function and check for errors - resPolicies, err := getResourcePoliciesFromConfig(cm) - require.NoError(t, err) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + resPolicies, err := getResourcePoliciesFromConfig(tc.cm) + if tc.expectedErr == "" { + require.NoError(t, err) + assert.Equal(t, "v1", resPolicies.version) + assert.Len(t, resPolicies.volumePolicies, 3) + } else { + require.ErrorContains(t, err, tc.expectedErr) + assert.Nil(t, resPolicies) + } + }) + } +} - // Check that the returned resourcePolicies object contains the expected data - assert.Equal(t, "v1", resPolicies.version) - - assert.Len(t, resPolicies.volumePolicies, 3) - - policies := ResourcePolicies{ - Version: "v1", - VolumePolicies: []VolumePolicy{ - { - Conditions: map[string]any{ - "capacity": "0,10Gi", - "csi": map[string]any{ - "driver": "disks.csi.driver", - }, - }, - Action: Action{ - Type: Skip, - }, - }, - { - Conditions: map[string]any{ - "csi": map[string]any{ - "driver": "files.csi.driver", - "volumeAttributes": map[string]string{"protocol": "nfs"}, - }, - }, - Action: Action{ - Type: Skip, - }, - }, - { - Conditions: map[string]any{ - "pvcLabels": map[string]string{ - "environment": "production", - }, - }, - Action: Action{ - Type: Skip, - }, - }, +func TestGetResourcePoliciesFromBackup(t *testing.T) { + validCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "test-data": `version: v1 +volumePolicies: + - conditions: + capacity: '0,10Gi' + csi: + driver: disks.csi.driver + action: + type: skip +`, }, } - p := &Policies{} - err = p.BuildPolicy(&policies) - if err != nil { - t.Fatalf("failed to build policy: %v", err) + invalidActionCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "invalid-action-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "test-data": `version: v1 +volumePolicies: + - conditions: + capacity: '0,10Gi' + csi: + driver: disks.csi.driver + action: + type: invalid-action +`, + }, } - assert.Equal(t, p, resPolicies) + invalidVersionCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "invalid-version-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "test-data": `version: v2 +volumePolicies: + - conditions: + capacity: '0,10Gi' + csi: + driver: disks.csi.driver + action: + type: skip +`, + }, + } + + emptyCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "empty-configmap", + Namespace: "test-namespace", + }, + } + + client := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(validCM, invalidActionCM, invalidVersionCM, emptyCM).Build() + logger := logrus.New() + + testCases := []struct { + name string + backup velerov1api.Backup + expectedErr string + }{ + { + name: "valid configmap", + backup: velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-backup", + }, + Spec: velerov1api.BackupSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: ConfigmapRefType, + Name: "test-configmap", + }, + }, + }, + expectedErr: "", + }, + { + name: "invalid kind", + backup: velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-backup", + }, + Spec: velerov1api.BackupSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: "Secret", + Name: "test-configmap", + }, + }, + }, + expectedErr: "", + }, + { + name: "configmap not found", + backup: velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-backup", + }, + Spec: velerov1api.BackupSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: ConfigmapRefType, + Name: "non-existent-configmap", + }, + }, + }, + expectedErr: "fail to get ResourcePolicies test-namespace/non-existent-configmap ConfigMap", + }, + { + name: "invalid action configmap", + backup: velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-backup", + }, + Spec: velerov1api.BackupSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: ConfigmapRefType, + Name: "invalid-action-configmap", + }, + }, + }, + expectedErr: "fail to validate ResourcePolicies in ConfigMap test-namespace/test-backup", + }, + { + name: "invalid version configmap", + backup: velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-backup", + }, + Spec: velerov1api.BackupSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: ConfigmapRefType, + Name: "invalid-version-configmap", + }, + }, + }, + expectedErr: "fail to validate ResourcePolicies in ConfigMap test-namespace/test-backup", + }, + { + name: "empty configmap", + backup: velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-backup", + }, + Spec: velerov1api.BackupSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: ConfigmapRefType, + Name: "empty-configmap", + }, + }, + }, + expectedErr: "fail to read the ResourcePolicies from ConfigMap test-namespace/test-backup", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + resPolicies, err := GetResourcePoliciesFromBackup(tc.backup, client, logger) + if tc.expectedErr == "" { + require.NoError(t, err) + if tc.backup.Spec.ResourcePolicy != nil && tc.backup.Spec.ResourcePolicy.Kind == ConfigmapRefType { + assert.NotNil(t, resPolicies) + } else { + assert.Nil(t, resPolicies) + } + } else { + require.ErrorContains(t, err, tc.expectedErr) + assert.Nil(t, resPolicies) + } + }) + } +} + +func TestGetResourcePoliciesFromRestore(t *testing.T) { + validCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "test-data": `version: v1 +namespacedFilterPolicies: + - namespaces: ["default"] + resourceFilters: + - kinds: ["Pod"] +`, + }, + } + + invalidNfpCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "invalid-action-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "test-data": `version: v1 +namespacedFilterPolicies: + - namespaces: [] + resourceFilters: + - kinds: ["Pod"] +`, + }, + } + + invalidVersionCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "invalid-version-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "test-data": `version: v2 +namespacedFilterPolicies: + - namespaces: ["default"] + resourceFilters: + - kinds: ["Pod"] +`, + }, + } + + emptyCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "empty-configmap", + Namespace: "test-namespace", + }, + } + + client := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(validCM, invalidNfpCM, invalidVersionCM, emptyCM).Build() + logger := logrus.New() + + testCases := []struct { + name string + restore *velerov1api.Restore + expectedErr string + }{ + { + name: "valid configmap", + restore: &velerov1api.Restore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-restore", + }, + Spec: velerov1api.RestoreSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: ConfigmapRefType, + Name: "test-configmap", + }, + }, + }, + expectedErr: "", + }, + { + name: "invalid kind", + restore: &velerov1api.Restore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-restore", + }, + Spec: velerov1api.RestoreSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: "Secret", + Name: "test-configmap", + }, + }, + }, + expectedErr: "invalid ResourcePolicy kind \"Secret\", only \"configmap\" is supported", + }, + { + name: "configmap not found", + restore: &velerov1api.Restore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-restore", + }, + Spec: velerov1api.RestoreSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: ConfigmapRefType, + Name: "non-existent-configmap", + }, + }, + }, + expectedErr: "fail to get ResourcePolicies test-namespace/non-existent-configmap ConfigMap", + }, + { + name: "invalid action configmap", + restore: &velerov1api.Restore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-restore", + }, + Spec: velerov1api.RestoreSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: ConfigmapRefType, + Name: "invalid-action-configmap", + }, + }, + }, + expectedErr: "fail to validate ResourcePolicies in ConfigMap test-namespace/invalid-action-configmap", + }, + { + name: "invalid version configmap", + restore: &velerov1api.Restore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-restore", + }, + Spec: velerov1api.RestoreSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: ConfigmapRefType, + Name: "invalid-version-configmap", + }, + }, + }, + expectedErr: "fail to validate ResourcePolicies in ConfigMap test-namespace/invalid-version-configmap", + }, + { + name: "empty configmap", + restore: &velerov1api.Restore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-restore", + }, + Spec: velerov1api.RestoreSpec{ + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: ConfigmapRefType, + Name: "empty-configmap", + }, + }, + }, + expectedErr: "fail to read the ResourcePolicies from ConfigMap test-namespace/empty-configmap", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + resPolicies, err := GetResourcePoliciesFromRestore(context.Background(), tc.restore, client, logger) + if tc.expectedErr == "" { + require.NoError(t, err) + assert.NotNil(t, resPolicies) + } else { + require.ErrorContains(t, err, tc.expectedErr) + assert.Nil(t, resPolicies) + } + }) + } } func TestGetMatchAction(t *testing.T) { @@ -1046,6 +1515,271 @@ volumePolicies: }, skip: true, }, + { + name: "PVC volume mode matching - Block volume mode should skip", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: Block + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-block", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeMode: pvcVolumeMode(corev1api.PersistentVolumeBlock), + }, + }, + skip: true, + }, + { + name: "PVC volume mode matching - Filesystem volume mode should not skip", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: Block + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-filesystem", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeMode: pvcVolumeMode(corev1api.PersistentVolumeFilesystem), + }, + }, + skip: false, + }, + { + name: "PVC volume mode matching - nil volume mode should not match Filesystem", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: Filesystem + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-without-volume-mode", + }, + }, + skip: false, + }, + { + name: "PVC volume mode matching - unknown condition value should not match empty volume mode", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: foo + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-without-volume-mode", + }, + }, + skip: false, + }, + { + name: "PVC volume mode matching - omitted condition should not restrict volume mode", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcAccessModes: ["ReadWriteOnce"] + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-block-rwo", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeMode: pvcVolumeMode(corev1api.PersistentVolumeBlock), + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOnce}, + }, + }, + skip: true, + }, + { + name: "PVC volume mode matching - non-PVC volume should not match", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: Filesystem + action: + type: skip`, + vol: nil, + podVol: &corev1api.Volume{ + Name: "empty-dir-volume", + VolumeSource: corev1api.VolumeSource{ + EmptyDir: &corev1api.EmptyDirVolumeSource{}, + }, + }, + pvc: nil, + skip: false, + }, + { + name: "PVC access modes matching - non-PVC volume should not match", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcAccessModes: ["ReadWriteOnce"] + action: + type: skip`, + vol: nil, + podVol: &corev1api.Volume{ + Name: "configmap-volume", + VolumeSource: corev1api.VolumeSource{ + ConfigMap: &corev1api.ConfigMapVolumeSource{}, + }, + }, + pvc: nil, + skip: false, + }, + + { + name: "PVC access modes matching - ReadWriteOnce should skip", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcAccessModes: ["ReadWriteOnce"] + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-rwo", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOnce}, + }, + }, + skip: true, + }, + { + name: "PVC access modes matching - extra PVC access mode should not skip", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcAccessModes: ["ReadWriteOnce"] + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-rwo-rom", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOnce, corev1api.ReadOnlyMany}, + }, + }, + skip: false, + }, + { + name: "PVC access modes matching - ReadWriteMany should not skip", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcAccessModes: ["ReadWriteOnce"] + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-rwx", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteMany}, + }, + }, + skip: false, + }, + { + name: "PVC access modes matching - exact access mode set should match regardless of order", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcAccessModes: ["ReadWriteMany", "ReadOnlyMany"] + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-rom-rwx", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadOnlyMany, corev1api.ReadWriteMany}, + }, + }, + skip: true, + }, + { + name: "PVC access modes matching - missing one configured access mode should not skip", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcAccessModes: ["ReadOnlyMany", "ReadWriteMany"] + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-rwx", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteMany}, + }, + }, + skip: false, + }, + { + name: "PVC access modes matching - Combined with volume mode", + yamlData: `version: v1 +volumePolicies: +- conditions: + pvcVolumeMode: Block + pvcAccessModes: ["ReadWriteOnce"] + action: + type: skip`, + vol: nil, + podVol: nil, + pvc: &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "pvc-block-rwo", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeMode: pvcVolumeMode(corev1api.PersistentVolumeBlock), + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOnce}, + }, + }, + skip: true, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -1119,28 +1853,36 @@ func TestGetMatchAction_Errors(t *testing.T) { func TestParsePVC(t *testing.T) { tests := []struct { - name string - pvc *corev1api.PersistentVolumeClaim - expectedLabels map[string]string - expectedPhase string - expectErr bool + name string + pvc *corev1api.PersistentVolumeClaim + expectedLabels map[string]string + expectedPhase string + expectedVolumeMode string + expectedAccessModes []string + expectErr bool }{ { - name: "valid PVC with labels and Pending phase", + name: "valid PVC with labels, Pending phase, Block volume mode, and access modes", pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"env": "prod"}, }, + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeMode: pvcVolumeMode(corev1api.PersistentVolumeBlock), + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOnce, corev1api.ReadOnlyMany}, + }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimPending, }, }, - expectedLabels: map[string]string{"env": "prod"}, - expectedPhase: "Pending", - expectErr: false, + expectedLabels: map[string]string{"env": "prod"}, + expectedPhase: "Pending", + expectedVolumeMode: "Block", + expectedAccessModes: []string{"ReadWriteOnce", "ReadOnlyMany"}, + expectErr: false, }, { - name: "valid PVC with Bound phase", + name: "valid PVC with Bound phase and nil volume mode", pvc: &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{}, @@ -1149,27 +1891,52 @@ func TestParsePVC(t *testing.T) { Phase: corev1api.ClaimBound, }, }, - expectedLabels: nil, - expectedPhase: "Bound", - expectErr: false, + expectedLabels: nil, + expectedPhase: "Bound", + expectedVolumeMode: "", + expectedAccessModes: nil, + expectErr: false, }, { - name: "valid PVC with Lost phase", + name: "valid PVC with Lost phase and Filesystem volume mode", pvc: &corev1api.PersistentVolumeClaim{ + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeMode: pvcVolumeMode(corev1api.PersistentVolumeFilesystem), + }, Status: corev1api.PersistentVolumeClaimStatus{ Phase: corev1api.ClaimLost, }, }, - expectedLabels: nil, - expectedPhase: "Lost", - expectErr: false, + expectedLabels: nil, + expectedPhase: "Lost", + expectedVolumeMode: "Filesystem", + expectedAccessModes: nil, + expectErr: false, }, { - name: "nil PVC pointer", - pvc: (*corev1api.PersistentVolumeClaim)(nil), - expectedLabels: nil, - expectedPhase: "", - expectErr: false, + name: "valid PVC with unknown non-nil volume mode", + pvc: &corev1api.PersistentVolumeClaim{ + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeMode: pvcVolumeMode(corev1api.PersistentVolumeMode("foo")), + }, + Status: corev1api.PersistentVolumeClaimStatus{ + Phase: corev1api.ClaimBound, + }, + }, + expectedLabels: nil, + expectedPhase: "Bound", + expectedVolumeMode: "foo", + expectedAccessModes: nil, + expectErr: false, + }, + { + name: "nil PVC pointer", + pvc: (*corev1api.PersistentVolumeClaim)(nil), + expectedLabels: nil, + expectedPhase: "", + expectedVolumeMode: "", + expectedAccessModes: nil, + expectErr: false, }, } @@ -1180,6 +1947,8 @@ func TestParsePVC(t *testing.T) { assert.Equal(t, tc.expectedLabels, s.pvcLabels) assert.Equal(t, tc.expectedPhase, s.pvcPhase) + assert.Equal(t, tc.expectedVolumeMode, s.pvcVolumeMode) + assert.Equal(t, tc.expectedAccessModes, s.pvcAccessModes) }) } } @@ -1434,6 +2203,17 @@ namespacedFilterPolicies: wantErr: true, errMsg: "invalid glob pattern", }, + { + name: "invalid - bad glob pattern in excludedNames", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["test"] + resourceFilters: + - kinds: ["Pod"] + excludedNames: ["[invalid"]`, + wantErr: true, + errMsg: "invalid glob pattern", + }, { name: "invalid - duplicate namespace pattern", yamlData: `version: v1 @@ -1447,6 +2227,16 @@ namespacedFilterPolicies: wantErr: true, errMsg: "duplicate namespace pattern", }, + { + name: "invalid - bad namespace pattern", + yamlData: `version: v1 +namespacedFilterPolicies: +- namespaces: ["prod**uction"] + resourceFilters: + - kinds: ["Pod"]`, + wantErr: true, + errMsg: "wildcard pattern contains consecutive asterisks", + }, } for _, tc := range testCases { @@ -1503,13 +2293,57 @@ namespacedFilterPolicies: assert.Equal(t, map[string]string{"app": "web"}, rf.LabelSelector) } +func TestClusterScopedFilterPoliciesAccessor(t *testing.T) { + yamlData := `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["ClusterRole"] + names: ["my-app-*"]` + + resPolicies, err := unmarshalResourcePolicies(&yamlData) + require.NoError(t, err) + + policies := &Policies{} + err = policies.BuildPolicy(resPolicies) + require.NoError(t, err) + + csfPolicy := policies.GetClusterScopedFilterPolicy() + require.NotNil(t, csfPolicy) + assert.Len(t, csfPolicy.ResourceFilters, 1) + + rf := csfPolicy.ResourceFilters[0] + assert.Equal(t, []string{"ClusterRole"}, rf.Kinds) + assert.Equal(t, []string{"my-app-*"}, rf.Names) +} + +func TestIncludeExcludePolicyAccessor(t *testing.T) { + yamlData := `version: v1 +includeExcludePolicy: + includedClusterScopedResources: + - ClusterRole + excludedClusterScopedResources: + - ClusterRoleBinding` + + resPolicies, err := unmarshalResourcePolicies(&yamlData) + require.NoError(t, err) + + policies := &Policies{} + err = policies.BuildPolicy(resPolicies) + require.NoError(t, err) + + iePolicy := policies.GetIncludeExcludePolicy() + require.NotNil(t, iePolicy) + assert.Equal(t, []string{"ClusterRole"}, iePolicy.IncludedClusterScopedResources) + assert.Equal(t, []string{"ClusterRoleBinding"}, iePolicy.ExcludedClusterScopedResources) +} + func TestFirstMatchSemantics(t *testing.T) { yamlData := `version: v1 namespacedFilterPolicies: - namespaces: ["team-frontend-*", "specific-ns"] resourceFilters: - kinds: ["Pod", "ConfigMap", "Secret"] -- namespaces: ["team-*", "another-pattern"] +- namespaces: ["team-*", "another-pattern"] resourceFilters: - kinds: ["Deployment", "Service"]` @@ -1536,3 +2370,545 @@ namespacedFilterPolicies: assert.Equal(t, []string{"team-*", "another-pattern"}, policy2.Namespaces) assert.Equal(t, []string{"Deployment", "Service"}, policy2.ResourceFilters[0].Kinds) } + +func TestClusterScopedFilterPolicies(t *testing.T) { + testCases := []struct { + name string + yamlData string + wantErr bool + errMsg string + }{ + { + name: "valid - single kind with names", + yamlData: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["ClusterRole"] + names: ["my-app-*"]`, + wantErr: false, + }, + { + name: "valid - multi-kind with labelSelector", + yamlData: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["ClusterRole", "ClusterRoleBinding"] + labelSelector: + app: my-app`, + wantErr: false, + }, + { + name: "valid - orLabelSelectors", + yamlData: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["CustomResourceDefinition"] + orLabelSelectors: + - app: my-app + - app: other-app`, + wantErr: false, + }, + { + name: "valid - excludedNames", + yamlData: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["ClusterRole"] + names: ["my-*"] + excludedNames: ["my-debug-*"]`, + wantErr: false, + }, + { + name: "invalid - empty resourceFilters", + yamlData: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: []`, + wantErr: true, + errMsg: "resourceFilters cannot be empty; remove the policy block entirely if it is not needed", + }, + { + name: "invalid - empty kinds in clusterScopedFilterPolicy", + yamlData: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: [] + names: ["my-app-*"]`, + wantErr: true, + errMsg: "kinds must be specified", + }, + { + name: "invalid - asterisk kinds (explicit catch-all) in clusterScopedFilterPolicy", + yamlData: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["*"] + labelSelector: + app: my-app`, + wantErr: true, + errMsg: "kinds must be specified", + }, + { + name: "invalid - duplicate kinds across entries", + yamlData: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["ClusterRole"] + names: ["my-app-*"] + - kinds: ["ClusterRole"] + labelSelector: + app: other`, + wantErr: true, + errMsg: `kind "ClusterRole" appears in both`, + }, + { + name: "invalid - labelSelector and orLabelSelectors co-exist", + yamlData: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["ClusterRole"] + labelSelector: + app: my-app + orLabelSelectors: + - app: other`, + wantErr: true, + errMsg: "labelSelector and orLabelSelectors cannot co-exist", + }, + { + name: "invalid - bad glob in names", + yamlData: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["ClusterRole"] + names: ["[invalid"]`, + wantErr: true, + errMsg: "invalid glob pattern", + }, + { + name: "invalid - bad glob in excludedNames", + yamlData: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["ClusterRole"] + excludedNames: ["[bad"]`, + wantErr: true, + errMsg: "invalid glob pattern", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + resPolicies, err := unmarshalResourcePolicies(&tc.yamlData) + require.NoError(t, err) + + policies := &Policies{} + err = policies.BuildPolicy(resPolicies) + require.NoError(t, err) + + err = policies.Validate() + if tc.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errMsg) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestPVCVolumeModeMatch(t *testing.T) { + tests := []struct { + name string + condition *pvcVolumeModeCondition + volume *structuredVolume + expectedMatch bool + }{ + { + name: "match Block volume mode", + condition: &pvcVolumeModeCondition{volumeMode: "Block"}, + volume: &structuredVolume{pvcVolumeMode: "Block"}, + expectedMatch: true, + }, + { + name: "match Filesystem volume mode", + condition: &pvcVolumeModeCondition{volumeMode: "Filesystem"}, + volume: &structuredVolume{pvcVolumeMode: "Filesystem"}, + expectedMatch: true, + }, + { + name: "no match for different volume mode", + condition: &pvcVolumeModeCondition{volumeMode: "Block"}, + volume: &structuredVolume{pvcVolumeMode: "Filesystem"}, + expectedMatch: false, + }, + { + name: "case-sensitive no match for lowercase volume mode", + condition: &pvcVolumeModeCondition{volumeMode: "block"}, + volume: &structuredVolume{pvcVolumeMode: "Block"}, + expectedMatch: false, + }, + { + name: "no match for unknown condition value against Filesystem", + condition: &pvcVolumeModeCondition{volumeMode: "foo"}, + volume: &structuredVolume{pvcVolumeMode: "Filesystem"}, + expectedMatch: false, + }, + { + name: "match unknown condition value only when volume has same value", + condition: &pvcVolumeModeCondition{volumeMode: "foo"}, + volume: &structuredVolume{pvcVolumeMode: "foo"}, + expectedMatch: true, + }, + { + name: "no match for empty volume mode", + condition: &pvcVolumeModeCondition{volumeMode: "Block"}, + volume: &structuredVolume{pvcVolumeMode: ""}, + expectedMatch: false, + }, + { + name: "match with empty volume mode condition (always match)", + condition: &pvcVolumeModeCondition{volumeMode: ""}, + volume: &structuredVolume{pvcVolumeMode: "Block"}, + expectedMatch: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := tc.condition.match(tc.volume) + assert.Equal(t, tc.expectedMatch, result) + }) + } +} + +func TestPVCAccessModesMatch(t *testing.T) { + tests := []struct { + name string + condition *pvcAccessModesCondition + volume *structuredVolume + expectedMatch bool + }{ + { + name: "match ReadWriteOnce access mode", + condition: &pvcAccessModesCondition{accessModes: []string{"ReadWriteOnce"}}, + volume: &structuredVolume{pvcAccessModes: []string{"ReadWriteOnce"}}, + expectedMatch: true, + }, + { + name: "match exact multiple access modes", + condition: &pvcAccessModesCondition{accessModes: []string{"ReadWriteOnce", "ReadOnlyMany"}}, + volume: &structuredVolume{pvcAccessModes: []string{"ReadWriteOnce", "ReadOnlyMany"}}, + expectedMatch: true, + }, + { + name: "match exact multiple access modes regardless of order", + condition: &pvcAccessModesCondition{accessModes: []string{"ReadOnlyMany", "ReadWriteOnce"}}, + volume: &structuredVolume{pvcAccessModes: []string{"ReadWriteOnce", "ReadOnlyMany"}}, + expectedMatch: true, + }, + { + name: "no match when one of multiple access modes is missing", + condition: &pvcAccessModesCondition{accessModes: []string{"ReadWriteOnce", "ReadOnlyMany"}}, + volume: &structuredVolume{pvcAccessModes: []string{"ReadOnlyMany"}}, + expectedMatch: false, + }, + { + name: "no match when PVC has extra access modes", + condition: &pvcAccessModesCondition{accessModes: []string{"ReadWriteMany"}}, + volume: &structuredVolume{pvcAccessModes: []string{"ReadWriteOnce", "ReadWriteMany"}}, + expectedMatch: false, + }, + { + name: "no match for different access mode", + condition: &pvcAccessModesCondition{accessModes: []string{"ReadWriteOnce"}}, + volume: &structuredVolume{pvcAccessModes: []string{"ReadWriteMany"}}, + expectedMatch: false, + }, + { + name: "case-sensitive no match for lowercase access mode", + condition: &pvcAccessModesCondition{accessModes: []string{"readwriteonce"}}, + volume: &structuredVolume{pvcAccessModes: []string{"ReadWriteOnce"}}, + expectedMatch: false, + }, + { + name: "no match for empty PVC access modes", + condition: &pvcAccessModesCondition{accessModes: []string{"ReadWriteOnce"}}, + volume: &structuredVolume{pvcAccessModes: []string{}}, + expectedMatch: false, + }, + { + name: "match with empty access modes list (always match)", + condition: &pvcAccessModesCondition{accessModes: []string{}}, + volume: &structuredVolume{pvcAccessModes: []string{"ReadWriteOnce"}}, + expectedMatch: true, + }, + { + name: "match with nil access modes list (always match)", + condition: &pvcAccessModesCondition{accessModes: nil}, + volume: &structuredVolume{pvcAccessModes: []string{"ReadWriteOnce"}}, + expectedMatch: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := tc.condition.match(tc.volume) + assert.Equal(t, tc.expectedMatch, result) + }) + } +} + +// ---- Global backup volume policies ---- + +func globalPolicyConfigMap(name, data string) *corev1api.ConfigMap { + return &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: "velero", Name: name}, + Data: map[string]string{"policies.yaml": data}, + } +} + +func backupWithPolicy(ref string) velerov1api.Backup { + b := velerov1api.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "velero", Name: "backup"}} + if ref != "" { + b.Spec.ResourcePolicy = &corev1api.TypedLocalObjectReference{Kind: ConfigmapRefType, Name: ref} + } + return b +} + +// firstActionFor returns the action type the policies select for a PV with the given storage +// class, or "" when nothing matches. It exercises the compiled match logic so the tests verify +// merge ordering rather than internal field layout. +func firstActionFor(p *Policies, storageClass string) VolumeActionType { + pv := &corev1api.PersistentVolume{Spec: corev1api.PersistentVolumeSpec{StorageClassName: storageClass}} + vol := &structuredVolume{} + vol.parsePV(pv) + if a := p.match(vol); a != nil { + return a.Type + } + return "" +} + +func TestGetResourcePoliciesFromBackupWithGlobal(t *testing.T) { + gp2Skip := `version: v1 +volumePolicies: + - conditions: + storageClass: + - gp2 + action: + type: skip +` + gp2Snapshot := `version: v1 +volumePolicies: + - conditions: + storageClass: + - gp2 + action: + type: snapshot +` + otherFsBackup := `version: v1 +volumePolicies: + - conditions: + storageClass: + - other + action: + type: fs-backup +` + + tests := []struct { + name string + backupCM *corev1api.ConfigMap + globalCMName string + globalCM *corev1api.ConfigMap + backupRef string + expectErr bool + expectedGp2Action VolumeActionType + expectedNumPolicies int + }{ + { + name: "no global, backup only - unchanged behavior", + backupRef: "backup01", + backupCM: globalPolicyConfigMap("backup01", gp2Snapshot), + expectedGp2Action: Snapshot, + expectedNumPolicies: 1, + }, + { + name: "global only, backup has no policy", + globalCMName: "global", + globalCM: globalPolicyConfigMap("global", gp2Skip), + expectedGp2Action: Skip, + expectedNumPolicies: 1, + }, + { + name: "no global configured and no backup policy", + expectedGp2Action: "", + expectedNumPolicies: 0, + }, + { + name: "merge - backup policy overrides global for gp2", + backupRef: "backup01", + backupCM: globalPolicyConfigMap("backup01", gp2Snapshot), + globalCMName: "global", + globalCM: globalPolicyConfigMap("global", gp2Skip), + expectedGp2Action: Snapshot, // backup-level wins (evaluated first) + expectedNumPolicies: 2, + }, + { + name: "merge - backup inherits non-overlapping global rule", + backupRef: "backup01", + backupCM: globalPolicyConfigMap("backup01", otherFsBackup), + globalCMName: "global", + globalCM: globalPolicyConfigMap("global", gp2Skip), + expectedGp2Action: Skip, // only global matches gp2 + expectedNumPolicies: 2, + }, + { + name: "global configmap missing - error", + globalCMName: "global", + expectErr: true, + }, + { + name: "global configmap invalid - error", + globalCMName: "global", + globalCM: globalPolicyConfigMap("global", "not: [valid"), + expectErr: true, + }, + { + // Parses cleanly but fails Policies.Validate() due to the unsupported version. + name: "global configmap fails validation - error", + globalCMName: "global", + globalCM: globalPolicyConfigMap("global", "version: v2\nvolumePolicies: []\n"), + expectErr: true, + }, + { + // Backup references a ResourcePolicy ConfigMap that does not exist, so resolving the + // backup-level policies fails before the global ones are consulted. + name: "backup configmap missing - error", + backupRef: "missing-backup-cm", + globalCMName: "global", + globalCM: globalPolicyConfigMap("global", gp2Skip), + expectErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + client := velerotest.NewFakeControllerRuntimeClient(t) + if tc.backupCM != nil { + require.NoError(t, client.Create(t.Context(), tc.backupCM)) + } + if tc.globalCM != nil { + require.NoError(t, client.Create(t.Context(), tc.globalCM)) + } + + b := backupWithPolicy(tc.backupRef) + + p, err := GetResourcePoliciesFromBackupWithGlobal(b, client, tc.globalCMName, "velero", logrus.New()) + if tc.expectErr { + require.Error(t, err) + return + } + require.NoError(t, err) + + if tc.expectedNumPolicies == 0 { + assert.Nil(t, p) + return + } + require.NotNil(t, p) + assert.Len(t, p.volumePolicies, tc.expectedNumPolicies) + assert.Equal(t, tc.expectedGp2Action, firstActionFor(p, "gp2")) + }) + } +} + +func TestGetGlobalResourcePoliciesIgnoresNonVolumePolicies(t *testing.T) { + data := `version: v1 +volumePolicies: + - conditions: + storageClass: + - gp2 + action: + type: skip +namespacedFilterPolicies: +- namespaces: ["frontend"] + resourceFilters: + - kinds: ["Pod"] +` + client := velerotest.NewFakeControllerRuntimeClient(t) + require.NoError(t, client.Create(t.Context(), globalPolicyConfigMap("global", data))) + + p, err := GetGlobalResourcePolicies(client, "velero", "global", logrus.New()) + require.NoError(t, err) + require.NotNil(t, p) + + // Only volumePolicies are kept; the namespaced filter policy is dropped. + assert.Len(t, p.volumePolicies, 1) + assert.Empty(t, p.GetNamespacedFilterPolicies()) + assert.Nil(t, p.GetIncludeExcludePolicy()) + assert.Nil(t, p.GetClusterScopedFilterPolicy()) +} + +func TestActionGetDataMover(t *testing.T) { + testCases := []struct { + name string + action *Action + expectedMove string + expectErr bool + }{ + { + name: "nil action", + action: nil, + expectErr: true, + }, + { + name: "snapshot action without parameters returns default mover", + action: &Action{Type: Snapshot}, + expectedMove: "velero-fs", + }, + { + name: "snapshot action without dataMover parameter returns default mover", + action: &Action{Type: Snapshot, Parameters: map[string]any{"other": "value"}}, + expectedMove: "velero-fs", + }, + { + name: "snapshot action with velero dataMover", + action: &Action{Type: Snapshot, Parameters: map[string]any{"dataMover": "velero"}}, + expectedMove: "velero", + }, + { + name: "snapshot action with velero-fs dataMover", + action: &Action{Type: Snapshot, Parameters: map[string]any{"dataMover": "velero-fs"}}, + expectedMove: "velero-fs", + }, + { + name: "snapshot action with velero-block dataMover", + action: &Action{Type: Snapshot, Parameters: map[string]any{"dataMover": "velero-block"}}, + expectedMove: "velero-block", + }, + { + name: "non-snapshot action returns error", + action: &Action{Type: FSBackup, Parameters: map[string]any{"dataMover": "velero-fs"}}, + expectErr: true, + }, + { + name: "snapshot action with non-string dataMover returns error", + action: &Action{Type: Snapshot, Parameters: map[string]any{"dataMover": 123}}, + expectErr: true, + }, + { + name: "snapshot action with invalid dataMover returns error", + action: &Action{Type: Snapshot, Parameters: map[string]any{"dataMover": "unknown"}}, + expectErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + dataMover, err := tc.action.GetDataMover() + if tc.expectErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tc.expectedMove, dataMover) + }) + } +} diff --git a/internal/resourcepolicies/volume_resources.go b/internal/resourcepolicies/volume_resources.go index 4ad34f484..29514b9ee 100644 --- a/internal/resourcepolicies/volume_resources.go +++ b/internal/resourcepolicies/volume_resources.go @@ -18,12 +18,14 @@ package resourcepolicies import ( "bytes" "fmt" + "slices" "strings" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" - "github.com/pkg/errors" - "gopkg.in/yaml.v3" + "github.com/cockroachdb/errors" + "go.yaml.in/yaml/v3" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" ) @@ -45,13 +47,15 @@ type capacity struct { } type structuredVolume struct { - capacity resource.Quantity - storageClass string - nfs *nFSVolumeSource - csi *csiVolumeSource - volumeType SupportedVolume - pvcLabels map[string]string - pvcPhase string + capacity resource.Quantity + storageClass string + nfs *nFSVolumeSource + csi *csiVolumeSource + volumeType SupportedVolume + pvcLabels map[string]string + pvcPhase string + pvcVolumeMode string + pvcAccessModes []string } func (s *structuredVolume) parsePV(pv *corev1api.PersistentVolume) { @@ -76,6 +80,15 @@ func (s *structuredVolume) parsePVC(pvc *corev1api.PersistentVolumeClaim) { s.pvcLabels = pvc.Labels } s.pvcPhase = string(pvc.Status.Phase) + if pvc.Spec.VolumeMode != nil { + s.pvcVolumeMode = string(*pvc.Spec.VolumeMode) + } + if len(pvc.Spec.AccessModes) > 0 { + s.pvcAccessModes = make([]string, 0, len(pvc.Spec.AccessModes)) + for _, accessMode := range pvc.Spec.AccessModes { + s.pvcAccessModes = append(s.pvcAccessModes, string(accessMode)) + } + } } } @@ -127,18 +140,55 @@ func (c *pvcPhaseCondition) match(v *structuredVolume) bool { if v.pvcPhase == "" { return false } - for _, phase := range c.phases { - if v.pvcPhase == phase { - return true - } - } - return false + return slices.Contains(c.phases, v.pvcPhase) } func (c *pvcPhaseCondition) validate() error { return nil } +// pvcVolumeModeCondition defines a condition that matches if the PVC's volume mode matches the provided volume mode. +type pvcVolumeModeCondition struct { + volumeMode string +} + +func (c *pvcVolumeModeCondition) match(v *structuredVolume) bool { + // No volume mode specified: always match. + if c.volumeMode == "" { + return true + } + + // Here allows unknown strings for forward compatibility. If Kubernetes adds another volume mode later, + // Velero would not reject the policy just because the string is unfamiliar. + return v.pvcVolumeMode == c.volumeMode +} + +func (c *pvcVolumeModeCondition) validate() error { + return nil +} + +// pvcAccessModesCondition defines a condition that matches if the PVC has exactly the provided access modes. +type pvcAccessModesCondition struct { + accessModes []string +} + +func (c *pvcAccessModesCondition) match(v *structuredVolume) bool { + // No access modes specified: always match. + if len(c.accessModes) == 0 { + return true + } + + if len(v.pvcAccessModes) != len(c.accessModes) { + return false + } + + return sets.New(c.accessModes...).Equal(sets.New(v.pvcAccessModes...)) +} + +func (c *pvcAccessModesCondition) validate() error { + return nil +} + type capacityCondition struct { capacity capacity } diff --git a/internal/resourcepolicies/volume_resources_test.go b/internal/resourcepolicies/volume_resources_test.go index b55692326..02850bb7f 100644 --- a/internal/resourcepolicies/volume_resources_test.go +++ b/internal/resourcepolicies/volume_resources_test.go @@ -430,6 +430,38 @@ func TestUnmarshalVolumeConditions(t *testing.T) { }, expectedError: "!!str `production` into map[string]string", }, + { + name: "Valid pvcVolumeMode input", + input: map[string]any{ + "capacity": "1Gi,10Gi", + "pvcVolumeMode": "Block", + }, + expectedError: "", + }, + { + name: "Invalid pvcVolumeMode input: not a string", + input: map[string]any{ + "capacity": "1Gi,10Gi", + "pvcVolumeMode": []string{"Filesystem", "Block"}, + }, + expectedError: "cannot unmarshal !!seq", + }, + { + name: "Valid pvcAccessModes input", + input: map[string]any{ + "capacity": "1Gi,10Gi", + "pvcAccessModes": []string{"ReadWriteOnce", "ReadWriteMany"}, + }, + expectedError: "", + }, + { + name: "Invalid pvcAccessModes input: not a list", + input: map[string]any{ + "capacity": "1Gi,10Gi", + "pvcAccessModes": "ReadWriteOnce", + }, + expectedError: "cannot unmarshal !!str", + }, } for _, tc := range testCases { diff --git a/internal/resourcepolicies/volume_resources_validator.go b/internal/resourcepolicies/volume_resources_validator.go index 652c41d30..332f98d2e 100644 --- a/internal/resourcepolicies/volume_resources_validator.go +++ b/internal/resourcepolicies/volume_resources_validator.go @@ -19,8 +19,10 @@ import ( "fmt" "io" - "github.com/pkg/errors" - "gopkg.in/yaml.v3" + "github.com/cockroachdb/errors" + "go.yaml.in/yaml/v3" + + datamover "github.com/vmware-tanzu/velero/pkg/util/datamover" ) const currentSupportDataVersion = "v1" @@ -40,13 +42,15 @@ type nFSVolumeSource struct { // volumeConditions defined the current format of conditions we parsed type volumeConditions struct { - Capacity string `yaml:"capacity,omitempty"` - StorageClass []string `yaml:"storageClass,omitempty"` - NFS *nFSVolumeSource `yaml:"nfs,omitempty"` - CSI *csiVolumeSource `yaml:"csi,omitempty"` - VolumeTypes []SupportedVolume `yaml:"volumeTypes,omitempty"` - PVCLabels map[string]string `yaml:"pvcLabels,omitempty"` - PVCPhase []string `yaml:"pvcPhase,omitempty"` + Capacity string `yaml:"capacity,omitempty"` + StorageClass []string `yaml:"storageClass,omitempty"` + NFS *nFSVolumeSource `yaml:"nfs,omitempty"` + CSI *csiVolumeSource `yaml:"csi,omitempty"` + VolumeTypes []SupportedVolume `yaml:"volumeTypes,omitempty"` + PVCLabels map[string]string `yaml:"pvcLabels,omitempty"` + PVCPhase []string `yaml:"pvcPhase,omitempty"` + PVCVolumeMode string `yaml:"pvcVolumeMode,omitempty"` + PVCAccessModes []string `yaml:"pvcAccessModes,omitempty"` } func (c *capacityCondition) validate() error { @@ -97,6 +101,22 @@ func (a *Action) validate() error { return fmt.Errorf("invalid action type %s", a.Type) } - // TODO validate parameters + // validate parameters + if raw, ok := a.Parameters[DataMoverParameter]; ok { + // the dataMover parameter is only meaningful for the snapshot action + if a.Type != Snapshot { + return fmt.Errorf("parameter %q is only supported for the %q action, but the action type is %q", + DataMoverParameter, Snapshot, a.Type) + } + dataMover, ok := raw.(string) + if !ok { + return fmt.Errorf("parameter %q must be a string, got %T", DataMoverParameter, raw) + } + if _, ok := validDataMovers[dataMover]; !ok { + return fmt.Errorf("invalid %q value %q, valid values are %q, %q, %q", + DataMoverParameter, dataMover, datamover.DataMoverTypeVelero, datamover.DataMoverTypeVeleroFs, datamover.DataMoverTypeVeleroBlock) + } + } + return nil } diff --git a/internal/resourcepolicies/volume_resources_validator_test.go b/internal/resourcepolicies/volume_resources_validator_test.go index f2812a786..489e9c653 100644 --- a/internal/resourcepolicies/volume_resources_validator_test.go +++ b/internal/resourcepolicies/volume_resources_validator_test.go @@ -549,6 +549,115 @@ func TestValidate(t *testing.T) { }, wantErr: false, }, + { + name: "snapshot action with valid dataMover velero-fs", + res: &ResourcePolicies{ + Version: "v1", + VolumePolicies: []VolumePolicy{ + { + Action: Action{ + Type: Snapshot, + Parameters: map[string]any{"dataMover": "velero-fs"}, + }, + Conditions: map[string]any{"storageClass": []string{"gp2"}}, + }, + }, + }, + wantErr: false, + }, + { + name: "snapshot action with valid dataMover velero-block", + res: &ResourcePolicies{ + Version: "v1", + VolumePolicies: []VolumePolicy{ + { + Action: Action{ + Type: Snapshot, + Parameters: map[string]any{"dataMover": "velero-block"}, + }, + Conditions: map[string]any{"storageClass": []string{"gp2"}}, + }, + }, + }, + wantErr: false, + }, + { + name: "snapshot action with valid dataMover velero", + res: &ResourcePolicies{ + Version: "v1", + VolumePolicies: []VolumePolicy{ + { + Action: Action{ + Type: Snapshot, + Parameters: map[string]any{"dataMover": "velero"}, + }, + Conditions: map[string]any{"storageClass": []string{"gp2"}}, + }, + }, + }, + wantErr: false, + }, + { + name: "snapshot action with invalid dataMover value", + res: &ResourcePolicies{ + Version: "v1", + VolumePolicies: []VolumePolicy{ + { + Action: Action{ + Type: Snapshot, + Parameters: map[string]any{"dataMover": "unknown-mover"}, + }, + Conditions: map[string]any{"storageClass": []string{"gp2"}}, + }, + }, + }, + wantErr: true, + }, + { + name: "snapshot action with non-string dataMover value", + res: &ResourcePolicies{ + Version: "v1", + VolumePolicies: []VolumePolicy{ + { + Action: Action{ + Type: Snapshot, + Parameters: map[string]any{"dataMover": 123}, + }, + Conditions: map[string]any{"storageClass": []string{"gp2"}}, + }, + }, + }, + wantErr: true, + }, + { + name: "dataMover parameter on non-snapshot action is rejected", + res: &ResourcePolicies{ + Version: "v1", + VolumePolicies: []VolumePolicy{ + { + Action: Action{ + Type: FSBackup, + Parameters: map[string]any{"dataMover": "velero-fs"}, + }, + Conditions: map[string]any{"storageClass": []string{"gp2"}}, + }, + }, + }, + wantErr: true, + }, + { + name: "snapshot action without parameters still valid", + res: &ResourcePolicies{ + Version: "v1", + VolumePolicies: []VolumePolicy{ + { + Action: Action{Type: Snapshot}, + Conditions: map[string]any{"storageClass": []string{"gp2"}}, + }, + }, + }, + wantErr: false, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -568,3 +677,85 @@ func TestValidate(t *testing.T) { }) } } + +func TestValidateForRestore(t *testing.T) { + testCases := []struct { + name string + res *ResourcePolicies + wantErr bool + }{ + { + name: "valid restore policies", + res: &ResourcePolicies{ + Version: "v1", + ClusterScopedFilterPolicy: &ClusterScopedFilterPolicy{ + ResourceFilters: []ResourceFilter{ + { + Kinds: []string{"ClusterRole"}, + }, + }, + }, + NamespacedFilterPolicies: []NamespacedFilterPolicy{ + { + Namespaces: []string{"default"}, + ResourceFilters: []ResourceFilter{ + { + Kinds: []string{"Pod"}, + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "unsupported volumePolicies for restore", + res: &ResourcePolicies{ + Version: "v1", + VolumePolicies: []VolumePolicy{ + { + Action: Action{Type: "skip"}, + Conditions: map[string]any{ + "capacity": "10Gi", + }, + }, + }, + }, + wantErr: true, + }, + { + name: "unsupported includeExcludePolicy for restore", + res: &ResourcePolicies{ + Version: "v1", + IncludeExcludePolicy: &IncludeExcludePolicy{ + IncludedClusterScopedResources: []string{"persistentvolumes"}, + }, + }, + wantErr: true, + }, + { + name: "wrong version", + res: &ResourcePolicies{ + Version: "v2", + }, + wantErr: true, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + policies := &Policies{} + err1 := policies.BuildPolicy(tc.res) + err2 := policies.ValidateForRestore() + + if tc.wantErr { + if err1 == nil && err2 == nil { + t.Fatalf("Expected error %v, but not get error", tc.wantErr) + } + } else { + if err1 != nil || err2 != nil { + t.Fatalf("Expected error %v, but got error %v %v", tc.wantErr, err1, err2) + } + } + }) + } +} diff --git a/internal/resourcepolicies/volume_types_conditions.go b/internal/resourcepolicies/volume_types_conditions.go index 0ee57166b..400af387a 100644 --- a/internal/resourcepolicies/volume_types_conditions.go +++ b/internal/resourcepolicies/volume_types_conditions.go @@ -45,6 +45,7 @@ const ( Glusterfs SupportedVolume = "glusterfs" GCEPersistentDisk SupportedVolume = "gcePersistentDisk" HostPath SupportedVolume = "hostPath" + Image SupportedVolume = "image" ISCSI SupportedVolume = "iscsi" Local SupportedVolume = "local" NFS SupportedVolume = "nfs" @@ -243,5 +244,8 @@ func getVolumeTypeFromVolume(vol *corev1api.Volume) SupportedVolume { if vol.EmptyDir != nil { return EmptyDir } + if vol.Image != nil { + return Image + } return "" } diff --git a/internal/resourcepolicies/volume_types_conditions_test.go b/internal/resourcepolicies/volume_types_conditions_test.go index 7b7be97ee..03f5bbb0b 100644 --- a/internal/resourcepolicies/volume_types_conditions_test.go +++ b/internal/resourcepolicies/volume_types_conditions_test.go @@ -563,6 +563,15 @@ func TestGetVolumeTypeFromVolume(t *testing.T) { }, expected: Ephemeral, }, + { + name: "Test Image", + inputVol: &corev1api.Volume{ + VolumeSource: corev1api.VolumeSource{ + Image: &corev1api.ImageVolumeSource{}, + }, + }, + expected: Image, + }, } for _, tc := range testCases { diff --git a/internal/restartabletest/restartable_delegate.go b/internal/restartabletest/restartable_delegate.go index 41d56cf31..cfc1668c7 100644 --- a/internal/restartabletest/restartable_delegate.go +++ b/internal/restartabletest/restartable_delegate.go @@ -19,7 +19,7 @@ import ( "reflect" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" diff --git a/internal/storage/storagelocation.go b/internal/storage/storagelocation.go index d5fe548c0..59afe6b79 100644 --- a/internal/storage/storagelocation.go +++ b/internal/storage/storagelocation.go @@ -20,7 +20,7 @@ import ( "context" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/internal/volume/snapshotlocation.go b/internal/volume/snapshotlocation.go index ad23fa1f4..594fbf3a5 100644 --- a/internal/volume/snapshotlocation.go +++ b/internal/volume/snapshotlocation.go @@ -17,7 +17,7 @@ limitations under the License. package volume import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" diff --git a/internal/volume/volumes_information.go b/internal/volume/volumes_information.go index 4d5961bdb..ad8993447 100644 --- a/internal/volume/volumes_information.go +++ b/internal/volume/volumes_information.go @@ -22,8 +22,8 @@ import ( "strings" "sync" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/internal/volumehelper/volume_policy_helper.go b/internal/volumehelper/volume_policy_helper.go index 339b80011..6931697c9 100644 --- a/internal/volumehelper/volume_policy_helper.go +++ b/internal/volumehelper/volume_policy_helper.go @@ -5,7 +5,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/apis/go.mod b/pkg/apis/go.mod new file mode 100644 index 000000000..eb7f20924 --- /dev/null +++ b/pkg/apis/go.mod @@ -0,0 +1,28 @@ +module github.com/vmware-tanzu/velero/pkg/apis + +go 1.26.0 + +require ( + k8s.io/api v0.36.0 + k8s.io/apimachinery v0.36.0 +) + +require ( + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/x448/float16 v0.8.4 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/text v0.37.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/klog/v2 v2.140.0 // indirect + k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect + k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect +) diff --git a/pkg/apis/go.sum b/pkg/apis/go.sum new file mode 100644 index 000000000..ec45c153b --- /dev/null +++ b/pkg/apis/go.sum @@ -0,0 +1,68 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.36.0 h1:SgqDhZzHdOtMk40xVSvCXkP9ME0H05hPM3p9AB1kL80= +k8s.io/api v0.36.0/go.mod h1:m1LVrGPNYax5NBHdO+QuAedXyuzTt4RryI/qnmNvs34= +k8s.io/apimachinery v0.36.0 h1:jZyPzhd5Z+3h9vJLt0z9XdzW9VzNzWAUw+P1xZ9PXtQ= +k8s.io/apimachinery v0.36.0/go.mod h1:FklypaRJt6n5wUIwWXIP6GJlIpUizTgfo1T/As+Tyxc= +k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= +k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/pkg/apis/velero/v1/backup_types.go b/pkg/apis/velero/v1/backup_types.go index 24af8132d..435e88f30 100644 --- a/pkg/apis/velero/v1/backup_types.go +++ b/pkg/apis/velero/v1/backup_types.go @@ -184,6 +184,10 @@ type BackupSpec struct { // +optional // +nullable UploaderConfig *UploaderConfigForBackup `json:"uploaderConfig,omitempty"` + + // BackupType specifies how volume data is backed up, with possible values including Full and Incremental. + // +optional + BackupType BackupType `json:"backupType,omitempty"` } // UploaderConfigForBackup defines the configuration for the uploader when doing backup. @@ -357,6 +361,15 @@ const ( BackupPhaseDeleting BackupPhase = "Deleting" ) +// BackupType specifies how volume data is backed up, with possible values including Full and Incremental. +// +kubebuilder:validation:Enum=Full;Incremental +type BackupType string + +const ( + BackupTypeFull BackupType = "Full" + BackupTypeIncremental BackupType = "Incremental" +) + // BackupStatus captures the current status of a Velero backup. type BackupStatus struct { // Version is the backup format major version. diff --git a/pkg/apis/velero/v1/labels_annotations.go b/pkg/apis/velero/v1/labels_annotations.go index 921af498e..13da279d8 100644 --- a/pkg/apis/velero/v1/labels_annotations.go +++ b/pkg/apis/velero/v1/labels_annotations.go @@ -80,6 +80,11 @@ const ( // timeout value for backup to plugins. ResourceTimeoutAnnotation = "velero.io/resource-timeout" + // GlobalBackupVolumePolicyConfigMapAnnotation is the annotation key used to record the + // name of the cluster-wide global backup volume policies ConfigMap that contributed to a + // backup, so that `velero backup describe` can surface it. + GlobalBackupVolumePolicyConfigMapAnnotation = "velero.io/global-backup-volume-policy-configmap" + // AsyncOperationIDLabel is the label key used to identify the async operation ID AsyncOperationIDLabel = "velero.io/async-operation-id" diff --git a/pkg/apis/velero/v1/restore_types.go b/pkg/apis/velero/v1/restore_types.go index 5dd99edb7..f6e6bf9cf 100644 --- a/pkg/apis/velero/v1/restore_types.go +++ b/pkg/apis/velero/v1/restore_types.go @@ -125,6 +125,16 @@ type RestoreSpec struct { // +nullable ResourceModifier *corev1api.TypedLocalObjectReference `json:"resourceModifier,omitempty"` + // ResourcePolicy specifies the reference to a ConfigMap containing resource + // filter policies for this restore. The ConfigMap can contain a + // namespacedFilterPolicies section that specifies per-namespace resource type + // filters, label selectors, and resource name patterns, and a + // clusterScopedFilterPolicy section for per-kind filtering of cluster-scoped + // resources. The ConfigMap format is the same as for BackupSpec.ResourcePolicy. + // +optional + // +nullable + ResourcePolicy *corev1api.TypedLocalObjectReference `json:"resourcePolicy,omitempty"` + // UploaderConfig specifies the configuration for the restore. // +optional // +nullable diff --git a/pkg/apis/velero/v1/zz_generated.deepcopy.go b/pkg/apis/velero/v1/zz_generated.deepcopy.go index 0702f8623..c40fbb806 100644 --- a/pkg/apis/velero/v1/zz_generated.deepcopy.go +++ b/pkg/apis/velero/v1/zz_generated.deepcopy.go @@ -1415,6 +1415,11 @@ func (in *RestoreSpec) DeepCopyInto(out *RestoreSpec) { *out = new(corev1.TypedLocalObjectReference) (*in).DeepCopyInto(*out) } + if in.ResourcePolicy != nil { + in, out := &in.ResourcePolicy, &out.ResourcePolicy + *out = new(corev1.TypedLocalObjectReference) + (*in).DeepCopyInto(*out) + } if in.UploaderConfig != nil { in, out := &in.UploaderConfig, &out.UploaderConfig *out = new(UploaderConfigForRestore) diff --git a/pkg/apis/velero/v2alpha1/data_download_types.go b/pkg/apis/velero/v2alpha1/data_download_types.go index 4ea7128ec..616876563 100644 --- a/pkg/apis/velero/v2alpha1/data_download_types.go +++ b/pkg/apis/velero/v2alpha1/data_download_types.go @@ -74,6 +74,10 @@ type TargetVolumeSpec struct { // Namespace is the target namespace Namespace string `json:"namespace"` + + // FSType is the file system type of the target volume. + // +optional + FSType string `json:"fsType,omitempty"` } // DataDownloadPhase represents the lifecycle phase of a DataDownload. diff --git a/pkg/apis/velero/v2alpha1/data_upload_types.go b/pkg/apis/velero/v2alpha1/data_upload_types.go index 751da4555..39ae349d6 100644 --- a/pkg/apis/velero/v2alpha1/data_upload_types.go +++ b/pkg/apis/velero/v2alpha1/data_upload_types.go @@ -60,6 +60,10 @@ type DataUploadSpec struct { // OperationTimeout specifies the time used to wait internal operations, // before returning error as timeout. OperationTimeout metav1.Duration `json:"operationTimeout"` + + // SourceFSType is the file system type of the source volume. + // +optional + SourceFSType string `json:"sourceFSType,omitempty"` } type SnapshotType string @@ -253,4 +257,8 @@ type DataUploadResult struct { // SnapshotSize is the logical size in Bytes of the snapshot. // +optional SnapshotSize int64 `json:"snapshotSize,omitempty"` + + // FSType is the file system type of the volume. + // +optional + FSType string `json:"fsType,omitempty"` } diff --git a/pkg/archive/parser.go b/pkg/archive/parser.go index 166e03114..426813801 100644 --- a/pkg/archive/parser.go +++ b/pkg/archive/parser.go @@ -21,7 +21,7 @@ import ( "path/filepath" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/backup/actions/backup_pv_action.go b/pkg/backup/actions/backup_pv_action.go index c3f378fac..32275544e 100644 --- a/pkg/backup/actions/backup_pv_action.go +++ b/pkg/backup/actions/backup_pv_action.go @@ -19,7 +19,7 @@ package actions import ( "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/backup/actions/csi/pvc_action.go b/pkg/backup/actions/csi/pvc_action.go index 7f5fd2afa..66c14b820 100644 --- a/pkg/backup/actions/csi/pvc_action.go +++ b/pkg/backup/actions/csi/pvc_action.go @@ -22,15 +22,14 @@ import ( "strconv" "time" - "k8s.io/client-go/util/retry" - + "github.com/cockroachdb/errors" volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" storagev1api "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" @@ -39,11 +38,10 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + "k8s.io/client-go/util/retry" crclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "k8s.io/apimachinery/pkg/api/resource" - velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" veleroclient "github.com/vmware-tanzu/velero/pkg/client" @@ -160,12 +158,13 @@ func (p *pvcBackupItemAction) getOrCreateVolumeHelper(backup *velerov1api.Backup return p.getVolumeHelperWithCache(backup) } -func (p *pvcBackupItemAction) validatePVCandPV( +func (p *pvcBackupItemAction) validatePVCAndPV( pvc corev1api.PersistentVolumeClaim, item runtime.Unstructured, ) ( valid bool, updateItem runtime.Unstructured, + fsType string, err error, ) { updateItem = item @@ -174,6 +173,7 @@ func (p *pvcBackupItemAction) validatePVCandPV( if pvc.Spec.StorageClassName == nil { return false, updateItem, + "", errors.Errorf( "Cannot snapshot PVC %s/%s, PVC has no storage class.", pvc.Namespace, pvc.Name) @@ -187,7 +187,7 @@ func (p *pvcBackupItemAction) validatePVCandPV( // Do nothing if this is not a CSI provisioned volume pv, err := kubeutil.GetPVForPVC(&pvc, p.crClient) if err != nil { - return false, updateItem, errors.WithStack(err) + return false, updateItem, "", errors.WithStack(err) } if pv.Spec.PersistentVolumeSource.CSI == nil { @@ -202,10 +202,10 @@ func (p *pvcBackupItemAction) validatePVCandPV( }) data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pvc) updateItem = &unstructured.Unstructured{Object: data} - return false, updateItem, err + return false, updateItem, "", err } - return true, updateItem, nil + return true, updateItem, pv.Spec.PersistentVolumeSource.CSI.FSType, nil } func (p *pvcBackupItemAction) createVolumeSnapshot( @@ -301,10 +301,12 @@ func (p *pvcBackupItemAction) Execute( ); err != nil { return nil, nil, "", nil, errors.WithStack(err) } - if valid, item, err := p.validatePVCandPV( + + valid, item, fsType, err := p.validatePVCAndPV( pvc, item, - ); !valid { + ) + if !valid { if err != nil { return nil, nil, "", nil, err } @@ -392,6 +394,7 @@ func (p *pvcBackupItemAction) Execute( &pvc, operationID, vsc, + fsType, ) if err != nil { dataUploadLog.WithError(err).Error("failed to submit DataUpload") @@ -530,6 +533,7 @@ func newDataUpload( pvc *corev1api.PersistentVolumeClaim, operationID string, vsc *snapshotv1api.VolumeSnapshotContent, + fsType string, ) *velerov2alpha1.DataUpload { dataUpload := &velerov2alpha1.DataUpload{ TypeMeta: metav1.TypeMeta{ @@ -567,6 +571,7 @@ func newDataUpload( BackupStorageLocation: backup.Spec.StorageLocation, SourceNamespace: pvc.Namespace, OperationTimeout: backup.Spec.CSISnapshotTimeout, + SourceFSType: fsType, }, } @@ -591,8 +596,9 @@ func createDataUpload( pvc *corev1api.PersistentVolumeClaim, operationID string, vsc *snapshotv1api.VolumeSnapshotContent, + fsType string, ) (*velerov2alpha1.DataUpload, error) { - dataUpload := newDataUpload(backup, vs, pvc, operationID, vsc) + dataUpload := newDataUpload(backup, vs, pvc, operationID, vsc, fsType) err := crClient.Create(ctx, dataUpload) if err != nil { diff --git a/pkg/backup/actions/csi/pvc_action_test.go b/pkg/backup/actions/csi/pvc_action_test.go index 629c5e9cd..e7320cd1a 100644 --- a/pkg/backup/actions/csi/pvc_action_test.go +++ b/pkg/backup/actions/csi/pvc_action_test.go @@ -29,7 +29,7 @@ import ( "github.com/stretchr/testify/assert" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" "github.com/vmware-tanzu/velero/pkg/label" @@ -37,7 +37,7 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" @@ -669,7 +669,7 @@ func TestFilterPVCsByVolumePolicy(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "pvc-1", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-1", - StorageClassName: pointer.String("sc-1"), + StorageClassName: ptr.To("sc-1"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, @@ -677,7 +677,7 @@ func TestFilterPVCsByVolumePolicy(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "pvc-2", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-2", - StorageClassName: pointer.String("sc-1"), + StorageClassName: ptr.To("sc-1"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, @@ -709,7 +709,7 @@ func TestFilterPVCsByVolumePolicy(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "pvc-csi", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-csi", - StorageClassName: pointer.String("sc-1"), + StorageClassName: ptr.To("sc-1"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, @@ -717,7 +717,7 @@ func TestFilterPVCsByVolumePolicy(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "pvc-nfs", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-nfs", - StorageClassName: pointer.String("sc-nfs"), + StorageClassName: ptr.To("sc-nfs"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, @@ -760,7 +760,7 @@ volumePolicies: ObjectMeta: metav1.ObjectMeta{Name: "pvc-nfs-1", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-nfs-1", - StorageClassName: pointer.String("sc-nfs"), + StorageClassName: ptr.To("sc-nfs"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, @@ -768,7 +768,7 @@ volumePolicies: ObjectMeta: metav1.ObjectMeta{Name: "pvc-nfs-2", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-nfs-2", - StorageClassName: pointer.String("sc-nfs"), + StorageClassName: ptr.To("sc-nfs"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, @@ -818,7 +818,7 @@ volumePolicies: }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-linstor", - StorageClassName: pointer.String("sc-linstor"), + StorageClassName: ptr.To("sc-linstor"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, @@ -830,7 +830,7 @@ volumePolicies: }, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-nfs", - StorageClassName: pointer.String("sc-nfs"), + StorageClassName: ptr.To("sc-nfs"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, @@ -943,7 +943,7 @@ func TestFilterPVCsByVolumePolicyWithVolumeHelper(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "pvc-csi", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-csi", - StorageClassName: pointer.String("sc-csi"), + StorageClassName: ptr.To("sc-csi"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, @@ -951,7 +951,7 @@ func TestFilterPVCsByVolumePolicyWithVolumeHelper(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "pvc-nfs", Namespace: "ns-1"}, Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "pv-nfs", - StorageClassName: pointer.String("sc-nfs"), + StorageClassName: ptr.To("sc-nfs"), }, Status: corev1api.PersistentVolumeClaimStatus{Phase: corev1api.ClaimBound}, }, @@ -1365,7 +1365,7 @@ func TestWaitForVGSAssociatedVS(t *testing.T) { }, Spec: snapshotv1api.VolumeSnapshotSpec{ Source: snapshotv1api.VolumeSnapshotSource{ - PersistentVolumeClaimName: pointer.String(pvcName), + PersistentVolumeClaimName: ptr.To(pvcName), }, }, } @@ -1373,7 +1373,7 @@ func TestWaitForVGSAssociatedVS(t *testing.T) { if hasStatus { vs.Status = &snapshotv1api.VolumeSnapshotStatus{} if hasVGSName { - vs.Status.VolumeGroupSnapshotName = pointer.String(vgs.Name) + vs.Status.VolumeGroupSnapshotName = ptr.To(vgs.Name) } } @@ -1527,12 +1527,12 @@ func TestUpdateVGSCreatedVS(t *testing.T) { }, }, Status: &snapshotv1api.VolumeSnapshotStatus{ - ReadyToUse: pointer.Bool(true), + ReadyToUse: ptr.To(true), VolumeGroupSnapshotName: vgsNamePtr, }, Spec: snapshotv1api.VolumeSnapshotSpec{ Source: snapshotv1api.VolumeSnapshotSource{ - PersistentVolumeClaimName: pointer.String(pvcName), + PersistentVolumeClaimName: ptr.To(pvcName), }, }, } @@ -1547,7 +1547,7 @@ func TestUpdateVGSCreatedVS(t *testing.T) { }{ { name: "should update owned VS", - vs: makeVS("vs-owned", true, pointer.String(vgs.Name), "pvc-1"), + vs: makeVS("vs-owned", true, ptr.To(vgs.Name), "pvc-1"), expectOwnerCleared: true, expectFinalizersCleared: true, expectLabelPatched: true, @@ -1640,7 +1640,7 @@ func TestPatchVGSCDeletionPolicy(t *testing.T) { Namespace: "ns", }, Status: &volumegroupsnapshotv1beta2.VolumeGroupSnapshotStatus{ - BoundVolumeGroupSnapshotContentName: pointer.String("test-vgsc"), + BoundVolumeGroupSnapshotContentName: ptr.To("test-vgsc"), }, } @@ -1695,14 +1695,14 @@ func TestDeleteVGSAndVGSC(t *testing.T) { }{ { name: "deletes both VGSC and VGS", - vgs: makeVGS("test-vgs", "ns", pointer.String("test-vgsc")), + vgs: makeVGS("test-vgs", "ns", ptr.To("test-vgsc")), existingVGSC: makeVGSC("test-vgsc"), expectVGSCDelete: true, expectVGSDelete: true, }, { name: "VGSC not found, still deletes VGS", - vgs: makeVGS("test-vgs", "ns", pointer.String("missing-vgsc")), + vgs: makeVGS("test-vgs", "ns", ptr.To("missing-vgsc")), existingVGSC: nil, expectVGSCDelete: false, expectVGSDelete: true, @@ -1768,7 +1768,7 @@ func TestFindExistingVSForBackup(t *testing.T) { }, Spec: snapshotv1api.VolumeSnapshotSpec{ Source: snapshotv1api.VolumeSnapshotSource{ - PersistentVolumeClaimName: pointer.String(pvc), + PersistentVolumeClaimName: ptr.To(pvc), }, }, } @@ -2121,7 +2121,7 @@ func TestPVCRequestSize(t *testing.T) { Name: "testVSC", }, Status: &snapshotv1api.VolumeSnapshotContentStatus{ - RestoreSize: pointer.Int64(rsQty.Value()), + RestoreSize: ptr.To(rsQty.Value()), }, } diff --git a/pkg/backup/actions/csi/volumesnapshot_action.go b/pkg/backup/actions/csi/volumesnapshot_action.go index 0e0e9a840..49e690e93 100644 --- a/pkg/backup/actions/csi/volumesnapshot_action.go +++ b/pkg/backup/actions/csi/volumesnapshot_action.go @@ -22,8 +22,8 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/backup/actions/csi/volumesnapshotclass_action.go b/pkg/backup/actions/csi/volumesnapshotclass_action.go index 8200b465f..8d70fae17 100644 --- a/pkg/backup/actions/csi/volumesnapshotclass_action.go +++ b/pkg/backup/actions/csi/volumesnapshotclass_action.go @@ -17,7 +17,7 @@ limitations under the License. package csi import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" diff --git a/pkg/backup/actions/csi/volumesnapshotcontent_action.go b/pkg/backup/actions/csi/volumesnapshotcontent_action.go index d4cd6d46c..f184230d1 100644 --- a/pkg/backup/actions/csi/volumesnapshotcontent_action.go +++ b/pkg/backup/actions/csi/volumesnapshotcontent_action.go @@ -19,8 +19,8 @@ package csi import ( "fmt" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/backup/actions/pod_action.go b/pkg/backup/actions/pod_action.go index 8ed5e3b44..f8693228f 100644 --- a/pkg/backup/actions/pod_action.go +++ b/pkg/backup/actions/pod_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/backup/actions/remap_crd_version_action.go b/pkg/backup/actions/remap_crd_version_action.go index 3f8c2f79d..59a84ee9e 100644 --- a/pkg/backup/actions/remap_crd_version_action.go +++ b/pkg/backup/actions/remap_crd_version_action.go @@ -20,7 +20,7 @@ import ( "context" "encoding/json" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" diff --git a/pkg/backup/actions/service_account_action.go b/pkg/backup/actions/service_account_action.go index b563f7a03..544fbcb23 100644 --- a/pkg/backup/actions/service_account_action.go +++ b/pkg/backup/actions/service_account_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/backup/backup.go b/pkg/backup/backup.go index 9edaf6a85..dc60bba8c 100644 --- a/pkg/backup/backup.go +++ b/pkg/backup/backup.go @@ -26,16 +26,19 @@ import ( "io" "os" "path/filepath" + "strings" "sync" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" + "github.com/gobwas/glob" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" kubeerrs "k8s.io/apimachinery/pkg/util/errors" @@ -43,6 +46,7 @@ import ( kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/hook" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/volume" "github.com/vmware-tanzu/velero/internal/volumehelper" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -352,6 +356,52 @@ func (kb *kubernetesBackupper) BackupWithResolvers( backupRequest.ResourceIncludesExcludes = srie } + if backupRequest.ResPolicies != nil { + clusterScopedFilterPolicy := backupRequest.ResPolicies.GetClusterScopedFilterPolicy() + if clusterScopedFilterPolicy != nil { + backupRequest.ClusterScopedFilterMap, err = resolveClusterScopedFilterPolicy( + clusterScopedFilterPolicy, + kb.discoveryHelper, + log, + ) + if err != nil { + return err + } + log.Infof("Resolved clusterScopedFilterPolicy: %d kind group(s) in cluster-scoped filter map", + len(backupRequest.ClusterScopedFilterMap)) + } + + nfPolicies := backupRequest.ResPolicies.GetNamespacedFilterPolicies() + if len(nfPolicies) > 0 { + backupRequest.NamespacedFilterMap, backupRequest.NamespacedFilterPatterns, err = resolveNamespacedFilterPolicies( + nfPolicies, + kb.discoveryHelper, + log, + ) + if err != nil { + return err + } + log.Infof("Resolved namespacedFilterPolicies: %d namespace pattern(s) registered", + len(backupRequest.NamespacedFilterPatterns)) + for _, p := range backupRequest.NamespacedFilterPatterns { + nsf := backupRequest.NamespacedFilterMap[p.Pattern] + log.WithFields(logrus.Fields{ + "namespacePattern": p.Pattern, + "kindCount": len(nsf.ResourceFilterMap), + "hasCatchAll": nsf.CatchAllFilter != nil, + }).Debug("namespacedFilterPolicies: namespace pattern registered") + for kind := range nsf.ResourceFilterMap { + if backupRequest.ResourceIncludesExcludes.ShouldExclude(kind) { + log.WithFields(logrus.Fields{ + "namespacePattern": p.Pattern, + "kind": kind, + }).Warn("namespacedFilterPolicies entry lists a kind that is globally excluded by includeExcludePolicy; the per-namespace filter entry has no effect") + } + } + } + } + } + log.Infof("Backing up all volumes using pod volume backup: %t", boolptr.IsSetToTrue(backupRequest.Backup.Spec.DefaultVolumesToFsBackup)) backupRequest.ResourceHooks, err = getResourceHooks(backupRequest.Spec.Hooks.Resources, kb.discoveryHelper) @@ -1341,3 +1391,140 @@ func putVolumeInfos( return backupStore.PutBackupVolumeInfos(backupName, backupVolumeInfoBuf) } + +func resolveClusterScopedFilterPolicy( + policy *resourcepolicies.ClusterScopedFilterPolicy, + helper discovery.Helper, + log logrus.FieldLogger, +) (map[string]*ResolvedResourceFilter, error) { + rfMap := make(map[string]*ResolvedResourceFilter) + + for _, rf := range policy.ResourceFilters { + resolved, err := resolveResourceFilter(rf) + if err != nil { + return nil, err + } + + for _, kind := range rf.Kinds { + gr, apiResource, err := helper.ResourceFor( + schema.GroupVersionResource{Resource: kind}, + ) + if err != nil { + log.WithField("kind", kind).Warnf( + "Cannot resolve kind via discovery, using as-is: %v", err) + rfMap[kind] = resolved + continue + } + if apiResource.Namespaced { + log.WithField("kind", kind).Warnf( + "kind %q in clusterScopedFilterPolicy is namespace-scoped; "+ + "it will never match in a cluster-scoped filter — did you mean namespacedFilterPolicies?", kind) + } + rfMap[gr.GroupResource().String()] = resolved + } + } + + return rfMap, nil +} + +func resolveResourceFilter(rf resourcepolicies.ResourceFilter) (*ResolvedResourceFilter, error) { + var selector labels.Selector + if len(rf.LabelSelector) > 0 { + var err error + selector, err = labels.ValidatedSelectorFromSet(labels.Set(rf.LabelSelector)) + if err != nil { + return nil, fmt.Errorf("invalid label selector in resource filter: %w", err) + } + } + + var orSelectors []labels.Selector + for _, ols := range rf.OrLabelSelectors { + s, err := labels.ValidatedSelectorFromSet(labels.Set(ols)) + if err != nil { + return nil, fmt.Errorf("invalid OR label selector in resource filter: %w", err) + } + orSelectors = append(orSelectors, s) + } + + var nameIE *collections.IncludesExcludes + if len(rf.Names) > 0 || len(rf.ExcludedNames) > 0 { + nameIE = collections.NewIncludesExcludes() + nameIE.Includes(rf.Names...) + nameIE.Excludes(rf.ExcludedNames...) + } + + return &ResolvedResourceFilter{ + LabelSelector: selector, + OrLabelSelectors: orSelectors, + NameIE: nameIE, + }, nil +} + +func resolveNamespacedFilterPolicies( + policies []resourcepolicies.NamespacedFilterPolicy, + helper discovery.Helper, + log logrus.FieldLogger, +) (map[string]*ResolvedNamespaceFilter, []NamespacedFilterPattern, error) { + result := make(map[string]*ResolvedNamespaceFilter) + var patternOrder []NamespacedFilterPattern + + for _, policy := range policies { + rfMap := make(map[string]*ResolvedResourceFilter) + var nsFilter *ResolvedNamespaceFilter + + for _, rf := range policy.ResourceFilters { + resolved, err := resolveResourceFilter(rf) + if err != nil { + return nil, nil, err + } + + if rf.IsCatchAll() { + if nsFilter == nil { + nsFilter = &ResolvedNamespaceFilter{ResourceFilterMap: rfMap} + } + nsFilter.CatchAllFilter = resolved + } else { + // Resolve each kind to a fully-qualified group-resource string with improved error handling + for _, kind := range rf.Kinds { + gr, apiResource, err := helper.ResourceFor( + schema.GroupVersionResource{Resource: kind}, + ) + if err != nil { + // Log warning but continue - allows for forward compatibility + log.WithField("kind", kind).Warnf( + "Cannot resolve kind via discovery, using as-is: %v", err) + rfMap[kind] = resolved + continue + } + if !apiResource.Namespaced { + log.WithField("kind", kind).Warnf( + "kind %q in namespacedFilterPolicies is cluster-scoped; "+ + "it will never match in a namespace-scoped filter — did you mean clusterScopedFilterPolicy?", kind) + } + rfMap[gr.GroupResource().String()] = resolved + } + } + } + + if nsFilter == nil { + nsFilter = &ResolvedNamespaceFilter{} + } + nsFilter.ResourceFilterMap = rfMap + for _, nsPattern := range policy.Namespaces { + result[nsPattern] = nsFilter + // Pre-compile glob patterns once here; exact names are matched via map + // and never reach the pattern loop, so only wildcard patterns need Compiled set. + entry := NamespacedFilterPattern{Pattern: nsPattern} + if strings.ContainsAny(nsPattern, "*?[") { + if compiled, cerr := glob.Compile(nsPattern); cerr == nil { + entry.Compiled = compiled + } else { + // Pattern already validated; this branch should not be reached + log.WithField("pattern", nsPattern).Warnf("Failed to pre-compile glob pattern: %v", cerr) + } + } + patternOrder = append(patternOrder, entry) + } + } + return result, patternOrder, nil +} diff --git a/pkg/backup/backup_test.go b/pkg/backup/backup_test.go index f9351245c..56f4aaf33 100644 --- a/pkg/backup/backup_test.go +++ b/pkg/backup/backup_test.go @@ -30,7 +30,8 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" + "github.com/gobwas/glob" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -39,7 +40,9 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/volume" @@ -5728,3 +5731,419 @@ func (f *fakeSingleObjectBackupStoreGetter) Get(*velerov1.BackupStorageLocation, func NewFakeSingleObjectBackupStoreGetter(store persistence.BackupStore) persistence.ObjectBackupStoreGetter { return &fakeSingleObjectBackupStoreGetter{store: store} } +func TestResolveResourceFilter(t *testing.T) { + tests := []struct { + name string + rf resourcepolicies.ResourceFilter + expectErr bool + checkResult func(*testing.T, *ResolvedResourceFilter) + }{ + { + name: "valid label selector", + rf: resourcepolicies.ResourceFilter{ + LabelSelector: map[string]string{"app": "foo"}, + }, + expectErr: false, + checkResult: func(t *testing.T, r *ResolvedResourceFilter) { + t.Helper() + require.NotNil(t, r) + require.NotNil(t, r.LabelSelector) + assert.True(t, r.LabelSelector.Matches(labels.Set{"app": "foo"})) + }, + }, + { + name: "invalid label selector", + rf: resourcepolicies.ResourceFilter{ + LabelSelector: map[string]string{"invalid/label/key": "value"}, + }, + expectErr: true, + }, + { + name: "valid or label selectors", + rf: resourcepolicies.ResourceFilter{ + OrLabelSelectors: []map[string]string{ + {"app": "foo"}, + {"app": "bar"}, + }, + }, + expectErr: false, + checkResult: func(t *testing.T, r *ResolvedResourceFilter) { + t.Helper() + require.NotNil(t, r) + require.Len(t, r.OrLabelSelectors, 2) + }, + }, + { + name: "invalid or label selectors", + rf: resourcepolicies.ResourceFilter{ + OrLabelSelectors: []map[string]string{ + {"invalid/label/key": "value"}, + }, + }, + expectErr: true, + }, + { + name: "names and excluded names", + rf: resourcepolicies.ResourceFilter{ + Names: []string{"inc1", "inc2"}, + ExcludedNames: []string{"exc1"}, + }, + expectErr: false, + checkResult: func(t *testing.T, r *ResolvedResourceFilter) { + t.Helper() + require.NotNil(t, r) + require.NotNil(t, r.NameIE) + assert.True(t, r.NameIE.ShouldInclude("inc1")) + assert.False(t, r.NameIE.ShouldInclude("exc1")) + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + res, err := resolveResourceFilter(tc.rf) + if tc.expectErr { + require.Error(t, err) + } else { + assert.NoError(t, err) + if tc.checkResult != nil { + tc.checkResult(t, res) + } + } + }) + } +} + +type mockDiscoveryHelper struct { + discovery.Helper + ResourceForFunc func(input schema.GroupVersionResource) (schema.GroupVersionResource, metav1.APIResource, error) +} + +func (m *mockDiscoveryHelper) ResourceFor(input schema.GroupVersionResource) (schema.GroupVersionResource, metav1.APIResource, error) { + if m.ResourceForFunc != nil { + return m.ResourceForFunc(input) + } + return m.Helper.ResourceFor(input) +} + +func TestResolveClusterScopedFilterPolicy(t *testing.T) { + helper := test.NewFakeDiscoveryHelper(true, nil) + log := test.NewLogger() + + policy := &resourcepolicies.ClusterScopedFilterPolicy{ + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"pods", "secrets"}, + LabelSelector: map[string]string{"app": "foo"}, + }, + { + Kinds: []string{"invalid-kind"}, + LabelSelector: map[string]string{"invalid/label/key": "value"}, + }, + }, + } + + // Test with invalid label selector to trigger error + _, err := resolveClusterScopedFilterPolicy(policy, helper, log) + require.Error(t, err) + + // Test valid policy + validPolicy := &resourcepolicies.ClusterScopedFilterPolicy{ + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"pods", "secrets"}, + LabelSelector: map[string]string{"app": "foo"}, + }, + }, + } + res, err := resolveClusterScopedFilterPolicy(validPolicy, helper, log) + require.NoError(t, err) + require.Len(t, res, 2) + assert.Contains(t, res, "pods") + assert.Contains(t, res, "secrets") + assert.True(t, res["pods"].LabelSelector.Matches(labels.Set{"app": "foo"})) + + // Test warning branches + mockHelper := &mockDiscoveryHelper{ + Helper: helper, + ResourceForFunc: func(input schema.GroupVersionResource) (schema.GroupVersionResource, metav1.APIResource, error) { + if input.Resource == "invalid-resource" { + return schema.GroupVersionResource{}, metav1.APIResource{}, errors.New("cannot resolve") + } + if input.Resource == "namespaced-resource" { + return schema.GroupVersionResource{Resource: "namespaced-resource"}, metav1.APIResource{Namespaced: true, Name: "namespaced-resource"}, nil + } + return helper.ResourceFor(input) + }, + } + + policyWithWarns := &resourcepolicies.ClusterScopedFilterPolicy{ + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"invalid-resource", "namespaced-resource"}, + }, + }, + } + res2, err2 := resolveClusterScopedFilterPolicy(policyWithWarns, mockHelper, log) + require.NoError(t, err2) + assert.Contains(t, res2, "invalid-resource") + assert.Contains(t, res2, "namespaced-resource") +} + +func TestResolveNamespacedFilterPolicies(t *testing.T) { + helper := test.NewFakeDiscoveryHelper(true, nil) + log := test.NewLogger() + + policies := []resourcepolicies.NamespacedFilterPolicy{ + { + Namespaces: []string{"ns1", "ns-*"}, + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"pods"}, + LabelSelector: map[string]string{"app": "foo"}, + }, + { + Kinds: []string{"*"}, + LabelSelector: map[string]string{"catch": "all"}, + }, + }, + }, + } + + res, patterns, err := resolveNamespacedFilterPolicies(policies, helper, log) + require.NoError(t, err) + require.Len(t, res, 2) + require.Len(t, patterns, 2) + + assert.Contains(t, res, "ns1") + assert.Contains(t, res, "ns-*") + + ns1Filter := res["ns1"] + require.NotNil(t, ns1Filter) + require.NotNil(t, ns1Filter.CatchAllFilter) + assert.True(t, ns1Filter.CatchAllFilter.LabelSelector.Matches(labels.Set{"catch": "all"})) + require.Contains(t, ns1Filter.ResourceFilterMap, "pods") + assert.True(t, ns1Filter.ResourceFilterMap["pods"].LabelSelector.Matches(labels.Set{"app": "foo"})) + + // Test with invalid label selector + invalidPolicies := []resourcepolicies.NamespacedFilterPolicy{ + { + Namespaces: []string{"ns1"}, + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"pods"}, + LabelSelector: map[string]string{"invalid/label/key": "value"}, + }, + }, + }, + } + _, _, err = resolveNamespacedFilterPolicies(invalidPolicies, helper, log) + require.Error(t, err) + + // Test warning branches + mockHelper := &mockDiscoveryHelper{ + Helper: helper, + ResourceForFunc: func(input schema.GroupVersionResource) (schema.GroupVersionResource, metav1.APIResource, error) { + if input.Resource == "invalid-resource" { + return schema.GroupVersionResource{}, metav1.APIResource{}, errors.New("cannot resolve") + } + if input.Resource == "cluster-scoped-resource" { + return schema.GroupVersionResource{Resource: "cluster-scoped-resource"}, metav1.APIResource{Namespaced: false, Name: "cluster-scoped-resource"}, nil + } + return schema.GroupVersionResource{Resource: input.Resource}, metav1.APIResource{Namespaced: true, Name: input.Resource}, nil + }, + } + + policyWithWarns := []resourcepolicies.NamespacedFilterPolicy{ + { + Namespaces: []string{"ns1"}, + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"invalid-resource", "cluster-scoped-resource"}, + }, + }, + }, + } + resWarns, _, errWarns := resolveNamespacedFilterPolicies(policyWithWarns, mockHelper, log) + require.NoError(t, errWarns) + require.Contains(t, resWarns["ns1"].ResourceFilterMap, "invalid-resource") + require.Contains(t, resWarns["ns1"].ResourceFilterMap, "cluster-scoped-resource") +} + +func TestBackupWithResPoliciesLogs(t *testing.T) { + itemBlockPool := StartItemBlockWorkerPool(t.Context(), 1, logrus.StandardLogger()) + defer itemBlockPool.Stop() + + h := newHarness(t, itemBlockPool) + + // Add some resources so discovery helper knows about them + h.addItems(t, test.Pods(builder.ForPod("ns1", "pod-1").Result())) + h.addItems(t, test.PVs(builder.ForPersistentVolume("pv-1").Result())) + + backupReq := &Request{ + Backup: defaultBackup().ExcludedNamespaceScopedResources("pods").Result(), + SkippedPVTracker: NewSkipPVTracker(), + BackedUpItems: NewBackedUpItemsMap(), + WorkerPool: itemBlockPool, + } + + p := new(resourcepolicies.Policies) + inputPolicy := &resourcepolicies.ResourcePolicies{ + Version: "v1", + ClusterScopedFilterPolicy: &resourcepolicies.ClusterScopedFilterPolicy{ + ResourceFilters: []resourcepolicies.ResourceFilter{ + {Kinds: []string{"pods", "invalid-cluster-kind"}}, + }, + }, + NamespacedFilterPolicies: []resourcepolicies.NamespacedFilterPolicy{ + { + Namespaces: []string{"ns1"}, + ResourceFilters: []resourcepolicies.ResourceFilter{ + {Kinds: []string{"persistentvolumes", "pods", "invalid-ns-kind"}}, + }, + }, + }, + } + require.NoError(t, p.BuildPolicy(inputPolicy)) + backupReq.ResPolicies = p + + backupFile := bytes.NewBuffer([]byte{}) + err := h.backupper.Backup(h.log, backupReq, backupFile, nil, nil, nil) + require.NoError(t, err) + + // Add test to cover error returns from resolve policies + badClusterPol := &resourcepolicies.ClusterScopedFilterPolicy{ + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"pods"}, + LabelSelector: map[string]string{"invalid/label/key": "value"}, + }, + }, + } + pBadCluster := new(resourcepolicies.Policies) + require.NoError(t, pBadCluster.BuildPolicy(&resourcepolicies.ResourcePolicies{ + Version: "v1", + ClusterScopedFilterPolicy: badClusterPol, + })) + backupReq.ResPolicies = pBadCluster + err = h.backupper.Backup(h.log, backupReq, backupFile, nil, nil, nil) + require.Error(t, err) + + badNsPol := []resourcepolicies.NamespacedFilterPolicy{ + { + Namespaces: []string{"ns1"}, + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"pods"}, + LabelSelector: map[string]string{"invalid/label/key": "value"}, + }, + }, + }, + } + pBadNs := new(resourcepolicies.Policies) + require.NoError(t, pBadNs.BuildPolicy(&resourcepolicies.ResourcePolicies{ + Version: "v1", + NamespacedFilterPolicies: badNsPol, + })) + backupReq.ResPolicies = pBadNs + err = h.backupper.Backup(h.log, backupReq, backupFile, nil, nil, nil) + require.Error(t, err) +} + +func TestGetNamespaceFilter(t *testing.T) { + // Pre-compile our globs to simulate what resolveNamespacedFilterPolicies does + teamFrontendGlob, err := glob.Compile("team-frontend-*") + require.NoError(t, err) + + teamGlob, err := glob.Compile("team-*") + require.NoError(t, err) + + // Define our filter map + filterMap := map[string]*ResolvedNamespaceFilter{ + "exact-match-ns": {CatchAllFilter: &ResolvedResourceFilter{}}, + "team-frontend-*": {CatchAllFilter: &ResolvedResourceFilter{}}, + "team-*": {CatchAllFilter: &ResolvedResourceFilter{}}, + } + + // Create request with patterns in a specific order (first-match semantics) + req := &Request{ + NamespacedFilterMap: filterMap, + NamespacedFilterPatterns: []NamespacedFilterPattern{ + {Pattern: "team-frontend-*", Compiled: teamFrontendGlob}, // Most specific first + {Pattern: "team-*", Compiled: teamGlob}, // Broader second + }, + } + + tests := []struct { + name string + namespace string + expectNil bool + expectMatched string // The pattern or exact string that should match + }{ + { + name: "exact string match bypasses glob matching", + namespace: "exact-match-ns", + expectNil: false, + expectMatched: "exact-match-ns", + }, + { + name: "reviewer requested: glob pattern matching", + namespace: "team-backend-prod", + expectNil: false, + expectMatched: "team-*", + }, + { + name: "reviewer requested: first-match ordering", + namespace: "team-frontend-prod", + expectNil: false, + expectMatched: "team-frontend-*", // Should match this because it's first in NamespacedFilterPatterns + }, + { + name: "no match returns nil", + namespace: "unrelated-ns", + expectNil: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // First call (populates cache) + result := req.GetNamespaceFilter(tt.namespace) + + if tt.expectNil { + assert.Nil(t, result) + + // Verify negative cache + val, ok := req.NamespaceFilterCache.Load(tt.namespace) + assert.True(t, ok) + assert.Nil(t, val) + } else { + assert.NotNil(t, result) + // Ensure the returned filter points to the correct reference in our map + assert.Same(t, filterMap[tt.expectMatched], result) + + // Verify positive cache + val, ok := req.NamespaceFilterCache.Load(tt.namespace) + assert.True(t, ok) + assert.Same(t, filterMap[tt.expectMatched], val) + } + + // Second call (hits cache) + result2 := req.GetNamespaceFilter(tt.namespace) + assert.Same(t, result, result2) + }) + } +} + +func TestGetNamespaceFilter_CacheBypass(t *testing.T) { + req := &Request{ + NamespacedFilterMap: make(map[string]*ResolvedNamespaceFilter), + } + + cachedFilter := &ResolvedNamespaceFilter{} + req.NamespaceFilterCache.Store("cached-ns", cachedFilter) + + // Since NamespacedFilterMap is empty, this would normally return nil, + // but the cache should return our cachedFilter. + assert.Same(t, cachedFilter, req.GetNamespaceFilter("cached-ns")) +} diff --git a/pkg/backup/item_backupper.go b/pkg/backup/item_backupper.go index 2ca266e91..f43888252 100644 --- a/pkg/backup/item_backupper.go +++ b/pkg/backup/item_backupper.go @@ -24,7 +24,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -142,6 +142,42 @@ func (ib *itemBackupper) itemInclusionChecks(log logrus.FieldLogger, mustInclude log.Info("Excluding item because resource is excluded") return false } + + // Per-kind name filter from ResourcePolicy namespace filter. + if namespace != "" { + if nsFilter := ib.backupRequest.GetNamespaceFilter(namespace); nsFilter != nil { + rf := nsFilter.ResourceFilterMap[groupResource.String()] + if rf == nil { + rf = nsFilter.CatchAllFilter + } + // When rf is still nil the item's kind is not listed in the namespace filter and + // there is no catch-all entry. This is an intentional permissive passthrough: + // plugin-injected additional items (returned by BackupItemAction) must be able + // to reach the archive even when their kind was not explicitly listed in + // namespacedFilterPolicies, because excluding them at Stage 2 would break backup + // completeness. For example, a CSI plugin may inject a VolumeSnapshotContent + // as an additional item that is required for a correct restore. Kind-level + // exclusion for the primary collection pass is enforced earlier in + // item_collector.go (Stage 1). + if rf != nil && rf.NameIE != nil { + if !rf.NameIE.ShouldInclude(metadata.GetName()) { + log.Infof("Excluding item: name does not match resource filter for kind %s", + groupResource) + return false + } + } + } + } else { + // Cluster-scoped resource name filter + if ib.backupRequest.ClusterScopedFilterMap != nil { + if rf, ok := ib.backupRequest.ClusterScopedFilterMap[groupResource.String()]; ok && rf.NameIE != nil { + if !rf.NameIE.ShouldInclude(metadata.GetName()) { + log.Infof("Excluding item: name does not match clusterScopedFilterPolicy for kind %s", groupResource) + return false + } + } + } + } } if metadata.GetDeletionTimestamp() != nil { diff --git a/pkg/backup/item_backupper_test.go b/pkg/backup/item_backupper_test.go index be91b6d34..f3769a998 100644 --- a/pkg/backup/item_backupper_test.go +++ b/pkg/backup/item_backupper_test.go @@ -21,20 +21,20 @@ import ( "testing" "github.com/sirupsen/logrus" - "github.com/stretchr/testify/require" - "k8s.io/apimachinery/pkg/runtime/schema" - ctrlfake "sigs.k8s.io/controller-runtime/pkg/client/fake" - - "github.com/vmware-tanzu/velero/internal/resourcepolicies" - "github.com/vmware-tanzu/velero/pkg/kuberesource" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrlfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" + "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/util/collections" ) func Test_resourceKey(t *testing.T) { @@ -494,3 +494,284 @@ func TestUnTrackSkippedPV_PendingLostPVC(t *testing.T) { }) } } + +// includeAllIE is a minimal IncludesExcludesInterface that includes everything — +// used in tests where the global resource include/exclude logic is not under test. +type includeAllIE struct{} + +func (includeAllIE) ShouldInclude(string) bool { return true } +func (includeAllIE) ShouldExclude(string) bool { return false } + +// makeTestUnstructured creates an unstructured object with the given namespace, name, and labels. +func makeTestUnstructured(namespace, name string, labels map[string]string) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetNamespace(namespace) + obj.SetName(name) + if labels != nil { + obj.SetLabels(labels) + } + return obj +} + +// makeNameIE creates an IncludesExcludes that includes only the given glob patterns. +func makeNameIE(include ...string) *collections.IncludesExcludes { + ie := collections.NewIncludesExcludes() + ie.Includes(include...) + return ie +} + +// newTestItemBackupper builds a minimal itemBackupper suitable for itemInclusionChecks tests. +func newTestItemBackupper(req *Request) *itemBackupper { + return &itemBackupper{ + backupRequest: req, + } +} + +// baseRequest returns a Request with NamespaceIncludesExcludes and ResourceIncludesExcludes +// configured to include everything, so only the filter-map logic under test is exercised. +func baseRequest() *Request { + return &Request{ + Backup: builder.ForBackup("velero", "test-backup").Result(), + NamespaceIncludesExcludes: collections.NewNamespaceIncludesExcludes().Includes("*"), + ResourceIncludesExcludes: includeAllIE{}, + SkippedPVTracker: NewSkipPVTracker(), + } +} + +var configMapsGR = schema.GroupResource{Group: "", Resource: "configmaps"} +var clusterRolesGR = schema.GroupResource{Group: "rbac.authorization.k8s.io", Resource: "clusterroles"} + +// TestItemInclusionChecks_ExcludeLabel_OverridesNamespaceFilter verifies that +// velero.io/exclude-from-backup=true takes precedence over a namespacedFilterPolicies +// entry that would otherwise include the resource. +func TestItemInclusionChecks_ExcludeLabel_OverridesNamespaceFilter(t *testing.T) { + req := baseRequest() + req.NamespacedFilterMap = map[string]*ResolvedNamespaceFilter{ + "ns-a": { + ResourceFilterMap: map[string]*ResolvedResourceFilter{ + configMapsGR.String(): {}, // include all ConfigMaps in ns-a + }, + }, + } + req.NamespacedFilterPatterns = []NamespacedFilterPattern{} + + ib := newTestItemBackupper(req) + log := logrus.New() + + obj := makeTestUnstructured("ns-a", "my-config", map[string]string{ + velerov1api.ExcludeFromBackupLabel: "true", + }) + + result := ib.itemInclusionChecks(log, false, obj, obj, configMapsGR) + assert.False(t, result, "resource with exclude-from-backup=true must be excluded even when matched by namespacedFilterPolicies") +} + +// TestItemInclusionChecks_ExcludeLabel_OverridesCatchAll verifies that +// velero.io/exclude-from-backup=true takes precedence over the catch-all filter. +func TestItemInclusionChecks_ExcludeLabel_OverridesCatchAll(t *testing.T) { + catchAllFilter := &ResolvedResourceFilter{} // include everything via catch-all + req := baseRequest() + req.NamespacedFilterMap = map[string]*ResolvedNamespaceFilter{ + "ns-a": { + ResourceFilterMap: map[string]*ResolvedResourceFilter{}, + CatchAllFilter: catchAllFilter, + }, + } + req.NamespacedFilterPatterns = []NamespacedFilterPattern{} + + ib := newTestItemBackupper(req) + log := logrus.New() + + obj := makeTestUnstructured("ns-a", "my-config", map[string]string{ + velerov1api.ExcludeFromBackupLabel: "true", + }) + + result := ib.itemInclusionChecks(log, false, obj, obj, configMapsGR) + assert.False(t, result, "resource with exclude-from-backup=true must be excluded even when matched by catch-all filter") +} + +// TestItemInclusionChecks_ExcludeLabel_OverridesClusterScopedFilter verifies that +// velero.io/exclude-from-backup=true takes precedence over clusterScopedFilterPolicy. +func TestItemInclusionChecks_ExcludeLabel_OverridesClusterScopedFilter(t *testing.T) { + req := baseRequest() + req.ClusterScopedFilterMap = map[string]*ResolvedResourceFilter{ + clusterRolesGR.String(): {}, // include all ClusterRoles + } + + ib := newTestItemBackupper(req) + log := logrus.New() + + // Cluster-scoped object: no namespace + obj := makeTestUnstructured("", "my-role", map[string]string{ + velerov1api.ExcludeFromBackupLabel: "true", + }) + + result := ib.itemInclusionChecks(log, false, obj, obj, clusterRolesGR) + assert.False(t, result, "cluster-scoped resource with exclude-from-backup=true must be excluded even when in clusterScopedFilterPolicy") +} + +// TestItemInclusionChecks_ClusterScoped_NotInFilterMap_PassesThrough verifies that +// a dynamically injected cluster-scoped resource NOT listed in ClusterScopedFilterMap +// passes through itemInclusionChecks (permissive passthrough at Stage 2). +func TestItemInclusionChecks_ClusterScoped_NotInFilterMap_PassesThrough(t *testing.T) { + req := baseRequest() + req.ClusterScopedFilterMap = map[string]*ResolvedResourceFilter{ + clusterRolesGR.String(): {}, // only ClusterRoles are listed + } + + ib := newTestItemBackupper(req) + log := logrus.New() + + // VolumeSnapshotClass is NOT in the filter map + volumeSnapshotClassGR := schema.GroupResource{Group: "snapshot.storage.k8s.io", Resource: "volumesnapshotclasses"} + obj := makeTestUnstructured("", "standard", nil) + + result := ib.itemInclusionChecks(log, false, obj, obj, volumeSnapshotClassGR) + assert.True(t, result, "cluster-scoped resource not in ClusterScopedFilterMap must pass through (permissive Stage 2 for unlisted kinds)") +} + +// TestItemInclusionChecks_ClusterScoped_NameIE_Matching verifies that a cluster-scoped +// resource listed in ClusterScopedFilterMap with a NameIE filter is included/excluded +// based on its name. +func TestItemInclusionChecks_ClusterScoped_NameIE_Matching(t *testing.T) { + req := baseRequest() + req.ClusterScopedFilterMap = map[string]*ResolvedResourceFilter{ + clusterRolesGR.String(): { + NameIE: makeNameIE("my-app-*"), + }, + } + + ib := newTestItemBackupper(req) + log := logrus.New() + + // Matching name + matching := makeTestUnstructured("", "my-app-reader", nil) + assert.True(t, ib.itemInclusionChecks(log, false, matching, matching, clusterRolesGR), + "ClusterRole matching name pattern must be included") + + // Non-matching name + nonMatching := makeTestUnstructured("", "other-role", nil) + assert.False(t, ib.itemInclusionChecks(log, false, nonMatching, nonMatching, clusterRolesGR), + "ClusterRole not matching name pattern must be excluded") +} + +// TestItemInclusionChecks_GlobalExclusion_OverridesNamespaceFilter verifies that +// a resource kind globally excluded by includeExcludePolicy is rejected at Stage 2 +// even when a namespacedFilterPolicies entry lists that kind. The global +// ResourceIncludesExcludes.ShouldInclude check fires before the per-namespace filter. +func TestItemInclusionChecks_GlobalExclusion_OverridesNamespaceFilter(t *testing.T) { + // excludeSecretsIE excludes "secrets" globally, includes everything else. + excludeSecretsIE := &excludeResourceIE{excluded: "secrets"} + + req := &Request{ + Backup: builder.ForBackup("velero", "test-backup").Result(), + NamespaceIncludesExcludes: collections.NewNamespaceIncludesExcludes().Includes("*"), + ResourceIncludesExcludes: excludeSecretsIE, + SkippedPVTracker: NewSkipPVTracker(), + // namespacedFilterPolicies says to back up Secrets in ns-a + NamespacedFilterMap: map[string]*ResolvedNamespaceFilter{ + "ns-a": { + ResourceFilterMap: map[string]*ResolvedResourceFilter{ + "secrets.": {}, // Secret listed in per-namespace filter + }, + }, + }, + NamespacedFilterPatterns: []NamespacedFilterPattern{}, + } + + ib := newTestItemBackupper(req) + log := logrus.New() + + secretsGR := schema.GroupResource{Group: "", Resource: "secrets"} + obj := makeTestUnstructured("ns-a", "my-secret", nil) + + result := ib.itemInclusionChecks(log, false, obj, obj, secretsGR) + assert.False(t, result, + "Secret must be excluded because it is globally excluded by ResourceIncludesExcludes, "+ + "even though namespacedFilterPolicies lists it") +} + +// TestItemInclusionChecks_PluginItem_UnlistedKind_NoCatchAll_PassesThrough verifies the +// intentional permissive passthrough at Stage 2 for plugin-injected additional items. +// When a namespace has a namespacedFilterPolicies entry but the item's kind is not listed +// in that policy and there is no catch-all entry, itemInclusionChecks must still allow +// the item through. +// +// Rationale: plugin-injected additional items (returned by BackupItemAction) must be able +// to reach the archive even when their kind was not explicitly listed in the filter policy, +// because rejecting them here would break backup completeness. For example, a CSI plugin +// may inject a VolumeSnapshotContent that is required for a correct restore. +// Kind-level exclusion for the primary collection pass is enforced at Stage 1 in +// item_collector.go, not at Stage 2 here. +func TestItemInclusionChecks_PluginItem_UnlistedKind_NoCatchAll_PassesThrough(t *testing.T) { + req := baseRequest() + // Namespace filter only lists ConfigMaps; Secrets are not listed and there is no catch-all. + req.NamespacedFilterMap = map[string]*ResolvedNamespaceFilter{ + "ns-a": { + ResourceFilterMap: map[string]*ResolvedResourceFilter{ + configMapsGR.String(): {}, + }, + CatchAllFilter: nil, + }, + } + req.NamespacedFilterPatterns = []NamespacedFilterPattern{} + + ib := newTestItemBackupper(req) + log := logrus.New() + + secretsGR := schema.GroupResource{Group: "", Resource: "secrets"} + obj := makeTestUnstructured("ns-a", "plugin-injected-secret", nil) + + result := ib.itemInclusionChecks(log, false, obj, obj, secretsGR) + assert.True(t, result, + "plugin-injected additional item of an unlisted kind must pass through Stage 2 "+ + "even when its namespace has a namespacedFilterPolicies entry with no catch-all; "+ + "kind exclusion is enforced at Stage 1 (item_collector.go), not here") +} + +// TestItemInclusionChecks_PluginItem_UnlistedKind_WithCatchAll_PassesThrough verifies that +// a plugin-injected additional item of a kind not listed in the namespace filter also passes +// through Stage 2 when a catch-all entry is present. The catch-all is validated to never +// carry a NameIE (names/excludedNames are prohibited on catch-all entries), so the name +// check is always a no-op for catch-all-matched items and the item is included. +func TestItemInclusionChecks_PluginItem_UnlistedKind_WithCatchAll_PassesThrough(t *testing.T) { + req := baseRequest() + // Namespace filter lists ConfigMaps explicitly; a catch-all covers everything else. + // The catch-all has no NameIE — this is enforced by validation. + req.NamespacedFilterMap = map[string]*ResolvedNamespaceFilter{ + "ns-a": { + ResourceFilterMap: map[string]*ResolvedResourceFilter{ + configMapsGR.String(): {}, + }, + CatchAllFilter: &ResolvedResourceFilter{ + // NameIE intentionally nil: validation forbids names/excludedNames on catch-all + NameIE: nil, + }, + }, + } + req.NamespacedFilterPatterns = []NamespacedFilterPattern{} + + ib := newTestItemBackupper(req) + log := logrus.New() + + secretsGR := schema.GroupResource{Group: "", Resource: "secrets"} + obj := makeTestUnstructured("ns-a", "plugin-injected-secret", nil) + + result := ib.itemInclusionChecks(log, false, obj, obj, secretsGR) + assert.True(t, result, + "plugin-injected additional item matched by catch-all must pass through Stage 2; "+ + "the catch-all has no NameIE so the name check is a no-op") +} + +// excludeResourceIE is an IncludesExcludesInterface that excludes a single resource +// type and includes everything else. Used to simulate includeExcludePolicy global exclusions. +type excludeResourceIE struct { + excluded string +} + +func (e *excludeResourceIE) ShouldInclude(typeName string) bool { + return typeName != e.excluded +} +func (e *excludeResourceIE) ShouldExclude(typeName string) bool { + return typeName == e.excluded +} diff --git a/pkg/backup/item_collector.go b/pkg/backup/item_collector.go index 3dace71fd..f4c712921 100644 --- a/pkg/backup/item_collector.go +++ b/pkg/backup/item_collector.go @@ -24,7 +24,7 @@ import ( "sort" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -462,6 +462,7 @@ func (r *itemCollector) getResourceItems( } clusterScoped := !resource.Namespaced + namespacesToList := getNamespacesToList(r.backupRequest.NamespaceIncludesExcludes) // If we get here, we're backing up something other than namespaces @@ -472,6 +473,16 @@ func (r *itemCollector) getResourceItems( var items []*kubernetesResource for _, namespace := range namespacesToList { + // Check per-namespace resource type filter from ResourcePolicy + if nsFilter := r.backupRequest.GetNamespaceFilter(namespace); nsFilter != nil { + _, hasSpecific := nsFilter.ResourceFilterMap[gr.String()] + if !hasSpecific && nsFilter.CatchAllFilter == nil { + log.Debugf("Skipping resource %s in namespace %s: not in resourceFilters", + gr, namespace) + continue + } + } + unstructuredItems, err := r.listResourceByLabelsPerNamespace( namespace, gr, gv, resource, log) if err != nil { @@ -527,13 +538,55 @@ func (r *itemCollector) listResourceByLabelsPerNamespace( return nil, err } + // 1. Start with global selectors (existing default behavior) var orLabelSelectors []string + var labelSelector string + if r.backupRequest.Spec.OrLabelSelectors != nil { for _, s := range r.backupRequest.Spec.OrLabelSelectors { orLabelSelectors = append(orLabelSelectors, metav1.FormatLabelSelector(s)) } - } else { - orLabelSelectors = []string{} + } + if selector := r.backupRequest.Spec.LabelSelector; selector != nil { + labelSelector = metav1.FormatLabelSelector(selector) + } + + // 2. Apply fine-grained filter overrides if applicable + if !resource.Namespaced && r.backupRequest.ClusterScopedFilterMap != nil { + if rf := r.backupRequest.ClusterScopedFilterMap[gr.String()]; rf != nil { + // Overwrite global selectors with specific filter + orLabelSelectors = nil + labelSelector = "" + if rf.LabelSelector != nil { + labelSelector = rf.LabelSelector.String() + } + for _, s := range rf.OrLabelSelectors { + orLabelSelectors = append(orLabelSelectors, s.String()) + } + } + // ClusterScopedFilterPolicy: If rf == nil, it intentionally falls back to the global selectors initialized above + } else if nsFilter := r.backupRequest.GetNamespaceFilter(namespace); nsFilter != nil { + rf := nsFilter.ResourceFilterMap[gr.String()] + if rf == nil { + rf = nsFilter.CatchAllFilter + } + + if rf != nil { + // Overwrite global selectors with specific filter + orLabelSelectors = nil + labelSelector = "" + if rf.LabelSelector != nil { + labelSelector = rf.LabelSelector.String() + } + for _, s := range rf.OrLabelSelectors { + orLabelSelectors = append(orLabelSelectors, s.String()) + } + } else { + // NamespacedFilterPolicies: namespacedFilterPolicies acts as an exclusive allowlist. + // If neither a kind-specific entry nor a catch-all entry exists, skip the kind. + logger.Debug("Skipping resource kind for namespace as it is not present in the namespace filter policy") + return nil, nil + } } logger.Info("Listing items") @@ -553,11 +606,6 @@ func (r *itemCollector) listResourceByLabelsPerNamespace( return nil, err } - var labelSelector string - if selector := r.backupRequest.Spec.LabelSelector; selector != nil { - labelSelector = metav1.FormatLabelSelector(selector) - } - // Listing items for labelSelector (singular) if len(orLabelSelectors) == 0 { unstructuredItems, err = r.listItemsForLabel( diff --git a/pkg/backup/item_collector_test.go b/pkg/backup/item_collector_test.go index 54e2ed4c3..084d5b5ff 100644 --- a/pkg/backup/item_collector_test.go +++ b/pkg/backup/item_collector_test.go @@ -26,7 +26,9 @@ import ( corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" @@ -279,8 +281,9 @@ func TestItemCollectorBackupNamespaces(t *testing.T) { Backup: tc.backup, NamespaceIncludesExcludes: tc.ie, }, - dynamicFactory: factory, - dir: tempDir, + dynamicFactory: factory, + discoveryHelper: test.NewFakeDiscoveryHelper(true, nil), + dir: tempDir, } if tc.converter == nil { @@ -305,3 +308,140 @@ func TestItemCollectorBackupNamespaces(t *testing.T) { }) } } + +// TestNamespacedFilterMap_GlobalExclusionPrecedence verifies the precedence rule: +// ResourceIncludesExcludes (set by includeExcludePolicy) is checked before the +// NamespacedFilterMap. This is enforced at both Stage 1 (item_collector.go line ~430) +// and Stage 2 (item_backupper.go itemInclusionChecks). The unit below confirms that +// GetNamespaceFilter still returns a filter for the namespace — it is the caller's +// responsibility to check ResourceIncludesExcludes first, which item_collector does. +// +// Full coverage of the Stage 2 enforcement is in item_backupper_test.go +// TestItemInclusionChecks_GlobalExclusion_OverridesNamespaceFilter. +func TestNamespacedFilterMap_GlobalExclusionPrecedence(t *testing.T) { + req := &Request{ + Backup: builder.ForBackup("velero", "test-backup").Result(), + NamespaceIncludesExcludes: collections.NewNamespaceIncludesExcludes().Includes("ns-a"), + NamespacedFilterMap: map[string]*ResolvedNamespaceFilter{ + "ns-a": { + ResourceFilterMap: map[string]*ResolvedResourceFilter{ + "secrets.": {}, + }, + }, + }, + NamespacedFilterPatterns: []NamespacedFilterPattern{}, + } + + // GetNamespaceFilter returns the filter regardless of global exclusions. + // The caller (item_collector) is responsible for checking ResourceIncludesExcludes first. + nsFilter := req.GetNamespaceFilter("ns-a") + require.NotNil(t, nsFilter, "GetNamespaceFilter should return a filter for ns-a") + _, hasSecrets := nsFilter.ResourceFilterMap["secrets."] + assert.True(t, hasSecrets, "ns-a filter should list secrets GR") + + // When a global excludeAllIE is set, item_collector would return nil before consulting the map. + // This is verified by the Stage 1 check: ShouldInclude("secrets.") == false → skip. + ie := &excludeAllIE{} + assert.False(t, ie.ShouldInclude("secrets."), + "global exclusion must reject secrets before the per-namespace filter is consulted") +} + +// excludeAllIE is an IncludesExcludesInterface that excludes every resource kind. +type excludeAllIE struct{} + +func (excludeAllIE) ShouldInclude(string) bool { return false } +func (excludeAllIE) ShouldExclude(string) bool { return true } + +func TestGetResourceItems(t *testing.T) { + tests := []struct { + name string + namespaces []string + clusterScopedFilterMap map[string]*ResolvedResourceFilter + namespacedFilterMap map[string]*ResolvedNamespaceFilter + resource metav1.APIResource + gr schema.GroupResource + }{ + { + name: "cluster scoped resource with filter", + namespaces: []string{""}, + resource: metav1.APIResource{ + Name: "persistentvolumes", + Namespaced: false, + }, + gr: schema.GroupResource{Resource: "persistentvolumes"}, + clusterScopedFilterMap: map[string]*ResolvedResourceFilter{ + "persistentvolumes": { + LabelSelector: labels.Set{"app": "foo"}.AsSelector(), + }, + }, + }, + { + name: "namespace scoped resource with filter", + namespaces: []string{"ns1"}, + resource: metav1.APIResource{ + Name: "pods", + Namespaced: true, + }, + gr: schema.GroupResource{Resource: "pods"}, + namespacedFilterMap: map[string]*ResolvedNamespaceFilter{ + "ns1": { + ResourceFilterMap: map[string]*ResolvedResourceFilter{ + "pods": { + LabelSelector: labels.Set{"app": "bar"}.AsSelector(), + }, + }, + }, + }, + }, + { + name: "namespace scoped resource skipped due to no filter match", + namespaces: []string{"ns1"}, + resource: metav1.APIResource{ + Name: "secrets", + Namespaced: true, + }, + gr: schema.GroupResource{Resource: "secrets"}, + namespacedFilterMap: map[string]*ResolvedNamespaceFilter{ + "ns1": { + ResourceFilterMap: map[string]*ResolvedResourceFilter{ + "pods": { + LabelSelector: labels.Set{"app": "bar"}.AsSelector(), + }, + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dc := &test.FakeDynamicClient{} + dc.On("List", mock.Anything).Return(&unstructured.UnstructuredList{}, nil) + + factory := &test.FakeDynamicFactory{} + factory.On("ClientForGroupVersionResource", mock.Anything, mock.Anything, mock.Anything).Return(dc, nil) + + req := &Request{ + Backup: builder.ForBackup("velero", "backup").Result(), + ClusterScopedFilterMap: tc.clusterScopedFilterMap, + NamespacedFilterMap: tc.namespacedFilterMap, + ResourceIncludesExcludes: includeAllIE{}, + } + if len(tc.namespaces) > 0 && tc.namespaces[0] != "" { + req.NamespaceIncludesExcludes = collections.NewNamespaceIncludesExcludes().Includes(tc.namespaces...) + } else { + req.NamespaceIncludesExcludes = collections.NewNamespaceIncludesExcludes().Includes("*") + } + + r := &itemCollector{ + backupRequest: req, + dynamicFactory: factory, + discoveryHelper: test.NewFakeDiscoveryHelper(true, nil), + log: test.NewLogger(), + } + + _, err := r.getResourceItems(test.NewLogger(), schema.GroupVersion{}, tc.resource, nil) + assert.NoError(t, err) + }) + } +} diff --git a/pkg/backup/itemblock.go b/pkg/backup/itemblock.go index dee553f72..4619e23aa 100644 --- a/pkg/backup/itemblock.go +++ b/pkg/backup/itemblock.go @@ -20,7 +20,7 @@ import ( "encoding/json" "os" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/backup/request.go b/pkg/backup/request.go index ca2e638c2..7ace38125 100644 --- a/pkg/backup/request.go +++ b/pkg/backup/request.go @@ -100,6 +100,10 @@ type Request struct { // NamespacedFilterPatterns preserves the order of patterns for first-match semantics // and caches pre-compiled globs to avoid repeated compilation in the hot path. NamespacedFilterPatterns []NamespacedFilterPattern + + // NamespaceFilterCache memoizes the resolved filter for a given namespace. + // sync.Map is used because item backuppers access this concurrently. + NamespaceFilterCache sync.Map } // NamespacedFilterPattern pairs a namespace pattern string with its pre-compiled @@ -149,22 +153,37 @@ func (r *Request) StopWorkerPool() { // GetNamespaceFilter returns the resolved filter for a namespace, or nil // if the namespace should use global filters. Uses first-match semantics -// when multiple patterns could match the same namespace. +// when multiple patterns could match the same namespace, but exact matches +// always take precedence over glob patterns regardless of definition order. func (r *Request) GetNamespaceFilter(namespace string) *ResolvedNamespaceFilter { if r.NamespacedFilterMap == nil { return nil } - // First check for exact match + // 1. Check the concurrent cache first + if val, ok := r.NamespaceFilterCache.Load(namespace); ok { + if val == nil { + return nil + } + return val.(*ResolvedNamespaceFilter) + } + + // 2. Check for exact match first if f, ok := r.NamespacedFilterMap[namespace]; ok { + r.NamespaceFilterCache.Store(namespace, f) return f } - // Walk patterns in definition order using pre-compiled globs (no allocation per call) + // 3. Walk patterns in definition order using pre-compiled globs for _, p := range r.NamespacedFilterPatterns { if p.Compiled != nil && p.Compiled.Match(namespace) { - return r.NamespacedFilterMap[p.Pattern] + filter := r.NamespacedFilterMap[p.Pattern] + r.NamespaceFilterCache.Store(namespace, filter) + return filter } } + + // 4. Cache the miss + r.NamespaceFilterCache.Store(namespace, nil) return nil } diff --git a/pkg/builder/backup_builder.go b/pkg/builder/backup_builder.go index d5b955e43..0553116a4 100644 --- a/pkg/builder/backup_builder.go +++ b/pkg/builder/backup_builder.go @@ -321,6 +321,11 @@ func (b *BackupBuilder) ParallelFilesUpload(parallel int) *BackupBuilder { return b } +func (b *BackupBuilder) BackupType(backupType velerov1api.BackupType) *BackupBuilder { + b.object.Spec.BackupType = backupType + return b +} + // WithStatus sets the Backup's status. func (b *BackupBuilder) WithStatus(status velerov1api.BackupStatus) *BackupBuilder { b.object.Status = status diff --git a/pkg/builder/restore_builder.go b/pkg/builder/restore_builder.go index bad4327e9..22e880a98 100644 --- a/pkg/builder/restore_builder.go +++ b/pkg/builder/restore_builder.go @@ -19,6 +19,7 @@ package builder import ( "time" + corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -171,3 +172,12 @@ func (b *RestoreBuilder) ItemOperationTimeout(timeout time.Duration) *RestoreBui b.object.Spec.ItemOperationTimeout.Duration = timeout return b } + +// ResourcePoliciesConfigmap sets the Restore's resource policies configmap. +func (b *RestoreBuilder) ResourcePoliciesConfigmap(name string) *RestoreBuilder { + b.object.Spec.ResourcePolicy = &corev1api.TypedLocalObjectReference{ + Kind: "configmap", + Name: name, + } + return b +} diff --git a/pkg/builder/restore_builder_test.go b/pkg/builder/restore_builder_test.go new file mode 100644 index 000000000..b45bd80c6 --- /dev/null +++ b/pkg/builder/restore_builder_test.go @@ -0,0 +1,36 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRestoreBuilder_ResourcePoliciesConfigmap(t *testing.T) { + restore := ForRestore("velero", "my-restore"). + ResourcePoliciesConfigmap("my-policy-cm"). + Result() + + assert.Equal(t, "velero", restore.Namespace) + assert.Equal(t, "my-restore", restore.Name) + assert.NotNil(t, restore.Spec.ResourcePolicy) + assert.Equal(t, "configmap", restore.Spec.ResourcePolicy.Kind) + assert.Equal(t, "my-policy-cm", restore.Spec.ResourcePolicy.Name) + assert.Equal(t, (*string)(nil), restore.Spec.ResourcePolicy.APIGroup) +} diff --git a/pkg/cbtservice/csi_service_impl.go b/pkg/cbtservice/csi_service_impl.go index 8918d36fa..4d0ea3fca 100644 --- a/pkg/cbtservice/csi_service_impl.go +++ b/pkg/cbtservice/csi_service_impl.go @@ -111,8 +111,8 @@ func (s *ServiceImpl) GetChangedBlocks(ctx context.Context, snapshot string, cha } args := iterator.Args{ - SnapshotName: snapshot, - PrevSnapshotName: changeID, + SnapshotName: snapshot, + PrevSnapshotID: changeID, Emitter: &emitterImpl{ logger: s.logger, recordCallBack: record, diff --git a/pkg/cbtservice/csi_service_impl_test.go b/pkg/cbtservice/csi_service_impl_test.go index 6ecad0850..cb6b311a9 100644 --- a/pkg/cbtservice/csi_service_impl_test.go +++ b/pkg/cbtservice/csi_service_impl_test.go @@ -234,7 +234,7 @@ func TestServiceImplGetChangedBlocks(t *testing.T) { require.NoError(t, err) assert.Equal(t, "snap-2", capturedArgs.SnapshotName) - assert.Equal(t, "snap-1", capturedArgs.PrevSnapshotName) + assert.Equal(t, "snap-1", capturedArgs.PrevSnapshotID) assert.Equal(t, "velero-ns", capturedArgs.Namespace) assert.Equal(t, iterator.DefaultTokenExpirySeconds, capturedArgs.TokenExpirySecs) assert.Zero(t, capturedArgs.MaxResults) diff --git a/pkg/client/client.go b/pkg/client/client.go index dccdc05fc..39cdc9141 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -20,7 +20,7 @@ import ( "fmt" "runtime" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" diff --git a/pkg/client/config.go b/pkg/client/config.go index 687c303e7..2a96e3467 100644 --- a/pkg/client/config.go +++ b/pkg/client/config.go @@ -23,7 +23,7 @@ import ( "strconv" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" ) const ( diff --git a/pkg/client/factory.go b/pkg/client/factory.go index 51dfb62c3..17e2a243a 100644 --- a/pkg/client/factory.go +++ b/pkg/client/factory.go @@ -27,8 +27,8 @@ import ( k8scheme "k8s.io/client-go/kubernetes/scheme" kbclient "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" diff --git a/pkg/cmd/cli/backup/create.go b/pkg/cmd/cli/backup/create.go index 31564aae8..5e18f468f 100644 --- a/pkg/cmd/cli/backup/create.go +++ b/pkg/cmd/cli/backup/create.go @@ -108,6 +108,7 @@ type CreateOptions struct { ResPoliciesConfigmap string client kbclient.WithWatch ParallelFilesUpload int + BackupType string } func NewCreateOptions() *CreateOptions { @@ -156,6 +157,7 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) { flags.StringVar(&o.ResPoliciesConfigmap, "resource-policies-configmap", "", "Reference to the resource policies configmap that backup should use") flags.StringVar(&o.DataMover, "data-mover", "", "Specify the data mover to be used by the backup. If the parameter is not set or set as 'velero', the built-in data mover will be used") flags.IntVar(&o.ParallelFilesUpload, "parallel-files-upload", 0, "Number of files uploads simultaneously when running a backup. This is only applicable for the kopia uploader") + flags.StringVar(&o.BackupType, "backup-type", "", "Specify how volume data is backed up, with possible values including Full and Incremental.") } // BindWait binds the wait flag separately so it is not called by other create @@ -217,6 +219,10 @@ func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Facto } } + if err := o.validateBackupType(); err != nil { + return err + } + return nil } @@ -231,6 +237,18 @@ func (o *CreateOptions) validateFromScheduleFlag(c *cobra.Command) error { return nil } +func (o *CreateOptions) validateBackupType() error { + backupType := strings.TrimSpace(o.BackupType) + + switch backupType { + case "", "Incremental", "Full": + default: + return fmt.Errorf("invalid backup type %s - valid values are 'Incremental', and 'Full'", backupType) + } + + return nil +} + func (o *CreateOptions) Complete(args []string, f client.Factory) error { // If an explicit name is specified, use that name if len(args) > 0 { @@ -393,7 +411,8 @@ func (o *CreateOptions) BuildBackup(namespace string) (*velerov1api.Backup, erro VolumeSnapshotLocations(o.SnapshotLocations...). CSISnapshotTimeout(o.CSISnapshotTimeout). ItemOperationTimeout(o.ItemOperationTimeout). - DataMover(o.DataMover) + DataMover(o.DataMover). + BackupType(velerov1api.BackupType(o.BackupType)) if len(o.OrderedResources) > 0 { orders, err := ParseOrderedResources(o.OrderedResources) if err != nil { diff --git a/pkg/cmd/cli/backup/create_test.go b/pkg/cmd/cli/backup/create_test.go index c8fd15baa..718ab0e96 100644 --- a/pkg/cmd/cli/backup/create_test.go +++ b/pkg/cmd/cli/backup/create_test.go @@ -122,6 +122,42 @@ func TestCreateOptions_ValidateFromScheduleFlag(t *testing.T) { }) } +func TestCreateOptions_ValidateBackupType(t *testing.T) { + t.Run("valid backup types", func(t *testing.T) { + o := NewCreateOptions() + + o.BackupType = "" + err := o.validateBackupType() + require.NoError(t, err) + + o.BackupType = "Incremental" + err = o.validateBackupType() + require.NoError(t, err) + + o.BackupType = "Full" + err = o.validateBackupType() + require.NoError(t, err) + + o.BackupType = " Incremental " + err = o.validateBackupType() + require.NoError(t, err) + }) + + t.Run("invalid backup type", func(t *testing.T) { + o := NewCreateOptions() + + o.BackupType = "incremental" + err := o.validateBackupType() + require.Error(t, err) + require.Equal(t, "invalid backup type incremental - valid values are 'Incremental', and 'Full'", err.Error()) + + o.BackupType = "invalid" + err = o.validateBackupType() + require.Error(t, err) + require.Equal(t, "invalid backup type invalid - valid values are 'Incremental', and 'Full'", err.Error()) + }) +} + func TestCreateOptions_BuildBackupFromSchedule(t *testing.T) { o := NewCreateOptions() o.FromSchedule = "test" @@ -231,6 +267,7 @@ func TestCreateCommand(t *testing.T) { resPoliciesConfigmap := "cm-name-2" dataMover := "velero" parallelFilesUpload := 10 + backupType := "Incremental" flags := new(flag.FlagSet) o := NewCreateOptions() o.BindFlags(flags) @@ -260,6 +297,7 @@ func TestCreateCommand(t *testing.T) { flags.Parse([]string{"--resource-policies-configmap", resPoliciesConfigmap}) flags.Parse([]string{"--data-mover", dataMover}) flags.Parse([]string{"--parallel-files-upload", strconv.Itoa(parallelFilesUpload)}) + flags.Parse([]string{"--backup-type", backupType}) //flags.Parse([]string{"--wait"}) client := velerotest.NewFakeControllerRuntimeClient(t).(kbclient.WithWatch) @@ -310,6 +348,7 @@ func TestCreateCommand(t *testing.T) { require.Equal(t, resPoliciesConfigmap, o.ResPoliciesConfigmap) require.Equal(t, dataMover, o.DataMover) require.Equal(t, parallelFilesUpload, o.ParallelFilesUpload) + require.Equal(t, backupType, o.BackupType) //assert.Equal(t, true, o.Wait) // verify oldAndNewFilterParametersUsedTogether diff --git a/pkg/cmd/cli/backup/delete.go b/pkg/cmd/cli/backup/delete.go index 692b82dbf..f4eaf1b83 100644 --- a/pkg/cmd/cli/backup/delete.go +++ b/pkg/cmd/cli/backup/delete.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/cmd/cli/backup/download.go b/pkg/cmd/cli/backup/download.go index 8bb973ff0..e4afd216c 100644 --- a/pkg/cmd/cli/backup/download.go +++ b/pkg/cmd/cli/backup/download.go @@ -23,7 +23,7 @@ import ( "path/filepath" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" controllerclient "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/pkg/cmd/cli/backuplocation/create.go b/pkg/cmd/cli/backuplocation/create.go index 343bc790a..391c7b376 100644 --- a/pkg/cmd/cli/backuplocation/create.go +++ b/pkg/cmd/cli/backuplocation/create.go @@ -24,7 +24,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/cmd/cli/backuplocation/delete.go b/pkg/cmd/cli/backuplocation/delete.go index f2c3bcc3d..9c1e60507 100644 --- a/pkg/cmd/cli/backuplocation/delete.go +++ b/pkg/cmd/cli/backuplocation/delete.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/cmd/cli/backuplocation/set.go b/pkg/cmd/cli/backuplocation/set.go index 8aa018e29..c1b52e536 100644 --- a/pkg/cmd/cli/backuplocation/set.go +++ b/pkg/cmd/cli/backuplocation/set.go @@ -22,7 +22,7 @@ import ( "os" "path/filepath" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" diff --git a/pkg/cmd/cli/datamover/backup.go b/pkg/cmd/cli/datamover/backup.go index 718663198..f352c0aad 100644 --- a/pkg/cmd/cli/datamover/backup.go +++ b/pkg/cmd/cli/datamover/backup.go @@ -21,7 +21,7 @@ import ( "time" "github.com/bombsimon/logrusr/v3" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" corev1api "k8s.io/api/core/v1" @@ -58,6 +58,9 @@ type dataMoverBackupConfig struct { duName string resourceTimeout time.Duration cbtSAName string + changeID string + volumeID string + snapshotID string } func NewBackupCommand(f client.Factory) *cobra.Command { @@ -79,7 +82,7 @@ func NewBackupCommand(f client.Factory) *cobra.Command { logger.Infof("Starting Velero data-mover backup %s (%s)", buildinfo.Version, buildinfo.FormattedGitSHA()) f.SetBasename(fmt.Sprintf("%s-%s", c.Parent().Name(), c.Name())) - s, err := newdataMoverBackup(logger, f, config) + s, err := newDataMoverBackup(logger, f, config) if err != nil { kube.ExitPodWithMessage(logger, false, "Failed to create data mover backup, %v", err) } @@ -95,6 +98,9 @@ func NewBackupCommand(f client.Factory) *cobra.Command { command.Flags().StringVar(&config.duName, "data-upload", config.duName, "The data upload name") command.Flags().DurationVar(&config.resourceTimeout, "resource-timeout", config.resourceTimeout, "How long to wait for resource processes which are not covered by other specific timeout parameters.") command.Flags().StringVar(&config.cbtSAName, "cbt-sa-name", config.cbtSAName, "The name of the service account used by CSI's CBT service") + command.Flags().StringVar(&config.changeID, "change-id", config.changeID, "The change ID of the snapshot") + command.Flags().StringVar(&config.volumeID, "volume-id", config.volumeID, "The volume ID of the snapshot") + command.Flags().StringVar(&config.snapshotID, "snapshot-id", config.snapshotID, "The ID of the snapshot") _ = command.MarkFlagRequired("volume-path") _ = command.MarkFlagRequired("volume-mode") @@ -118,7 +124,7 @@ type dataMoverBackup struct { cbtService cbtservice.Service } -func newdataMoverBackup(logger logrus.FieldLogger, factory client.Factory, config dataMoverBackupConfig) (*dataMoverBackup, error) { +func newDataMoverBackup(logger logrus.FieldLogger, factory client.Factory, config dataMoverBackupConfig) (*dataMoverBackup, error) { ctx, cancelFunc := context.WithCancel(context.Background()) clientConfig, err := factory.ClientConfig() @@ -303,8 +309,24 @@ func (s *dataMoverBackup) createDataPathService() (dataPathService, error) { repoEnsurer := repository.NewEnsurer(s.client, s.logger, s.config.resourceTimeout) - return datamover.NewBackupMicroService(s.ctx, s.client, s.kubeClient, s.config.duName, s.namespace, s.nodeName, datapath.AccessPoint{ - ByPath: s.config.volumePath, - VolMode: uploader.PersistentVolumeMode(s.config.volumeMode), - }, s.dataPathMgr, repoEnsurer, credGetter, duInformer, s.logger), nil + return datamover.NewBackupMicroService( + s.ctx, + s.client, + s.kubeClient, + s.config.duName, + s.namespace, + s.nodeName, + datapath.AccessPoint{ + ByPath: s.config.volumePath, + VolMode: uploader.PersistentVolumeMode(s.config.volumeMode), + }, + s.dataPathMgr, + repoEnsurer, + credGetter, + duInformer, + s.config.changeID, + s.config.volumeID, + s.config.snapshotID, + s.logger, + ), nil } diff --git a/pkg/cmd/cli/datamover/restore.go b/pkg/cmd/cli/datamover/restore.go index b2efdbc34..1d3cf84f4 100644 --- a/pkg/cmd/cli/datamover/restore.go +++ b/pkg/cmd/cli/datamover/restore.go @@ -21,7 +21,7 @@ import ( "time" "github.com/bombsimon/logrusr/v3" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" corev1api "k8s.io/api/core/v1" diff --git a/pkg/cmd/cli/debug/debug.go b/pkg/cmd/cli/debug/debug.go index 400eddde1..fac49d622 100644 --- a/pkg/cmd/cli/debug/debug.go +++ b/pkg/cmd/cli/debug/debug.go @@ -25,7 +25,7 @@ import ( "path/filepath" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -191,7 +191,7 @@ func runCrashd(o *option) error { if o.verbose { logrus.SetLevel(logrus.DebugLevel) } - return exec.Execute("velero-debug-collector", bytes.NewReader(scriptBytes), o.asCrashdArgMap()) + return exec.Execute("velero-debug-collector", bytes.NewReader(scriptBytes), o.asCrashdArgMap(), false) } func kubeconfigAndContext(fs *pflag.FlagSet) (string, string) { diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 8115e1353..0df53eb32 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -23,7 +23,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -81,6 +81,7 @@ type Options struct { DefaultVolumesToFsBackup bool UploaderType string DefaultSnapshotMoveData bool + CSISnapshotEarlyFrequentPolling bool DisableInformerCache bool ScheduleSkipImmediately bool PodResources kubeutil.PodResources @@ -141,6 +142,7 @@ func (o *Options) BindFlags(flags *pflag.FlagSet) { flags.BoolVar(&o.DefaultVolumesToFsBackup, "default-volumes-to-fs-backup", o.DefaultVolumesToFsBackup, "Bool flag to configure Velero server to use pod volume file system backup by default for all volumes on all backups. Optional.") flags.StringVar(&o.UploaderType, "uploader-type", o.UploaderType, fmt.Sprintf("The type of uploader to transfer the data of pod volumes, supported value: '%s'", uploader.KopiaType)) flags.BoolVar(&o.DefaultSnapshotMoveData, "default-snapshot-move-data", o.DefaultSnapshotMoveData, "Bool flag to configure Velero server to move data by default for all snapshots supporting data movement. Optional.") + flags.BoolVar(&o.CSISnapshotEarlyFrequentPolling, "csi-snapshot-early-frequent-polling", o.CSISnapshotEarlyFrequentPolling, "Bool flag to configure Velero server to use early frequent polling by default for all CSI snapshots. Optional.") flags.BoolVar(&o.DisableInformerCache, "disable-informer-cache", o.DisableInformerCache, "Disable informer cache for Get calls on restore. With this enabled, it will speed up restore in cases where there are backup resources which already exist in the cluster, but for very large clusters this will increase velero memory usage. Default is false (don't disable). Optional.") flags.BoolVar(&o.ScheduleSkipImmediately, "schedule-skip-immediately", o.ScheduleSkipImmediately, "Skip the first scheduled backup immediately after creating a schedule. Default is false (don't skip).") flags.BoolVar(&o.NodeAgentDisableHostPath, "node-agent-disable-host-path", o.NodeAgentDisableHostPath, "Don't mount the pod volume host path to node-agent. Optional. Pod volume host path mount is required by fs-backup but could be disabled for other backup methods.") @@ -238,16 +240,17 @@ func NewInstallOptions() *Options { NodeAgentPodCPULimit: install.DefaultNodeAgentPodCPULimit, NodeAgentPodMemLimit: install.DefaultNodeAgentPodMemLimit, // Default to creating a VSL unless we're told otherwise - UseVolumeSnapshots: true, - NoDefaultBackupLocation: false, - CRDsOnly: false, - DefaultVolumesToFsBackup: false, - UploaderType: uploader.KopiaType, - DefaultSnapshotMoveData: false, - DisableInformerCache: false, - ScheduleSkipImmediately: false, - kubeletRootDir: install.DefaultKubeletRootDir, - NodeAgentDisableHostPath: false, + UseVolumeSnapshots: true, + NoDefaultBackupLocation: false, + CRDsOnly: false, + DefaultVolumesToFsBackup: false, + UploaderType: uploader.KopiaType, + DefaultSnapshotMoveData: false, + CSISnapshotEarlyFrequentPolling: false, + DisableInformerCache: false, + ScheduleSkipImmediately: false, + kubeletRootDir: install.DefaultKubeletRootDir, + NodeAgentDisableHostPath: false, } } @@ -324,6 +327,7 @@ func (o *Options) AsVeleroOptions() (*install.VeleroOptions, error) { DefaultVolumesToFsBackup: o.DefaultVolumesToFsBackup, UploaderType: o.UploaderType, DefaultSnapshotMoveData: o.DefaultSnapshotMoveData, + CSISnapshotEarlyFrequentPolling: o.CSISnapshotEarlyFrequentPolling, DisableInformerCache: o.DisableInformerCache, ScheduleSkipImmediately: o.ScheduleSkipImmediately, PodResources: o.PodResources, diff --git a/pkg/cmd/cli/nodeagent/server.go b/pkg/cmd/cli/nodeagent/server.go index 374dbcac9..287e45591 100644 --- a/pkg/cmd/cli/nodeagent/server.go +++ b/pkg/cmd/cli/nodeagent/server.go @@ -26,8 +26,8 @@ import ( "time" "github.com/bombsimon/logrusr/v3" + "github.com/cockroachdb/errors" snapshotv1client "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" "github.com/spf13/cobra" diff --git a/pkg/cmd/cli/plugin/add.go b/pkg/cmd/cli/plugin/add.go index 9ea199fa9..45a112a46 100644 --- a/pkg/cmd/cli/plugin/add.go +++ b/pkg/cmd/cli/plugin/add.go @@ -22,8 +22,8 @@ import ( "fmt" "strings" + "github.com/cockroachdb/errors" jsonpatch "github.com/evanphx/json-patch/v5" - "github.com/pkg/errors" "github.com/spf13/cobra" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/cmd/cli/plugin/helpers.go b/pkg/cmd/cli/plugin/helpers.go index 28f681993..16684a71c 100644 --- a/pkg/cmd/cli/plugin/helpers.go +++ b/pkg/cmd/cli/plugin/helpers.go @@ -19,7 +19,7 @@ package plugin import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" appsv1api "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/cmd/cli/plugin/remove.go b/pkg/cmd/cli/plugin/remove.go index ed25dc680..d9b95cb37 100644 --- a/pkg/cmd/cli/plugin/remove.go +++ b/pkg/cmd/cli/plugin/remove.go @@ -20,8 +20,8 @@ import ( "context" "encoding/json" + "github.com/cockroachdb/errors" jsonpatch "github.com/evanphx/json-patch/v5" - "github.com/pkg/errors" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/cmd/cli/podvolume/backup.go b/pkg/cmd/cli/podvolume/backup.go index ed2d6c09a..8bef9c574 100644 --- a/pkg/cmd/cli/podvolume/backup.go +++ b/pkg/cmd/cli/podvolume/backup.go @@ -21,7 +21,7 @@ import ( "time" "github.com/bombsimon/logrusr/v3" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" corev1api "k8s.io/api/core/v1" diff --git a/pkg/cmd/cli/podvolume/restore.go b/pkg/cmd/cli/podvolume/restore.go index 4c1596a04..ab6554999 100644 --- a/pkg/cmd/cli/podvolume/restore.go +++ b/pkg/cmd/cli/podvolume/restore.go @@ -21,7 +21,7 @@ import ( "time" "github.com/bombsimon/logrusr/v3" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" corev1api "k8s.io/api/core/v1" diff --git a/pkg/cmd/cli/repomantenance/maintenance.go b/pkg/cmd/cli/repomantenance/maintenance.go index 46c54f7d2..f89aba257 100644 --- a/pkg/cmd/cli/repomantenance/maintenance.go +++ b/pkg/cmd/cli/repomantenance/maintenance.go @@ -8,7 +8,7 @@ import ( "time" "github.com/bombsimon/logrusr/v3" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/pflag" diff --git a/pkg/cmd/cli/restore/create.go b/pkg/cmd/cli/restore/create.go index 2fb21433e..3f59b6a6b 100644 --- a/pkg/cmd/cli/restore/create.go +++ b/pkg/cmd/cli/restore/create.go @@ -22,7 +22,7 @@ import ( "sort" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" corev1api "k8s.io/api/core/v1" @@ -32,6 +32,7 @@ import ( kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/resourcemodifiers" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" @@ -61,7 +62,13 @@ func NewCreateCommand(f client.Factory, use string) *cobra.Command { velero restore create --from-schedule schedule-1 --allow-partially-failed # Create a restore for only persistentvolumeclaims and persistentvolumes within a backup. - velero restore create --from-backup backup-2 --include-resources persistentvolumeclaims,persistentvolumes`, + velero restore create --from-backup backup-2 --include-resources persistentvolumeclaims,persistentvolumes + +Notes: +- Global filters (--include-resources, --selector, etc.) apply to all included namespaces +- Namespace-scoped filters defined in --resource-policies-configmap refine global filters for matching namespaces (globally excluded kinds cannot be re-included) +- Fine-grained global filter policies defined in --resource-policies-configmap refine global filters for cluster-scoped resources +- Use 'velero restore describe' to view the referenced resource policies ConfigMap after restore creation`, Args: cobra.MaximumNArgs(1), Run: func(c *cobra.Command, args []string) { cmd.CheckError(o.Complete(args, f)) @@ -100,6 +107,7 @@ type CreateOptions struct { AllowPartiallyFailed flag.OptionalBool ItemOperationTimeout time.Duration ResourceModifierConfigMap string + ResourcePoliciesConfigMap string WriteSparseFiles flag.OptionalBool ParallelFilesDownload int client kbclient.WithWatch @@ -154,6 +162,8 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) { flags.StringVar(&o.ResourceModifierConfigMap, "resource-modifier-configmap", "", "Reference to the resource modifier configmap that restore will use") + flags.StringVar(&o.ResourcePoliciesConfigMap, "resource-policies-configmap", "", "Reference to the ConfigMap containing restore resource filter policies") + f = flags.VarPF(&o.WriteSparseFiles, "write-sparse-files", "", "Whether to write sparse files during restoring volumes") f.NoOptDefVal = cmd.TRUE @@ -310,6 +320,15 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { } } + var resPolicies *corev1api.TypedLocalObjectReference + + if o.ResourcePoliciesConfigMap != "" { + resPolicies = &corev1api.TypedLocalObjectReference{ + Kind: resourcepolicies.ConfigmapRefType, + Name: o.ResourcePoliciesConfigMap, + } + } + restore := &api.Restore{ ObjectMeta: metav1.ObjectMeta{ Namespace: f.Namespace(), @@ -332,6 +351,7 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { PreserveNodePorts: o.PreserveNodePorts.Value, IncludeClusterResources: o.IncludeClusterResources.Value, ResourceModifier: resModifiers, + ResourcePolicy: resPolicies, ItemOperationTimeout: metav1.Duration{ Duration: o.ItemOperationTimeout, }, diff --git a/pkg/cmd/cli/restore/create_test.go b/pkg/cmd/cli/restore/create_test.go index 8cc369dea..9a6a92608 100644 --- a/pkg/cmd/cli/restore/create_test.go +++ b/pkg/cmd/cli/restore/create_test.go @@ -77,6 +77,8 @@ func TestCreateCommand(t *testing.T) { includeClusterResources := "true" allowPartiallyFailed := "true" itemOperationTimeout := "10m0s" + resourceModifierConfigMap := "modifier-cm" + ResourcePoliciesConfigMap := "policies-cm" writeSparseFiles := "true" parallel := 2 flags := new(pflag.FlagSet) @@ -101,6 +103,8 @@ func TestCreateCommand(t *testing.T) { flags.Parse([]string{"--include-cluster-resources", includeClusterResources}) flags.Parse([]string{"--allow-partially-failed", allowPartiallyFailed}) flags.Parse([]string{"--item-operation-timeout", itemOperationTimeout}) + flags.Parse([]string{"--resource-modifier-configmap", resourceModifierConfigMap}) + flags.Parse([]string{"--resource-policies-configmap", ResourcePoliciesConfigMap}) flags.Parse([]string{"--write-sparse-files", writeSparseFiles}) flags.Parse([]string{"--parallel-files-download", "2"}) client := velerotest.NewFakeControllerRuntimeClient(t).(kbclient.WithWatch) @@ -139,6 +143,8 @@ func TestCreateCommand(t *testing.T) { require.Equal(t, includeClusterResources, o.IncludeClusterResources.String()) require.Equal(t, allowPartiallyFailed, o.AllowPartiallyFailed.String()) require.Equal(t, itemOperationTimeout, o.ItemOperationTimeout.String()) + require.Equal(t, resourceModifierConfigMap, o.ResourceModifierConfigMap) + require.Equal(t, ResourcePoliciesConfigMap, o.ResourcePoliciesConfigMap) require.Equal(t, writeSparseFiles, o.WriteSparseFiles.String()) require.Equal(t, parallel, o.ParallelFilesDownload) }) @@ -189,4 +195,37 @@ func TestCreateCommand(t *testing.T) { err := o.Validate(c, []string{}, f) require.Equal(t, "backups.velero.io \"not-exist\" not found", err.Error()) }) + + t.Run("create a restore with resource policies configmap", func(t *testing.T) { + f := &factorymocks.Factory{} + c := NewCreateCommand(f, "") + require.Equal(t, "Create a restore", c.Short) + flags := new(pflag.FlagSet) + o := NewCreateOptions() + o.BindFlags(flags) + + backupName := "backup-with-policies" + ResourcePoliciesConfigMap := "test-policies-cm" + flags.Parse([]string{"--from-backup", backupName}) + flags.Parse([]string{"--resource-policies-configmap", ResourcePoliciesConfigMap}) + + kbclient := velerotest.NewFakeControllerRuntimeClient(t).(kbclient.WithWatch) + backup := builder.ForBackup(cmdtest.VeleroNameSpace, backupName).Phase(velerov1api.BackupPhaseCompleted).Result() + require.NoError(t, kbclient.Create(t.Context(), backup, &controllerclient.CreateOptions{})) + + f.On("Namespace").Return(cmdtest.VeleroNameSpace) + f.On("KubebuilderWatchClient").Return(kbclient, nil) + + require.NoError(t, o.Complete(args, f)) + require.NoError(t, o.Validate(c, []string{}, f)) + require.NoError(t, o.Run(c, f)) + + // Verify the created restore object + createdRestore := &velerov1api.Restore{} + err := kbclient.Get(t.Context(), controllerclient.ObjectKey{Namespace: cmdtest.VeleroNameSpace, Name: name}, createdRestore) + require.NoError(t, err) + require.NotNil(t, createdRestore.Spec.ResourcePolicy) + require.Equal(t, "configmap", createdRestore.Spec.ResourcePolicy.Kind) + require.Equal(t, ResourcePoliciesConfigMap, createdRestore.Spec.ResourcePolicy.Name) + }) } diff --git a/pkg/cmd/cli/restore/delete.go b/pkg/cmd/cli/restore/delete.go index a2e70953d..51c31e1da 100644 --- a/pkg/cmd/cli/restore/delete.go +++ b/pkg/cmd/cli/restore/delete.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/cmd/cli/schedule/create.go b/pkg/cmd/cli/schedule/create.go index 47c19318f..2e4a1e8e9 100644 --- a/pkg/cmd/cli/schedule/create.go +++ b/pkg/cmd/cli/schedule/create.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" corev1api "k8s.io/api/core/v1" diff --git a/pkg/cmd/cli/schedule/delete.go b/pkg/cmd/cli/schedule/delete.go index 77b0bf883..78e8c9104 100644 --- a/pkg/cmd/cli/schedule/delete.go +++ b/pkg/cmd/cli/schedule/delete.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/cmd/cli/schedule/pause.go b/pkg/cmd/cli/schedule/pause.go index 820e887a7..41a17f384 100644 --- a/pkg/cmd/cli/schedule/pause.go +++ b/pkg/cmd/cli/schedule/pause.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/cmd/cli/serverstatus/server_status.go b/pkg/cmd/cli/serverstatus/server_status.go index ab994e2d6..da052f579 100644 --- a/pkg/cmd/cli/serverstatus/server_status.go +++ b/pkg/cmd/cli/serverstatus/server_status.go @@ -20,7 +20,7 @@ import ( "context" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/util/wait" kbclient "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/pkg/cmd/cli/snapshotlocation/create.go b/pkg/cmd/cli/snapshotlocation/create.go index db55ad834..d0f0203a8 100644 --- a/pkg/cmd/cli/snapshotlocation/create.go +++ b/pkg/cmd/cli/snapshotlocation/create.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/cmd/cli/snapshotlocation/set.go b/pkg/cmd/cli/snapshotlocation/set.go index f6b8ac368..0814bdfe7 100644 --- a/pkg/cmd/cli/snapshotlocation/set.go +++ b/pkg/cmd/cli/snapshotlocation/set.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" diff --git a/pkg/cmd/cli/uninstall/uninstall.go b/pkg/cmd/cli/uninstall/uninstall.go index 80a349c92..93e0118c6 100644 --- a/pkg/cmd/cli/uninstall/uninstall.go +++ b/pkg/cmd/cli/uninstall/uninstall.go @@ -23,7 +23,7 @@ import ( "sync" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" diff --git a/pkg/cmd/cli/version/version_test.go b/pkg/cmd/cli/version/version_test.go index 355626802..71233431f 100644 --- a/pkg/cmd/cli/version/version_test.go +++ b/pkg/cmd/cli/version/version_test.go @@ -21,7 +21,7 @@ import ( "fmt" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" kbclient "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/pkg/cmd/server/config/config.go b/pkg/cmd/server/config/config.go index e19086217..c8080da21 100644 --- a/pkg/cmd/server/config/config.go +++ b/pkg/cmd/server/config/config.go @@ -145,42 +145,43 @@ var ( ) type Config struct { - PluginDir string - MetricsAddress string - DefaultBackupLocation string // TODO(2.0) Deprecate defaultBackupLocation - BackupSyncPeriod time.Duration - PodVolumeOperationTimeout time.Duration - ResourceTerminatingTimeout time.Duration - DefaultBackupTTL time.Duration - DefaultVGSLabelKey string - StoreValidationFrequency time.Duration - DefaultCSISnapshotTimeout time.Duration - DefaultItemOperationTimeout time.Duration - ResourceTimeout time.Duration - RestoreResourcePriorities types.Priorities - DefaultVolumeSnapshotLocations flag.Map - RestoreOnly bool - DisabledControllers []string - ClientQPS float32 - ClientBurst int - ClientPageSize int - ProfilerAddress string - LogLevel *logging.LevelFlag - LogFormat *logging.FormatFlag - RepoMaintenanceFrequency time.Duration - GarbageCollectionFrequency time.Duration - ItemOperationSyncFrequency time.Duration - DefaultVolumesToFsBackup bool - UploaderType string - MaxConcurrentK8SConnections int - DefaultSnapshotMoveData bool - DisableInformerCache bool - ScheduleSkipImmediately bool - CredentialsDirectory string - BackupRepoConfig string - RepoMaintenanceJobConfig string - ItemBlockWorkerCount int - ConcurrentBackups int + PluginDir string + MetricsAddress string + DefaultBackupLocation string // TODO(2.0) Deprecate defaultBackupLocation + BackupSyncPeriod time.Duration + PodVolumeOperationTimeout time.Duration + ResourceTerminatingTimeout time.Duration + DefaultBackupTTL time.Duration + DefaultVGSLabelKey string + StoreValidationFrequency time.Duration + DefaultCSISnapshotTimeout time.Duration + DefaultItemOperationTimeout time.Duration + ResourceTimeout time.Duration + RestoreResourcePriorities types.Priorities + DefaultVolumeSnapshotLocations flag.Map + RestoreOnly bool + DisabledControllers []string + ClientQPS float32 + ClientBurst int + ClientPageSize int + ProfilerAddress string + LogLevel *logging.LevelFlag + LogFormat *logging.FormatFlag + RepoMaintenanceFrequency time.Duration + GarbageCollectionFrequency time.Duration + ItemOperationSyncFrequency time.Duration + DefaultVolumesToFsBackup bool + UploaderType string + MaxConcurrentK8SConnections int + DefaultSnapshotMoveData bool + DisableInformerCache bool + ScheduleSkipImmediately bool + CredentialsDirectory string + BackupRepoConfig string + RepoMaintenanceJobConfig string + ItemBlockWorkerCount int + ConcurrentBackups int + GlobalBackupVolumePoliciesConfigMap string } func GetDefaultConfig() *Config { @@ -275,4 +276,10 @@ func (c *Config) BindFlags(flags *pflag.FlagSet) { c.ConcurrentBackups, "Number of backups to process concurrently. Default is one. Optional.", ) + flags.StringVar( + &c.GlobalBackupVolumePoliciesConfigMap, + "global-backup-volume-policies-configmap", + c.GlobalBackupVolumePoliciesConfigMap, + "The name of a ConfigMap in the Velero install namespace holding global backup volume policies that are merged into every backup. Optional.", + ) } diff --git a/pkg/cmd/server/config/config_test.go b/pkg/cmd/server/config/config_test.go index ba17437f1..a0e33c413 100644 --- a/pkg/cmd/server/config/config_test.go +++ b/pkg/cmd/server/config/config_test.go @@ -5,6 +5,7 @@ import ( "github.com/spf13/pflag" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestGetDefaultConfig(t *testing.T) { @@ -17,3 +18,14 @@ func TestBindFlags(t *testing.T) { config.BindFlags(pflag.CommandLine) assert.Equal(t, 1, config.ItemBlockWorkerCount) } + +func TestGlobalBackupVolumePoliciesConfigMapFlag(t *testing.T) { + config := GetDefaultConfig() + // Opt-in: defaults to empty. + assert.Empty(t, config.GlobalBackupVolumePoliciesConfigMap) + + flags := pflag.NewFlagSet("test", pflag.ContinueOnError) + config.BindFlags(flags) + require.NoError(t, flags.Parse([]string{"--global-backup-volume-policies-configmap", "global-volume-policy"})) + assert.Equal(t, "global-volume-policy", config.GlobalBackupVolumePoliciesConfigMap) +} diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 44c88b980..83627f9d1 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -27,9 +27,9 @@ import ( "time" logrusr "github.com/bombsimon/logrusr/v3" + "github.com/cockroachdb/errors" volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -57,6 +57,7 @@ import ( "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/internal/hook" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/storage" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" @@ -390,6 +391,14 @@ func (s *server) setupBeforeControllerRun() error { if err := setDefaultBackupLocation(s.ctx, client, s.namespace, s.config.DefaultBackupLocation, s.logger); err != nil { return err } + + // Validate the global backup volume policies ConfigMap early, so misconfigurations fail fast. + if s.config.GlobalBackupVolumePoliciesConfigMap != "" { + if _, err := resourcepolicies.GetGlobalResourcePolicies(client, s.namespace, s.config.GlobalBackupVolumePoliciesConfigMap, s.logger); err != nil { + return err + } + s.logger.WithField("configmap", s.config.GlobalBackupVolumePoliciesConfigMap).Info("Loaded global backup volume policies") + } return nil } @@ -671,6 +680,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.config.ItemBlockWorkerCount, s.config.ConcurrentBackups, s.crClient, + s.config.GlobalBackupVolumePoliciesConfigMap, ).SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerBackup) } diff --git a/pkg/cmd/util/cacert/bsl_cacert.go b/pkg/cmd/util/cacert/bsl_cacert.go index d11729945..9d69d6c03 100644 --- a/pkg/cmd/util/cacert/bsl_cacert.go +++ b/pkg/cmd/util/cacert/bsl_cacert.go @@ -19,7 +19,7 @@ package cacert import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/cmd/util/downloadrequest/downloadrequest.go b/pkg/cmd/util/downloadrequest/downloadrequest.go index 6e1d30c37..f0956b1cb 100644 --- a/pkg/cmd/util/downloadrequest/downloadrequest.go +++ b/pkg/cmd/util/downloadrequest/downloadrequest.go @@ -28,8 +28,8 @@ import ( "os" "time" + "github.com/cockroachdb/errors" "github.com/google/uuid" - "github.com/pkg/errors" kbclient "sigs.k8s.io/controller-runtime/pkg/client" veleroV1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" diff --git a/pkg/cmd/util/flag/enum.go b/pkg/cmd/util/flag/enum.go index bc36aef68..f2334a612 100644 --- a/pkg/cmd/util/flag/enum.go +++ b/pkg/cmd/util/flag/enum.go @@ -17,7 +17,7 @@ limitations under the License. package flag import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" ) // Enum is a Cobra-compatible wrapper for defining diff --git a/pkg/cmd/util/flag/map.go b/pkg/cmd/util/flag/map.go index 1b4a6e21c..fae6329e1 100644 --- a/pkg/cmd/util/flag/map.go +++ b/pkg/cmd/util/flag/map.go @@ -21,7 +21,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" ) // Map is a Cobra-compatible wrapper for defining a flag containing diff --git a/pkg/cmd/util/output/backup_describer.go b/pkg/cmd/util/output/backup_describer.go index e0637a4bd..4c8222f81 100644 --- a/pkg/cmd/util/output/backup_describer.go +++ b/pkg/cmd/util/output/backup_describer.go @@ -21,6 +21,7 @@ import ( "context" "encoding/json" "fmt" + "io" "sort" "strconv" "strings" @@ -28,8 +29,9 @@ import ( corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/fatih/color" kbclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -40,6 +42,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" "github.com/vmware-tanzu/velero/pkg/itemoperation" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/volume" "github.com/vmware-tanzu/velero/pkg/util/collections" "github.com/vmware-tanzu/velero/pkg/util/results" @@ -91,8 +94,13 @@ func DescribeBackup( if backup.Spec.ResourcePolicy != nil { d.Println() DescribeResourcePolicies(d, backup.Spec.ResourcePolicy) + + // Display fine-grained filter policies if they exist + DescribeFineGrainedFilterPolicies(ctx, kbClient, d, backup) } + DescribeGlobalVolumePolicy(d, backup) + if backup.Spec.UploaderConfig != nil && backup.Spec.UploaderConfig.ParallelFilesUpload > 0 { d.Println() DescribeUploaderConfigForBackup(d, backup.Spec) @@ -130,6 +138,132 @@ func DescribeResourcePolicies(d *Describer, resPolicies *corev1api.TypedLocalObj d.Printf("\tName:\t%s\n", resPolicies.Name) } +// DescribeGlobalVolumePolicy describes the cluster-wide global backup volume policies +// ConfigMap that contributed to the backup, if any. +func DescribeGlobalVolumePolicy(d *Describer, backup *velerov1api.Backup) { + name := backup.Annotations[velerov1api.GlobalBackupVolumePolicyConfigMapAnnotation] + if name == "" { + return + } + d.Println() + d.Printf("Global volume policies:\n") + d.Printf("\tType:\t%s\n", resourcepolicies.ConfigmapRefType) + d.Printf("\tName:\t%s\n", name) +} + +// DescribeFineGrainedFilterPolicies describes cluster-scoped and namespace-scoped filter policies if present +func DescribeFineGrainedFilterPolicies(ctx context.Context, kbClient kbclient.Client, d *Describer, backup *velerov1api.Backup) { + if backup.Spec.ResourcePolicy == nil { + return + } + + // Create a discard logger for the resource policies function since this is CLI output context + discardLogger := logrus.New() + discardLogger.Out = io.Discard + + resourcePolicies, err := resourcepolicies.GetResourcePoliciesFromBackup(*backup, kbClient, discardLogger) + if err != nil { + // Don't fail the describe if we can't read policies, just skip + return + } + + if resourcePolicies == nil { + return + } + + clusterScopedFilterPolicy := resourcePolicies.GetClusterScopedFilterPolicy() + if clusterScopedFilterPolicy != nil { + d.Printf("\nCluster Scoped Filter Policy:\n") + d.Printf(" Resource Filters:\n") + for _, rf := range clusterScopedFilterPolicy.ResourceFilters { + kindsStr := strings.Join(rf.Kinds, ", ") + d.Printf(" %s:\n", kindsStr) + + // Label selector + if len(rf.LabelSelector) > 0 { + selectorStr := formatLabelMap(rf.LabelSelector) + d.Printf(" Label selector: %s\n", selectorStr) + } else if len(rf.OrLabelSelectors) > 0 { + var orStrs []string + for _, ols := range rf.OrLabelSelectors { + orStrs = append(orStrs, formatLabelMap(ols)) + } + d.Printf(" OR label selectors: [%s]\n", strings.Join(orStrs, ", ")) + } else { + d.Printf(" Label selector: \n") + } + + // Name patterns + if len(rf.Names) > 0 { + d.Printf(" Included names: [%s]\n", strings.Join(rf.Names, ", ")) + } else { + d.Printf(" Included names: \n") + } + + if len(rf.ExcludedNames) > 0 { + d.Printf(" Excluded names: [%s]\n", strings.Join(rf.ExcludedNames, ", ")) + } else { + d.Printf(" Excluded names: \n") + } + } + } + + nfPolicies := resourcePolicies.GetNamespacedFilterPolicies() + if len(nfPolicies) > 0 { + d.Printf("\nNamespace-Scoped Filter Policies:\n") + for _, policy := range nfPolicies { + for _, ns := range policy.Namespaces { + d.Printf(" %s:\n", ns) + d.Printf(" Resource Filters:\n") + for _, rf := range policy.ResourceFilters { + var kindsStr string + if rf.IsCatchAll() { + kindsStr = " (all other kinds)" + } else { + kindsStr = strings.Join(rf.Kinds, ", ") + } + d.Printf(" %s:\n", kindsStr) + + // Label selector + if len(rf.LabelSelector) > 0 { + selectorStr := formatLabelMap(rf.LabelSelector) + d.Printf(" Label selector: %s\n", selectorStr) + } else if len(rf.OrLabelSelectors) > 0 { + var orStrs []string + for _, ols := range rf.OrLabelSelectors { + orStrs = append(orStrs, formatLabelMap(ols)) + } + d.Printf(" OR label selectors: [%s]\n", strings.Join(orStrs, ", ")) + } else { + d.Printf(" Label selector: \n") + } + + // Name patterns + if len(rf.Names) > 0 { + d.Printf(" Included names: [%s]\n", strings.Join(rf.Names, ", ")) + } else { + d.Printf(" Included names: \n") + } + + if len(rf.ExcludedNames) > 0 { + d.Printf(" Excluded names: [%s]\n", strings.Join(rf.ExcludedNames, ", ")) + } else { + d.Printf(" Excluded names: \n") + } + } + } + } + } +} + +func formatLabelMap(labelMap map[string]string) string { + var pairs []string + for k, v := range labelMap { + pairs = append(pairs, fmt.Sprintf("%s=%s", k, v)) + } + return strings.Join(pairs, ",") +} + // DescribeUploaderConfigForBackup describes uploader config in human-readable format func DescribeUploaderConfigForBackup(d *Describer, spec velerov1api.BackupSpec) { d.Printf("Uploader config:\n") diff --git a/pkg/cmd/util/output/backup_describer_test.go b/pkg/cmd/util/output/backup_describer_test.go index 0de03bdaa..248b0a45b 100644 --- a/pkg/cmd/util/output/backup_describer_test.go +++ b/pkg/cmd/util/output/backup_describer_test.go @@ -18,6 +18,7 @@ package output import ( "bytes" + "context" "testing" "text/tabwriter" "time" @@ -25,6 +26,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -69,6 +72,34 @@ func TestDescribeResourcePolicies(t *testing.T) { assert.Equal(t, expect, d.buf.String()) } +func TestDescribeGlobalVolumePolicy(t *testing.T) { + newDescriber := func() *Describer { + d := &Describer{out: &tabwriter.Writer{}, buf: &bytes.Buffer{}} + d.out.Init(d.buf, 0, 8, 2, ' ', 0) + return d + } + + // No annotation: nothing is printed. + d := newDescriber() + DescribeGlobalVolumePolicy(d, builder.ForBackup("velero", "b").Result()) + d.out.Flush() + assert.Empty(t, d.buf.String()) + + // Annotation present: ConfigMap name is surfaced. + d = newDescriber() + backup := builder.ForBackup("velero", "b"). + ObjectMeta(builder.WithAnnotations(velerov1api.GlobalBackupVolumePolicyConfigMapAnnotation, "global-volume-policy")). + Result() + DescribeGlobalVolumePolicy(d, backup) + d.out.Flush() + expect := ` +Global volume policies: + Type: configmap + Name: global-volume-policy +` + assert.Equal(t, expect, d.buf.String()) +} + func TestDescribeBackupSpec(t *testing.T) { input1 := builder.ForBackup("test-ns", "test-backup-1"). IncludedNamespaces("inc-ns-1", "inc-ns-2"). @@ -866,3 +897,85 @@ func TestDescribeBackupItemOperation(t *testing.T) { d.out.Flush() assert.Equal(t, expected, d.buf.String()) } + +func TestDescribeFineGrainedFilterPolicies(t *testing.T) { + yamlData := ` +version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["StorageClass"] + labelSelector: {"app": "velero"} + - kinds: ["ClusterRole"] + orLabelSelectors: + - {"app": "velero"} + - {"app": "test"} + names: ["role1"] + excludedNames: ["role2"] +namespacedFilterPolicies: +- namespaces: ["ns1", "ns2"] + resourceFilters: + - kinds: ["Pod", "ConfigMap"] + labelSelector: {"app": "velero"} + - kinds: ["*"] +` + cm := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-policy", + Namespace: "velero", + }, + Data: map[string]string{ + "policy.yaml": yamlData, + }, + } + + client := fake.NewClientBuilder().WithRuntimeObjects(cm).Build() + + backup := builder.ForBackup("velero", "test-backup"). + ResourcePolicies("test-policy").Result() + + d := &Describer{ + Prefix: "", + out: &tabwriter.Writer{}, + buf: &bytes.Buffer{}, + } + d.out.Init(d.buf, 0, 8, 2, ' ', 0) + + DescribeFineGrainedFilterPolicies(context.Background(), client, d, backup) + d.out.Flush() + + expected := ` +Cluster Scoped Filter Policy: + Resource Filters: + StorageClass: + Label selector: app=velero + Included names: + Excluded names: + ClusterRole: + OR label selectors: [app=velero, app=test] + Included names: [role1] + Excluded names: [role2] + +Namespace-Scoped Filter Policies: + ns1: + Resource Filters: + Pod, ConfigMap: + Label selector: app=velero + Included names: + Excluded names: + (all other kinds): + Label selector: + Included names: + Excluded names: + ns2: + Resource Filters: + Pod, ConfigMap: + Label selector: app=velero + Included names: + Excluded names: + (all other kinds): + Label selector: + Included names: + Excluded names: +` + assert.Equal(t, expected, d.buf.String()) +} diff --git a/pkg/cmd/util/output/backup_structured_describer.go b/pkg/cmd/util/output/backup_structured_describer.go index 904afa34e..dfffcda06 100644 --- a/pkg/cmd/util/output/backup_structured_describer.go +++ b/pkg/cmd/util/output/backup_structured_describer.go @@ -21,13 +21,16 @@ import ( "context" "encoding/json" "fmt" + "io" "strings" + "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kbclient "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/cmd/util/cacert" @@ -54,8 +57,11 @@ func DescribeBackupInSF( if backup.Spec.ResourcePolicy != nil { DescribeResourcePoliciesInSF(d, backup.Spec.ResourcePolicy) + DescribeFineGrainedFilterPoliciesInSF(ctx, kbClient, d, backup) } + DescribeGlobalVolumePolicyInSF(d, backup) + status := backup.Status if len(status.ValidationErrors) > 0 { d.Describe("validationErrors", status.ValidationErrors) @@ -222,6 +228,88 @@ func DescribeBackupSpecInSF(d *StructuredDescriber, spec velerov1api.BackupSpec) d.Describe("spec", backupSpecInfo) } +// DescribeFineGrainedFilterPoliciesInSF adds the clusterScopedFilterPolicy +// and namespacedFilterPolicies sections to the structured describer output when present +// in the ResourcePolicy ConfigMap referenced by the backup. +func DescribeFineGrainedFilterPoliciesInSF(ctx context.Context, kbClient kbclient.Client, d *StructuredDescriber, backup *velerov1api.Backup) { + if backup.Spec.ResourcePolicy == nil { + return + } + + discardLogger := logrus.New() + discardLogger.Out = io.Discard + + resPolicies, err := resourcepolicies.GetResourcePoliciesFromBackup(*backup, kbClient, discardLogger) + if err != nil || resPolicies == nil { + return + } + + clusterScopedFilterPolicy := resPolicies.GetClusterScopedFilterPolicy() + if clusterScopedFilterPolicy != nil { + var clusterScopedFilters []map[string]any + for _, rf := range clusterScopedFilterPolicy.ResourceFilters { + entry := map[string]any{ + "kinds": rf.Kinds, + } + if len(rf.LabelSelector) > 0 { + entry["labelSelector"] = rf.LabelSelector + } + if len(rf.OrLabelSelectors) > 0 { + entry["orLabelSelectors"] = rf.OrLabelSelectors + } + if len(rf.Names) > 0 { + entry["names"] = rf.Names + } + if len(rf.ExcludedNames) > 0 { + entry["excludedNames"] = rf.ExcludedNames + } + clusterScopedFilters = append(clusterScopedFilters, entry) + } + d.Describe("clusterScopedFilterPolicy", map[string]any{ + "resourceFilters": clusterScopedFilters, + }) + } + + nfPolicies := resPolicies.GetNamespacedFilterPolicies() + if len(nfPolicies) == 0 { + return + } + + var structuredPolicies []map[string]any + for _, policy := range nfPolicies { + for _, ns := range policy.Namespaces { + var rfEntries []map[string]any + for _, rf := range policy.ResourceFilters { + entry := map[string]any{} + if rf.IsCatchAll() { + entry["kinds"] = []string{} + entry["isCatchAll"] = true + } else { + entry["kinds"] = rf.Kinds + } + if len(rf.LabelSelector) > 0 { + entry["labelSelector"] = rf.LabelSelector + } + if len(rf.OrLabelSelectors) > 0 { + entry["orLabelSelectors"] = rf.OrLabelSelectors + } + if len(rf.Names) > 0 { + entry["names"] = rf.Names + } + if len(rf.ExcludedNames) > 0 { + entry["excludedNames"] = rf.ExcludedNames + } + rfEntries = append(rfEntries, entry) + } + structuredPolicies = append(structuredPolicies, map[string]any{ + "namespace": ns, + "resourceFilters": rfEntries, + }) + } + } + d.Describe("namespacedFilterPolicies", structuredPolicies) +} + // DescribeBackupStatusInSF describes a backup status in structured format. func DescribeBackupStatusInSF(ctx context.Context, kbClient kbclient.Client, d *StructuredDescriber, backup *velerov1api.Backup, details bool, insecureSkipTLSVerify bool, caCertPath string, podVolumeBackups []velerov1api.PodVolumeBackup) { @@ -613,6 +701,19 @@ func DescribeResourcePoliciesInSF(d *StructuredDescriber, resPolicies *corev1api d.Describe("resourcePolicies", policiesInfo) } +// DescribeGlobalVolumePolicyInSF describes the global backup volume policies ConfigMap that +// contributed to the backup, if any, in structured format. +func DescribeGlobalVolumePolicyInSF(d *StructuredDescriber, backup *velerov1api.Backup) { + name := backup.Annotations[velerov1api.GlobalBackupVolumePolicyConfigMapAnnotation] + if name == "" { + return + } + d.Describe("globalVolumePolicies", map[string]any{ + "type": resourcepolicies.ConfigmapRefType, + "name": name, + }) +} + func describeResultInSF(m map[string]any, result results.Result) { m["velero"], m["cluster"], m["namespace"] = []string{}, []string{}, []string{} diff --git a/pkg/cmd/util/output/backup_structured_describer_test.go b/pkg/cmd/util/output/backup_structured_describer_test.go index c5ede1b36..cb46a4676 100644 --- a/pkg/cmd/util/output/backup_structured_describer_test.go +++ b/pkg/cmd/util/output/backup_structured_describer_test.go @@ -17,6 +17,7 @@ limitations under the License. package output import ( + "context" "reflect" "testing" "time" @@ -24,6 +25,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -624,6 +627,27 @@ func TestDescribeResourcePoliciesInSF(t *testing.T) { assert.True(t, reflect.DeepEqual(sd.output, expect)) } +func TestDescribeGlobalVolumePolicyInSF(t *testing.T) { + // No annotation: nothing is added to the output. + sd := &StructuredDescriber{output: make(map[string]any), format: ""} + DescribeGlobalVolumePolicyInSF(sd, builder.ForBackup("velero", "b").Result()) + assert.Empty(t, sd.output) + + // Annotation present: the ConfigMap name is surfaced. + sd = &StructuredDescriber{output: make(map[string]any), format: ""} + backup := builder.ForBackup("velero", "b"). + ObjectMeta(builder.WithAnnotations(velerov1api.GlobalBackupVolumePolicyConfigMapAnnotation, "global-volume-policy")). + Result() + DescribeGlobalVolumePolicyInSF(sd, backup) + expectGlobal := map[string]any{ + "globalVolumePolicies": map[string]any{ + "type": "configmap", + "name": "global-volume-policy", + }, + } + assert.True(t, reflect.DeepEqual(sd.output, expectGlobal)) +} + func TestDescribeBackupResultInSF(t *testing.T) { input := results.Result{ Velero: []string{"msg-1", "msg-2"}, @@ -707,3 +731,96 @@ func TestDescribeDeleteBackupRequestsInSF(t *testing.T) { }) } } + +func TestDescribeFineGrainedFilterPoliciesInSF(t *testing.T) { + yamlData := ` +version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["StorageClass"] + labelSelector: {"app": "velero"} + - kinds: ["ClusterRole"] + orLabelSelectors: + - {"app": "velero"} + - {"app": "test"} + names: ["role1"] + excludedNames: ["role2"] +namespacedFilterPolicies: +- namespaces: ["ns1", "ns2"] + resourceFilters: + - kinds: ["Pod", "ConfigMap"] + labelSelector: {"app": "velero"} + - kinds: ["*"] +` + cm := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-policy", + Namespace: "velero", + }, + Data: map[string]string{ + "policy.yaml": yamlData, + }, + } + + client := fake.NewClientBuilder().WithRuntimeObjects(cm).Build() + + backup := builder.ForBackup("velero", "test-backup"). + ResourcePolicies("test-policy").Result() + + sd := &StructuredDescriber{ + output: make(map[string]any), + format: "", + } + + DescribeFineGrainedFilterPoliciesInSF(context.Background(), client, sd, backup) + + expect := map[string]any{ + "clusterScopedFilterPolicy": map[string]any{ + "resourceFilters": []map[string]any{ + { + "kinds": []string{"StorageClass"}, + "labelSelector": map[string]string{"app": "velero"}, + }, + { + "kinds": []string{"ClusterRole"}, + "orLabelSelectors": []map[string]string{ + {"app": "velero"}, + {"app": "test"}, + }, + "names": []string{"role1"}, + "excludedNames": []string{"role2"}, + }, + }, + }, + "namespacedFilterPolicies": []map[string]any{ + { + "namespace": "ns1", + "resourceFilters": []map[string]any{ + { + "kinds": []string{"Pod", "ConfigMap"}, + "labelSelector": map[string]string{"app": "velero"}, + }, + { + "kinds": []string{}, + "isCatchAll": true, + }, + }, + }, + { + "namespace": "ns2", + "resourceFilters": []map[string]any{ + { + "kinds": []string{"Pod", "ConfigMap"}, + "labelSelector": map[string]string{"app": "velero"}, + }, + { + "kinds": []string{}, + "isCatchAll": true, + }, + }, + }, + }, + } + + assert.True(t, reflect.DeepEqual(sd.output, expect)) +} diff --git a/pkg/cmd/util/output/output.go b/pkg/cmd/util/output/output.go index a0f8ce704..9dfca040b 100644 --- a/pkg/cmd/util/output/output.go +++ b/pkg/cmd/util/output/output.go @@ -21,7 +21,7 @@ import ( "os" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/api/meta" diff --git a/pkg/cmd/util/output/restore_describer.go b/pkg/cmd/util/output/restore_describer.go index a89943e74..c33da9f69 100644 --- a/pkg/cmd/util/output/restore_describer.go +++ b/pkg/cmd/util/output/restore_describer.go @@ -219,6 +219,11 @@ func DescribeRestore( DescribeResourceModifier(d, restore.Spec.ResourceModifier) } + if restore.Spec.ResourcePolicy != nil { + d.Println() + DescribeResourcePolicies(d, restore.Spec.ResourcePolicy) + } + describeUploaderConfigForRestore(d, restore.Spec) d.Println() diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 2de09db5a..74b857fd2 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -24,8 +24,8 @@ import ( "slices" "time" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -84,32 +84,33 @@ var autoExcludeClusterScopedResources = []string{ } type backupReconciler struct { - ctx context.Context - logger logrus.FieldLogger - discoveryHelper discovery.Helper - backupper pkgbackup.Backupper - kbClient kbclient.Client - clock clock.WithTickerAndDelayedExecution - backupLogLevel logrus.Level - newPluginManager func(logrus.FieldLogger) clientmgmt.Manager - backupTracker BackupTracker - defaultBackupLocation string - defaultVolumesToFsBackup bool - defaultBackupTTL time.Duration - defaultVGSLabelKey string - defaultCSISnapshotTimeout time.Duration - resourceTimeout time.Duration - defaultItemOperationTimeout time.Duration - defaultSnapshotLocations map[string]string - metrics *metrics.ServerMetrics - backupStoreGetter persistence.ObjectBackupStoreGetter - formatFlag logging.Format - credentialFileStore credentials.FileStore - maxConcurrentK8SConnections int - defaultSnapshotMoveData bool - globalCRClient kbclient.Client - itemBlockWorkerCount int - concurrentBackups int + ctx context.Context + logger logrus.FieldLogger + discoveryHelper discovery.Helper + backupper pkgbackup.Backupper + kbClient kbclient.Client + clock clock.WithTickerAndDelayedExecution + backupLogLevel logrus.Level + newPluginManager func(logrus.FieldLogger) clientmgmt.Manager + backupTracker BackupTracker + defaultBackupLocation string + defaultVolumesToFsBackup bool + defaultBackupTTL time.Duration + defaultVGSLabelKey string + defaultCSISnapshotTimeout time.Duration + resourceTimeout time.Duration + defaultItemOperationTimeout time.Duration + defaultSnapshotLocations map[string]string + metrics *metrics.ServerMetrics + backupStoreGetter persistence.ObjectBackupStoreGetter + formatFlag logging.Format + credentialFileStore credentials.FileStore + maxConcurrentK8SConnections int + defaultSnapshotMoveData bool + globalCRClient kbclient.Client + itemBlockWorkerCount int + concurrentBackups int + globalVolumePoliciesConfigMap string } func NewBackupReconciler( @@ -138,34 +139,36 @@ func NewBackupReconciler( itemBlockWorkerCount int, concurrentBackups int, globalCRClient kbclient.Client, + globalVolumePoliciesConfigMap string, ) *backupReconciler { b := &backupReconciler{ - ctx: ctx, - discoveryHelper: discoveryHelper, - backupper: backupper, - clock: &clock.RealClock{}, - logger: logger, - backupLogLevel: backupLogLevel, - newPluginManager: newPluginManager, - backupTracker: backupTracker, - kbClient: kbClient, - defaultBackupLocation: defaultBackupLocation, - defaultVolumesToFsBackup: defaultVolumesToFsBackup, - defaultBackupTTL: defaultBackupTTL, - defaultVGSLabelKey: defaultVGSLabelKey, - defaultCSISnapshotTimeout: defaultCSISnapshotTimeout, - resourceTimeout: resourceTimeout, - defaultItemOperationTimeout: defaultItemOperationTimeout, - defaultSnapshotLocations: defaultSnapshotLocations, - metrics: metrics, - backupStoreGetter: backupStoreGetter, - formatFlag: formatFlag, - credentialFileStore: credentialStore, - maxConcurrentK8SConnections: maxConcurrentK8SConnections, - defaultSnapshotMoveData: defaultSnapshotMoveData, - itemBlockWorkerCount: itemBlockWorkerCount, - concurrentBackups: max(concurrentBackups, 1), - globalCRClient: globalCRClient, + ctx: ctx, + discoveryHelper: discoveryHelper, + backupper: backupper, + clock: &clock.RealClock{}, + logger: logger, + backupLogLevel: backupLogLevel, + newPluginManager: newPluginManager, + backupTracker: backupTracker, + kbClient: kbClient, + defaultBackupLocation: defaultBackupLocation, + defaultVolumesToFsBackup: defaultVolumesToFsBackup, + defaultBackupTTL: defaultBackupTTL, + defaultVGSLabelKey: defaultVGSLabelKey, + defaultCSISnapshotTimeout: defaultCSISnapshotTimeout, + resourceTimeout: resourceTimeout, + defaultItemOperationTimeout: defaultItemOperationTimeout, + defaultSnapshotLocations: defaultSnapshotLocations, + metrics: metrics, + backupStoreGetter: backupStoreGetter, + formatFlag: formatFlag, + credentialFileStore: credentialStore, + maxConcurrentK8SConnections: maxConcurrentK8SConnections, + defaultSnapshotMoveData: defaultSnapshotMoveData, + itemBlockWorkerCount: itemBlockWorkerCount, + concurrentBackups: max(concurrentBackups, 1), + globalCRClient: globalCRClient, + globalVolumePoliciesConfigMap: globalVolumePoliciesConfigMap, } b.updateTotalBackupMetric() return b @@ -407,6 +410,11 @@ func (b *backupReconciler) prepareBackupRequest(ctx context.Context, backup *vel request.Spec.ItemOperationTimeout.Duration = b.defaultItemOperationTimeout } + if len(request.Spec.BackupType) == 0 { + // default backup type to incremental if not specified + request.Spec.BackupType = velerov1api.BackupTypeIncremental + } + // calculate expiration request.Status.Expiration = &metav1.Time{Time: b.clock.Now().Add(request.Spec.TTL.Duration)} @@ -587,9 +595,13 @@ func (b *backupReconciler) prepareBackupRequest(ctx context.Context, backup *vel request.Status.ValidationErrors = append(request.Status.ValidationErrors, "encountered labelSelector as well as orLabelSelectors in backup spec, only one can be specified") } - resourcePolicies, err := resourcepolicies.GetResourcePoliciesFromBackup(*request.Backup, b.kbClient, logger) + resourcePolicies, err := resourcepolicies.GetResourcePoliciesFromBackupWithGlobal( + *request.Backup, b.kbClient, b.globalVolumePoliciesConfigMap, request.Namespace, logger) if err != nil { request.Status.ValidationErrors = append(request.Status.ValidationErrors, err.Error()) + } else if b.globalVolumePoliciesConfigMap != "" { + // Record the contributing global volume policies ConfigMap so `velero backup describe` can surface it. + request.Annotations[velerov1api.GlobalBackupVolumePolicyConfigMapAnnotation] = b.globalVolumePoliciesConfigMap } if resourcePolicies != nil && resourcePolicies.GetIncludeExcludePolicy() != nil && collections.UseOldResourceFilters(request.Spec) { request.Status.ValidationErrors = append(request.Status.ValidationErrors, "include-resources, exclude-resources and include-cluster-resources are old filter parameters.\n"+ diff --git a/pkg/controller/backup_controller_test.go b/pkg/controller/backup_controller_test.go index 3710bb28a..bab98efb6 100644 --- a/pkg/controller/backup_controller_test.go +++ b/pkg/controller/backup_controller_test.go @@ -27,10 +27,10 @@ import ( "testing" "time" + "github.com/cockroachdb/errors" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -46,6 +46,7 @@ import ( kbclient "sigs.k8s.io/controller-runtime/pkg/client" fakeClient "sigs.k8s.io/controller-runtime/pkg/client/fake" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/builder" @@ -523,6 +524,63 @@ func TestDefaultBackupTTL(t *testing.T) { } } +func TestPrepareBackupRequest_SetBackupType(t *testing.T) { + now, err := time.Parse(time.RFC1123Z, time.RFC1123Z) + require.NoError(t, err) + now = now.Local() + + tests := []struct { + name string + backup *velerov1api.Backup + expectedBackupType velerov1api.BackupType + }{ + { + name: "default backup type is Incremental", + backup: defaultBackup().Result(), + expectedBackupType: velerov1api.BackupTypeIncremental, + }, + { + name: "backup type is set to Full", + backup: defaultBackup().BackupType(velerov1api.BackupTypeFull).Result(), + expectedBackupType: velerov1api.BackupTypeFull, + }, + { + name: "backup type is set to Incremental", + backup: defaultBackup().BackupType(velerov1api.BackupTypeIncremental).Result(), + expectedBackupType: velerov1api.BackupTypeIncremental, + }, + } + + for _, test := range tests { + formatFlag := logging.FormatText + var ( + fakeClient kbclient.Client + logger = logging.DefaultLogger(logrus.DebugLevel, formatFlag) + ) + + t.Run(test.name, func(t *testing.T) { + apiServer := velerotest.NewAPIServer(t) + discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, logger) + require.NoError(t, err) + // add the test's backup storage location if it's different than the default + fakeClient = velerotest.NewFakeControllerRuntimeClient(t) + c := &backupReconciler{ + logger: logger, + discoveryHelper: discoveryHelper, + kbClient: fakeClient, + formatFlag: formatFlag, + clock: testclocks.NewFakeClock(now), + } + + res := c.prepareBackupRequest(ctx, test.backup, logger) + defer res.WorkerPool.Stop() + assert.NotNil(t, res) + + assert.Equal(t, test.expectedBackupType, res.Spec.BackupType) + }) + } +} + func TestPrepareBackupRequest_SetsVGSLabelKey(t *testing.T) { now, err := time.Parse(time.RFC1123Z, time.RFC1123Z) require.NoError(t, err) @@ -745,6 +803,7 @@ func TestProcessBackupCompletions(t *testing.T) { SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, + BackupType: velerov1api.BackupTypeIncremental, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -785,6 +844,7 @@ func TestProcessBackupCompletions(t *testing.T) { SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, + BackupType: velerov1api.BackupTypeIncremental, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -829,6 +889,7 @@ func TestProcessBackupCompletions(t *testing.T) { SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, + BackupType: velerov1api.BackupTypeIncremental, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -870,6 +931,7 @@ func TestProcessBackupCompletions(t *testing.T) { SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, + BackupType: velerov1api.BackupTypeIncremental, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -911,6 +973,7 @@ func TestProcessBackupCompletions(t *testing.T) { SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, + BackupType: velerov1api.BackupTypeIncremental, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -953,6 +1016,7 @@ func TestProcessBackupCompletions(t *testing.T) { SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, + BackupType: velerov1api.BackupTypeIncremental, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -995,6 +1059,7 @@ func TestProcessBackupCompletions(t *testing.T) { SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, + BackupType: velerov1api.BackupTypeIncremental, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1037,6 +1102,7 @@ func TestProcessBackupCompletions(t *testing.T) { SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, + BackupType: velerov1api.BackupTypeIncremental, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1079,6 +1145,7 @@ func TestProcessBackupCompletions(t *testing.T) { SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, + BackupType: velerov1api.BackupTypeIncremental, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1122,6 +1189,7 @@ func TestProcessBackupCompletions(t *testing.T) { SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, + BackupType: velerov1api.BackupTypeIncremental, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFailed, @@ -1165,6 +1233,7 @@ func TestProcessBackupCompletions(t *testing.T) { SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, + BackupType: velerov1api.BackupTypeIncremental, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFailed, @@ -1208,6 +1277,7 @@ func TestProcessBackupCompletions(t *testing.T) { SnapshotMoveData: boolptr.True(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, + BackupType: velerov1api.BackupTypeIncremental, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1252,6 +1322,7 @@ func TestProcessBackupCompletions(t *testing.T) { SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, + BackupType: velerov1api.BackupTypeIncremental, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1296,6 +1367,7 @@ func TestProcessBackupCompletions(t *testing.T) { SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, + BackupType: velerov1api.BackupTypeIncremental, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1340,6 +1412,7 @@ func TestProcessBackupCompletions(t *testing.T) { SnapshotMoveData: boolptr.True(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, + BackupType: velerov1api.BackupTypeIncremental, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1385,6 +1458,7 @@ func TestProcessBackupCompletions(t *testing.T) { SnapshotMoveData: boolptr.False(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, + BackupType: velerov1api.BackupTypeIncremental, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1429,6 +1503,7 @@ func TestProcessBackupCompletions(t *testing.T) { SnapshotMoveData: boolptr.True(), ExcludedClusterScopedResources: autoExcludeClusterScopedResources, ExcludedNamespaceScopedResources: autoExcludeNamespaceScopedResources, + BackupType: velerov1api.BackupTypeIncremental, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1479,6 +1554,7 @@ func TestProcessBackupCompletions(t *testing.T) { ExcludedClusterScopedResources: append([]string{"clusterroles"}, autoExcludeClusterScopedResources...), IncludedNamespaceScopedResources: []string{"pods"}, ExcludedNamespaceScopedResources: append([]string{"secrets"}, autoExcludeNamespaceScopedResources...), + BackupType: velerov1api.BackupTypeIncremental, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -1529,6 +1605,7 @@ func TestProcessBackupCompletions(t *testing.T) { ExcludedClusterScopedResources: append([]string{"clusterroles"}, autoExcludeClusterScopedResources...), IncludedNamespaceScopedResources: []string{"pods"}, ExcludedNamespaceScopedResources: append([]string{"secrets"}, autoExcludeNamespaceScopedResources...), + BackupType: velerov1api.BackupTypeIncremental, }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFinalizing, @@ -2076,6 +2153,100 @@ namespacedFilterPolicies: assert.True(t, hasTargetError, "expected validation error about namespacedFilterPolicies incompatibility with old-style filters, got: %v", res.Status.ValidationErrors) } +// TestPrepareBackupRequest_GlobalVolumePolicies verifies that the cluster-wide global backup +// volume policies are merged into the request and that the contributing ConfigMap is recorded +// on the backup so `velero backup describe` can surface it. +func TestPrepareBackupRequest_GlobalVolumePolicies(t *testing.T) { + formatFlag := logging.FormatText + logger := logging.DefaultLogger(logrus.DebugLevel, formatFlag) + + globalCM := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "global-volume-policy", Namespace: velerov1api.DefaultNamespace}, + Data: map[string]string{"policies.yaml": `version: v1 +volumePolicies: + - conditions: + storageClass: + - gp2 + action: + type: skip +`}, + } + + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, globalCM, + builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "loc-1").Result()) + apiServer := velerotest.NewAPIServer(t) + discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, logger) + require.NoError(t, err) + + c := &backupReconciler{ + logger: logger, + discoveryHelper: discoveryHelper, + kbClient: fakeClient, + clock: &clock.RealClock{}, + formatFlag: formatFlag, + defaultBackupLocation: "loc-1", + globalVolumePoliciesConfigMap: "global-volume-policy", + } + + backup := defaultBackup().StorageLocation("loc-1").Result() + res := c.prepareBackupRequest(ctx, backup, logger) + defer res.WorkerPool.Stop() + + // The global volume policies must load cleanly (no policy-related validation error). + for _, e := range res.Status.ValidationErrors { + assert.NotContains(t, e, "global backup volume policies") + } + require.NotNil(t, res.ResPolicies) + assert.Equal(t, "global-volume-policy", res.Annotations[velerov1api.GlobalBackupVolumePolicyConfigMapAnnotation]) + + action, err := res.ResPolicies.GetMatchAction(resourcepolicies.VolumeFilterData{ + PersistentVolume: &corev1api.PersistentVolume{Spec: corev1api.PersistentVolumeSpec{StorageClassName: "gp2"}}, + }) + require.NoError(t, err) + require.NotNil(t, action) + assert.Equal(t, resourcepolicies.Skip, action.Type) +} + +// TestPrepareBackupRequest_GlobalVolumePolicies_LoadError verifies that when the configured +// global backup volume policies ConfigMap cannot be loaded, a validation error is recorded and +// the contributing-ConfigMap annotation is not set on the backup. +func TestPrepareBackupRequest_GlobalVolumePolicies_LoadError(t *testing.T) { + formatFlag := logging.FormatText + logger := logging.DefaultLogger(logrus.DebugLevel, formatFlag) + + // No ConfigMap with this name exists, so loading the global policies fails. + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, + builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "loc-1").Result()) + apiServer := velerotest.NewAPIServer(t) + discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, logger) + require.NoError(t, err) + + c := &backupReconciler{ + logger: logger, + discoveryHelper: discoveryHelper, + kbClient: fakeClient, + clock: &clock.RealClock{}, + formatFlag: formatFlag, + defaultBackupLocation: "loc-1", + globalVolumePoliciesConfigMap: "missing-global-volume-policy", + } + + backup := defaultBackup().StorageLocation("loc-1").Result() + res := c.prepareBackupRequest(ctx, backup, logger) + defer res.WorkerPool.Stop() + + // The failure to load the global policies must surface as a validation error. + var hasGlobalPolicyError bool + for _, e := range res.Status.ValidationErrors { + if strings.Contains(e, "global backup volume policies") { + hasGlobalPolicyError = true + } + } + assert.True(t, hasGlobalPolicyError, "expected a validation error about global backup volume policies, got: %v", res.Status.ValidationErrors) + // The annotation is only set when the policies load successfully. + assert.Empty(t, res.Annotations[velerov1api.GlobalBackupVolumePolicyConfigMapAnnotation]) +} + // TestPrepareBackupRequest_ClusterScopedFilterPolicyIncompatibleWithOldFilters verifies // that a backup referencing a ResourcePolicy ConfigMap with clusterScopedFilterPolicy // produces a validation error when old-style resource filters are also set on the spec. diff --git a/pkg/controller/backup_deletion_controller.go b/pkg/controller/backup_deletion_controller.go index 5fe29c5f1..cd74a3a27 100644 --- a/pkg/controller/backup_deletion_controller.go +++ b/pkg/controller/backup_deletion_controller.go @@ -23,9 +23,9 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" jsonpatch "github.com/evanphx/json-patch/v5" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -295,7 +295,7 @@ func (r *backupDeletionReconciler) Reconcile(ctx context.Context, req ctrl.Reque err = delete.InvokeDeleteActions(deleteCtx) if err != nil { log.WithError(err).Error("Error invoking delete item actions") - err2 := r.patchDeleteBackupRequestWithError(ctx, dbr, errors.New("error invoking delete item actions")) + err2 := r.patchDeleteBackupRequestWithError(ctx, dbr, errors.Wrap(err, "error invoking delete item actions")) return ctrl.Result{}, err2 } } diff --git a/pkg/controller/backup_finalizer_controller.go b/pkg/controller/backup_finalizer_controller.go index b24c132fa..2d722ed51 100644 --- a/pkg/controller/backup_finalizer_controller.go +++ b/pkg/controller/backup_finalizer_controller.go @@ -22,7 +22,7 @@ import ( "os" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/controller/backup_operations_controller.go b/pkg/controller/backup_operations_controller.go index 1a5b49c0b..eda913d3f 100644 --- a/pkg/controller/backup_operations_controller.go +++ b/pkg/controller/backup_operations_controller.go @@ -24,7 +24,7 @@ import ( v2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/controller/backup_queue_controller.go b/pkg/controller/backup_queue_controller.go index c5cdc9461..d2de26c81 100644 --- a/pkg/controller/backup_queue_controller.go +++ b/pkg/controller/backup_queue_controller.go @@ -21,7 +21,7 @@ import ( "slices" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/controller/backup_repository_controller.go b/pkg/controller/backup_repository_controller.go index 11ebb5aec..1318720ba 100644 --- a/pkg/controller/backup_repository_controller.go +++ b/pkg/controller/backup_repository_controller.go @@ -25,8 +25,8 @@ import ( "slices" "time" + "github.com/cockroachdb/errors" "github.com/petar/GoLLRB/llrb" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/controller/backup_storage_location_controller.go b/pkg/controller/backup_storage_location_controller.go index abcd1e59e..32c7c69a3 100644 --- a/pkg/controller/backup_storage_location_controller.go +++ b/pkg/controller/backup_storage_location_controller.go @@ -24,7 +24,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/metrics" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" diff --git a/pkg/controller/backup_storage_location_controller_test.go b/pkg/controller/backup_storage_location_controller_test.go index 8a5dc4a4a..7a99decaf 100644 --- a/pkg/controller/backup_storage_location_controller_test.go +++ b/pkg/controller/backup_storage_location_controller_test.go @@ -22,9 +22,9 @@ import ( "github.com/vmware-tanzu/velero/pkg/metrics" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/pkg/controller/backup_sync_controller.go b/pkg/controller/backup_sync_controller.go index b84ae6f0b..07b5b460f 100644 --- a/pkg/controller/backup_sync_controller.go +++ b/pkg/controller/backup_sync_controller.go @@ -21,7 +21,7 @@ import ( "fmt" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" diff --git a/pkg/controller/data_download_controller.go b/pkg/controller/data_download_controller.go index 738334ceb..fc7cb1a53 100644 --- a/pkg/controller/data_download_controller.go +++ b/pkg/controller/data_download_controller.go @@ -22,7 +22,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -44,7 +44,6 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/constant" - datamover "github.com/vmware-tanzu/velero/pkg/datamover" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/exposer" "github.com/vmware-tanzu/velero/pkg/metrics" @@ -53,6 +52,7 @@ import ( velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util" + datamover "github.com/vmware-tanzu/velero/pkg/util/datamover" "github.com/vmware-tanzu/velero/pkg/util/kube" ) @@ -475,7 +475,12 @@ func (r *DataDownloadReconciler) OnDataDownloadCompleted(ctx context.Context, na } objRef := getDataDownloadOwnerObject(&dd) - err := r.restoreExposer.RebindVolume(ctx, objRef, dd.Spec.TargetVolume.PVC, dd.Spec.TargetVolume.Namespace, dd.Spec.OperationTimeout.Duration) + err := r.restoreExposer.RebindVolume(ctx, objRef, exposer.GenericRestoreRebindVolumeParam{ + TargetPVCName: dd.Spec.TargetVolume.PVC, + TargetNamespace: dd.Spec.TargetVolume.Namespace, + OperationTimeout: dd.Spec.OperationTimeout.Duration, + TargetFSType: dd.Spec.TargetVolume.FSType, + }) if err != nil { log.WithError(err).Error("Failed to rebind PV to target PVC on completion") return @@ -944,6 +949,7 @@ func (r *DataDownloadReconciler) setupExposeParam(dd *velerov2alpha1api.DataDown PriorityClassName: r.dataMovePriorityClass, RestoreSize: dd.Spec.SnapshotSize, CacheVolume: cacheVolume, + DataMover: dd.Spec.DataMover, }, nil } diff --git a/pkg/controller/data_download_controller_test.go b/pkg/controller/data_download_controller_test.go index 397f931c0..518788635 100644 --- a/pkg/controller/data_download_controller_test.go +++ b/pkg/controller/data_download_controller_test.go @@ -22,7 +22,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -548,7 +548,7 @@ func TestDataDownloadReconcile(t *testing.T) { r.restoreExposer = nil } else { r.restoreExposer = func() exposer.GenericRestoreExposer { - ep := exposermockes.NewMockGenericRestoreExposer(t) + ep := exposermockes.NewGenericRestoreExposer(t) if test.isExposeErr { ep.On("Expose", mock.Anything, mock.Anything, mock.Anything).Return(errors.New("Error to expose restore exposer")) } else if test.notNilExpose { @@ -712,7 +712,7 @@ func TestOnDataDownloadCompleted(t *testing.T) { needErrs := []bool{test.isGetErr, false, false, false} r, err := initDataDownloadReconciler(t, nil, needErrs...) r.restoreExposer = func() exposer.GenericRestoreExposer { - ep := exposermockes.NewMockGenericRestoreExposer(t) + ep := exposermockes.NewGenericRestoreExposer(t) if test.rebindVolumeErr { ep.On("RebindVolume", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errors.New("Error to rebind volume")) } else { @@ -1092,7 +1092,7 @@ func (dt *ddResumeTestHelper) DiagnoseExpose(context.Context, corev1api.ObjectRe return "" } -func (dt *ddResumeTestHelper) RebindVolume(context.Context, corev1api.ObjectReference, string, string, time.Duration) error { +func (dt *ddResumeTestHelper) RebindVolume(context.Context, corev1api.ObjectReference, exposer.GenericRestoreRebindVolumeParam) error { return nil } @@ -1429,6 +1429,7 @@ func TestDataDownloadSetupExposeParam(t *testing.T) { // Core fields assert.Equal(t, baseDataDownload.Spec.TargetVolume.PVC, got.TargetPVCName) assert.Equal(t, baseDataDownload.Spec.TargetVolume.Namespace, got.TargetNamespace) + assert.Equal(t, baseDataDownload.Spec.DataMover, got.DataMover) // Labels and Annotations assert.Equal(t, tt.want.labels, got.HostingPodLabels) diff --git a/pkg/controller/data_upload_controller.go b/pkg/controller/data_upload_controller.go index 9aaf3d653..78e4d1ed3 100644 --- a/pkg/controller/data_upload_controller.go +++ b/pkg/controller/data_upload_controller.go @@ -22,8 +22,8 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -45,7 +45,6 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/constant" - "github.com/vmware-tanzu/velero/pkg/datamover" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/exposer" "github.com/vmware-tanzu/velero/pkg/metrics" @@ -53,6 +52,7 @@ import ( velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util" + "github.com/vmware-tanzu/velero/pkg/util/datamover" "github.com/vmware-tanzu/velero/pkg/util/kube" ) @@ -463,9 +463,13 @@ func (r *DataUploadReconciler) initCancelableDataPath(ctx context.Context, async func (r *DataUploadReconciler) startCancelableDataPath(asyncBR datapath.AsyncBR, du *velerov2alpha1api.DataUpload, res *exposer.ExposeResult, log logrus.FieldLogger) error { log.Info("Start cancelable dataUpload") - if err := asyncBR.StartBackup(datapath.AccessPoint{ - ByPath: res.ByPod.VolumeName, - }, du.Spec.DataMoverConfig, nil); err != nil { + if err := asyncBR.StartBackup( + datapath.AccessPoint{ + ByPath: res.ByPod.VolumeName, + }, + du.Spec.DataMoverConfig, + nil, + ); err != nil { return errors.Wrapf(err, "error starting async backup for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName) } diff --git a/pkg/controller/data_upload_controller_test.go b/pkg/controller/data_upload_controller_test.go index e6d5474f3..9703abe92 100644 --- a/pkg/controller/data_upload_controller_test.go +++ b/pkg/controller/data_upload_controller_test.go @@ -22,9 +22,9 @@ import ( "testing" "time" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" snapshotFake "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/fake" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -72,6 +72,7 @@ type FakeClient struct { patchError error updateConflict error listError error + getErrorMap map[string]error // key: object kind or name } func (c *FakeClient) Get(ctx context.Context, key kbclient.ObjectKey, obj kbclient.Object, opts ...kbclient.GetOption) error { @@ -79,6 +80,19 @@ func (c *FakeClient) Get(ctx context.Context, key kbclient.ObjectKey, obj kbclie return c.getError } + // Check if there's a specific error for this object type + if c.getErrorMap != nil { + objType := fmt.Sprintf("%T", obj) + if err, ok := c.getErrorMap[objType]; ok { + return err + } + + // Check if there's a specific error for this object name + if err, ok := c.getErrorMap[key.Name]; ok { + return err + } + } + return c.Client.Get(ctx, key, obj) } @@ -209,9 +223,13 @@ func initDataUploaderReconcilerWithError(needError ...error) (*DataUploadReconci if err != nil { return nil, err } + err = snapshotv1api.AddToScheme(scheme) + if err != nil { + return nil, err + } fakeClient := &FakeClient{ - Client: fake.NewClientBuilder().WithScheme(scheme).Build(), + Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(vsObject, node).Build(), } for k := range needError { @@ -505,7 +523,7 @@ func TestReconcile(t *testing.T) { { name: "du succeeds for accepted", du: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).SnapshotType(fakeSnapshotType).Result(), - pvc: builder.ForPersistentVolumeClaim("fake-ns", "test-pvc").Result(), + pvc: builder.ForPersistentVolumeClaim("fake-ns", "test-pvc").VolumeName("test-pv").Result(), expected: dataUploadBuilder().Finalizers([]string{DataUploadDownloadFinalizer}).Phase(velerov2alpha1api.DataUploadPhaseAccepted).Result(), }, { @@ -636,6 +654,15 @@ func TestReconcile(t *testing.T) { if test.pvc != nil { err = r.client.Create(ctx, test.pvc) require.NoError(t, err) + + // Create the corresponding PV if PVC references one + if test.pvc.Spec.VolumeName != "" { + pv := builder.ForPersistentVolume(test.pvc.Spec.VolumeName). + CSI("csi.driver", "test-volume-id"). + ClaimRef(test.pvc.Namespace, test.pvc.Name).Result() + err = r.client.Create(ctx, pv) + require.NoError(t, err) + } } if test.dataMgr != nil { diff --git a/pkg/controller/download_request_controller.go b/pkg/controller/download_request_controller.go index bd9565895..02d385bec 100644 --- a/pkg/controller/download_request_controller.go +++ b/pkg/controller/download_request_controller.go @@ -20,7 +20,7 @@ import ( "context" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/controller/gc_controller.go b/pkg/controller/gc_controller.go index bb3c60ae5..6b3ade484 100644 --- a/pkg/controller/gc_controller.go +++ b/pkg/controller/gc_controller.go @@ -21,7 +21,7 @@ import ( "fmt" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" clocks "k8s.io/utils/clock" diff --git a/pkg/controller/pod_volume_backup_controller.go b/pkg/controller/pod_volume_backup_controller.go index 0bcbfa6d2..2372bf25b 100644 --- a/pkg/controller/pod_volume_backup_controller.go +++ b/pkg/controller/pod_volume_backup_controller.go @@ -22,7 +22,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/controller/pod_volume_backup_controller_test.go b/pkg/controller/pod_volume_backup_controller_test.go index b49d7eb5b..8b05f0e3b 100644 --- a/pkg/controller/pod_volume_backup_controller_test.go +++ b/pkg/controller/pod_volume_backup_controller_test.go @@ -22,7 +22,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/controller/pod_volume_restore_controller.go b/pkg/controller/pod_volume_restore_controller.go index f40de528b..3e2fba39c 100644 --- a/pkg/controller/pod_volume_restore_controller.go +++ b/pkg/controller/pod_volume_restore_controller.go @@ -22,7 +22,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/controller/pod_volume_restore_controller_test.go b/pkg/controller/pod_volume_restore_controller_test.go index 1819cbc32..4401a7c32 100644 --- a/pkg/controller/pod_volume_restore_controller_test.go +++ b/pkg/controller/pod_volume_restore_controller_test.go @@ -22,7 +22,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/controller/restore_controller.go b/pkg/controller/restore_controller.go index 469951f27..5b055bc6c 100644 --- a/pkg/controller/restore_controller.go +++ b/pkg/controller/restore_controller.go @@ -28,7 +28,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -44,6 +44,7 @@ import ( "github.com/vmware-tanzu/velero/internal/hook" "github.com/vmware-tanzu/velero/internal/resourcemodifiers" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/volume" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/constant" @@ -232,7 +233,7 @@ func (r *restoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct original := restore.DeepCopy() // Validate the restore and fetch the backup - info, resourceModifiers := r.validateAndComplete(restore) + info, resourceModifiers, restoreResPolicies := r.validateAndComplete(ctx, restore) // Register attempts after validation so we don't have to fetch the backup multiple times backupScheduleName := restore.Spec.ScheduleName @@ -267,7 +268,7 @@ func (r *restoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, nil } - if err := r.runValidatedRestore(restore, info, resourceModifiers); err != nil { + if err := r.runValidatedRestore(restore, info, resourceModifiers, restoreResPolicies); err != nil { log.WithError(err).Debug("Restore failed") restore.Status.Phase = api.RestorePhaseFailed restore.Status.FailureReason = err.Error() @@ -303,7 +304,7 @@ func (r *restoreReconciler) SetupWithManager(mgr ctrl.Manager) error { Complete(r) } -func (r *restoreReconciler) validateAndComplete(restore *api.Restore) (backupInfo, *resourcemodifiers.ResourceModifiers) { +func (r *restoreReconciler) validateAndComplete(ctx context.Context, restore *api.Restore) (backupInfo, *resourcemodifiers.ResourceModifiers, *resourcepolicies.Policies) { // add non-restorable resources to restore's excluded resources excludedResources := sets.NewString(restore.Spec.ExcludedResources...) for _, nonrestorable := range nonRestorableResources { @@ -338,7 +339,7 @@ func (r *restoreReconciler) validateAndComplete(restore *api.Restore) (backupInf // validate that exactly one of BackupName and ScheduleName have been specified if !backupXorScheduleProvided(restore) { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "Either a backup or schedule must be specified as a source for the restore, but not both") - return backupInfo{}, nil + return backupInfo{}, nil, nil } // validate Restore Init Hook's InitContainers @@ -372,9 +373,9 @@ func (r *restoreReconciler) validateAndComplete(restore *api.Restore) (backupInf })) backupList := &api.BackupList{} - if err := r.kbClient.List(context.Background(), backupList, &client.ListOptions{LabelSelector: selector}); err != nil { + if err := r.kbClient.List(ctx, backupList, &client.ListOptions{LabelSelector: selector}); err != nil { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "Unable to list backups for schedule") - return backupInfo{}, nil + return backupInfo{}, nil, nil } if len(backupList.Items) == 0 { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "No backups found for schedule") @@ -384,19 +385,30 @@ func (r *restoreReconciler) validateAndComplete(restore *api.Restore) (backupInf restore.Spec.BackupName = backup.Name } else { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "No completed backups found for schedule") - return backupInfo{}, nil + return backupInfo{}, nil, nil } } info, err := r.fetchBackupInfo(restore.Spec.BackupName) if err != nil { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("Error retrieving backup: %v", err)) - return backupInfo{}, nil + return backupInfo{}, nil, nil } if !veleroutil.BSLIsAvailable(*info.location) { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("The BSL %s is unavailable, cannot retrieve the backup", info.location.Name)) - return backupInfo{}, nil + return backupInfo{}, nil, nil + } + + // reject restores from backups that are not in a usable phase + switch info.backup.Status.Phase { + case api.BackupPhaseCompleted, api.BackupPhasePartiallyFailed: + // ok + default: + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, + fmt.Sprintf("backup %q is in phase %q and cannot be used as a restore source", + info.backup.Name, info.backup.Status.Phase)) + return backupInfo{}, nil, nil } // Fill in the ScheduleName so it's easier to consume for metrics. @@ -404,26 +416,40 @@ func (r *restoreReconciler) validateAndComplete(restore *api.Restore) (backupInf restore.Spec.ScheduleName = info.backup.GetLabels()[api.ScheduleNameLabel] } + var restoreResPolicies *resourcepolicies.Policies + if restore.Spec.ResourcePolicy != nil { + var err error + restoreResPolicies, err = resourcepolicies.GetResourcePoliciesFromRestore( + ctx, restore, r.kbClient, r.logger, + ) + if err != nil { + restore.Status.ValidationErrors = append( + restore.Status.ValidationErrors, err.Error(), + ) + return backupInfo{}, nil, nil + } + } + var resourceModifiers *resourcemodifiers.ResourceModifiers if restore.Spec.ResourceModifier != nil && strings.EqualFold(restore.Spec.ResourceModifier.Kind, resourcemodifiers.ConfigmapRefType) { ResourceModifierConfigMap := &corev1api.ConfigMap{} - err := r.kbClient.Get(context.Background(), client.ObjectKey{Namespace: restore.Namespace, Name: restore.Spec.ResourceModifier.Name}, ResourceModifierConfigMap) + err := r.kbClient.Get(ctx, client.ObjectKey{Namespace: restore.Namespace, Name: restore.Spec.ResourceModifier.Name}, ResourceModifierConfigMap) if err != nil { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("failed to get resource modifiers configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name)) - return backupInfo{}, nil + return backupInfo{}, nil, nil } resourceModifiers, err = resourcemodifiers.GetResourceModifiersFromConfig(ResourceModifierConfigMap) if err != nil { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, errors.Wrapf(err, "Error in parsing resource modifiers provided in configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name).Error()) - return backupInfo{}, nil + return backupInfo{}, nil, nil } else if err = resourceModifiers.Validate(); err != nil { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, errors.Wrapf(err, "Validation error in resource modifiers provided in configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name).Error()) - return backupInfo{}, nil + return backupInfo{}, nil, nil } r.logger.Infof("Retrieved Resource modifiers provided in configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name) } - return info, resourceModifiers + return info, resourceModifiers, restoreResPolicies } // backupXorScheduleProvided returns true if exactly one of BackupName and @@ -496,7 +522,7 @@ func fetchBackupInfoInternal(kbClient client.Client, namespace, backupName strin // The log and results files are uploaded to backup storage. Any error returned from this function // means that the restore failed. This function updates the restore API object with warning and error // counts, but *does not* update its phase or patch it via the API. -func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backupInfo, resourceModifiers *resourcemodifiers.ResourceModifiers) error { +func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backupInfo, resourceModifiers *resourcemodifiers.ResourceModifiers, restoreResPolicies *resourcepolicies.Policies) error { // instantiate the per-restore logger that will output both to a temp file // (for upload to object storage) and to stdout. restoreLog, err := logging.NewTempFileLogger(r.restoreLogLevel, r.logFormat, nil, logrus.Fields{"restore": kubeutil.NamespaceAndName(restore)}) @@ -575,6 +601,7 @@ func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backu VolumeSnapshots: volumeSnapshots, BackupReader: backupFile, ResourceModifiers: resourceModifiers, + ResPolicies: restoreResPolicies, DisableInformerCache: r.disableInformerCache, CSIVolumeSnapshots: csiVolumeSnapshots, BackupVolumeInfoMap: backupVolumeInfoMap, diff --git a/pkg/controller/restore_controller_test.go b/pkg/controller/restore_controller_test.go index b013ee64d..6a2f4d8d1 100644 --- a/pkg/controller/restore_controller_test.go +++ b/pkg/controller/restore_controller_test.go @@ -22,8 +22,8 @@ import ( "testing" "time" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -305,7 +305,7 @@ func TestRestoreReconcile(t *testing.T) { name: "restorer throwing an error causes the restore to fail", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(), - backup: defaultBackup().StorageLocation("default").Result(), + backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseCompleted).Result(), restorerError: errors.New("blarg"), expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseInProgress), @@ -319,7 +319,7 @@ func TestRestoreReconcile(t *testing.T) { name: "valid restore with none existingresourcepolicy gets executed", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).ExistingResourcePolicy("none").Result(), - backup: defaultBackup().StorageLocation("default").Result(), + backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseCompleted).Result(), expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseInProgress), expectedStartTime: ×tamp, @@ -330,7 +330,7 @@ func TestRestoreReconcile(t *testing.T) { name: "valid restore with update existingresourcepolicy gets executed", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).ExistingResourcePolicy("update").Result(), - backup: defaultBackup().StorageLocation("default").Result(), + backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseCompleted).Result(), expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseInProgress), expectedStartTime: ×tamp, @@ -352,7 +352,7 @@ func TestRestoreReconcile(t *testing.T) { name: "valid restore gets executed", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(), - backup: defaultBackup().StorageLocation("default").Result(), + backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseCompleted).Result(), expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseInProgress), expectedStartTime: ×tamp, @@ -363,7 +363,7 @@ func TestRestoreReconcile(t *testing.T) { name: "valid restore gets executed and only includes pod volume backups from restore namespace", location: defaultStorageLocation, restore: NewRestore("foo", "bar2", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(), - backup: defaultBackup().StorageLocation("default").Result(), + backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseCompleted).Result(), podVolumeBackups: []*velerov1api.PodVolumeBackup{ builder.ForPodVolumeBackup("foo", "pvb-1").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), builder.ForPodVolumeBackup("other-ns", "pvb-2").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), @@ -444,7 +444,7 @@ func TestRestoreReconcile(t *testing.T) { expectedStartTime: ×tamp, expectedCompletedTime: ×tamp, backupStoreGetBackupContentsErr: errors.New("Couldn't download backup"), - backup: defaultBackup().StorageLocation("default").Result(), + backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseCompleted).Result(), }, { name: "restore attached with an expected finalizer gets cleaned up successfully", @@ -473,7 +473,7 @@ func TestRestoreReconcile(t *testing.T) { name: "valid restore with empty VolumeInfos", location: defaultStorageLocation, restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(), - backup: defaultBackup().StorageLocation("default").Result(), + backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseCompleted).Result(), emptyVolumeInfo: true, expectedErr: false, expectedPhase: string(velerov1api.RestorePhaseInProgress), @@ -497,6 +497,44 @@ func TestRestoreReconcile(t *testing.T) { backup: defaultBackup().StorageLocation("default").Result(), expectedErr: true, }, + { + name: "restore from backup in Deleting phase fails validation", + location: defaultStorageLocation, + restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(), + backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseDeleting).Result(), + expectedErr: false, + expectedPhase: string(velerov1api.RestorePhaseFailedValidation), + expectedValidationErrors: []string{`backup "backup-1" is in phase "Deleting" and cannot be used as a restore source`}, + }, + { + name: "restore from backup in InProgress phase fails validation", + location: defaultStorageLocation, + restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(), + backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseInProgress).Result(), + expectedErr: false, + expectedPhase: string(velerov1api.RestorePhaseFailedValidation), + expectedValidationErrors: []string{`backup "backup-1" is in phase "InProgress" and cannot be used as a restore source`}, + }, + { + name: "restore from backup in PartiallyFailed phase succeeds", + location: defaultStorageLocation, + restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(), + backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhasePartiallyFailed).Result(), + expectedErr: false, + expectedPhase: string(velerov1api.RestorePhaseInProgress), + expectedStartTime: ×tamp, + expectedCompletedTime: ×tamp, + expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseInProgress).Result(), + }, + { + name: "restore from backup in Failed phase fails validation", + location: defaultStorageLocation, + restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(), + backup: defaultBackup().StorageLocation("default").Phase(velerov1api.BackupPhaseFailed).Result(), + expectedErr: false, + expectedPhase: string(velerov1api.RestorePhaseFailedValidation), + expectedValidationErrors: []string{`backup "backup-1" is in phase "Failed" and cannot be used as a restore source`}, + }, } formatFlag := logging.FormatText @@ -747,7 +785,7 @@ func TestValidateAndCompleteWhenScheduleNameSpecified(t *testing.T) { Phase(velerov1api.BackupPhaseCompleted). Result())) - r.validateAndComplete(restore) + r.validateAndComplete(t.Context(), restore) assert.Contains(t, restore.Status.ValidationErrors, "No backups found for schedule") assert.Empty(t, restore.Spec.BackupName) @@ -763,7 +801,7 @@ func TestValidateAndCompleteWhenScheduleNameSpecified(t *testing.T) { Result(), )) - r.validateAndComplete(restore) + r.validateAndComplete(t.Context(), restore) assert.Contains(t, restore.Status.ValidationErrors, "No completed backups found for schedule") assert.Empty(t, restore.Spec.BackupName) @@ -794,11 +832,140 @@ func TestValidateAndCompleteWhenScheduleNameSpecified(t *testing.T) { ScheduleName: "schedule-1", }, } - r.validateAndComplete(restore) + r.validateAndComplete(t.Context(), restore) assert.Nil(t, restore.Status.ValidationErrors) assert.Equal(t, "foo", restore.Spec.BackupName) } +func TestValidateAndCompleteWithResourcePolicySpecified(t *testing.T) { + formatFlag := logging.FormatText + + var ( + logger = velerotest.NewLogger() + pluginManager = &pluginmocks.Manager{} + fakeClient = velerotest.NewFakeControllerRuntimeClient(t) + fakeGlobalClient = velerotest.NewFakeControllerRuntimeClient(t) + backupStore = &persistencemocks.BackupStore{} + ) + + r := NewRestoreReconciler( + t.Context(), + velerov1api.DefaultNamespace, + nil, + fakeClient, + logger, + logrus.DebugLevel, + func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, + NewFakeSingleObjectBackupStoreGetter(backupStore), + metrics.NewServerMetrics(), + formatFlag, + 60*time.Minute, + false, + fakeGlobalClient, + 10*time.Minute, + ) + + restore := &velerov1api.Restore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1api.DefaultNamespace, + Name: "restore-1", + }, + Spec: velerov1api.RestoreSpec{ + BackupName: "backup-1", + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: "configmap", + Name: "test-configmap", + }, + }, + } + + location := builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() + require.NoError(t, r.kbClient.Create(t.Context(), location)) + + require.NoError(t, r.kbClient.Create( + t.Context(), + defaultBackup(). + ObjectMeta( + builder.WithName("backup-1"), + ).StorageLocation("default"). + Phase(velerov1api.BackupPhaseCompleted). + Result(), + )) + + r.validateAndComplete(t.Context(), restore) + assert.Contains(t, restore.Status.ValidationErrors[0], "fail to get ResourcePolicies velero/test-configmap ConfigMap") + + restore1 := &velerov1api.Restore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1api.DefaultNamespace, + Name: "restore-1", + }, + Spec: velerov1api.RestoreSpec{ + BackupName: "backup-1", + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + Kind: "configmap", + Name: "test-configmap", + }, + }, + } + + cm1 := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: velerov1api.DefaultNamespace, + }, + Data: map[string]string{ + "policy.yaml": `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: + - pods +`, + }, + } + require.NoError(t, r.kbClient.Create(t.Context(), cm1)) + + r.validateAndComplete(t.Context(), restore1) + assert.Nil(t, restore1.Status.ValidationErrors) + + restore2 := &velerov1api.Restore{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: velerov1api.DefaultNamespace, + Name: "restore-1", + }, + Spec: velerov1api.RestoreSpec{ + BackupName: "backup-1", + ResourcePolicy: &corev1api.TypedLocalObjectReference{ + // intentional to ensure case insensitivity works as expected + Kind: "confIGMaP", + Name: "test-configmap-invalid", + }, + }, + } + + cm2 := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap-invalid", + Namespace: velerov1api.DefaultNamespace, + }, + Data: map[string]string{ + "policy.yaml": `version: v1 +volumePolicies: + - conditions: + capacity: '0,10Gi' + csi: + driver: disks.csi.driver + action: + type: invalid_action +`, + }, + } + require.NoError(t, r.kbClient.Create(t.Context(), cm2)) + + r.validateAndComplete(t.Context(), restore2) + assert.Contains(t, restore2.Status.ValidationErrors[0], "fail to validate ResourcePolicies in ConfigMap velero/test-configmap-invalid") +} + func TestValidateAndCompleteWithResourceModifierSpecified(t *testing.T) { formatFlag := logging.FormatText @@ -854,7 +1021,7 @@ func TestValidateAndCompleteWithResourceModifierSpecified(t *testing.T) { Result(), )) - r.validateAndComplete(restore) + r.validateAndComplete(t.Context(), restore) assert.Contains(t, restore.Status.ValidationErrors[0], "failed to get resource modifiers configmap") restore1 := &velerov1api.Restore{ @@ -882,7 +1049,7 @@ func TestValidateAndCompleteWithResourceModifierSpecified(t *testing.T) { } require.NoError(t, r.kbClient.Create(t.Context(), cm1)) - r.validateAndComplete(restore1) + r.validateAndComplete(t.Context(), restore1) assert.Nil(t, restore1.Status.ValidationErrors) restore2 := &velerov1api.Restore{ @@ -911,7 +1078,7 @@ func TestValidateAndCompleteWithResourceModifierSpecified(t *testing.T) { } require.NoError(t, r.kbClient.Create(t.Context(), invalidVersionCm)) - r.validateAndComplete(restore2) + r.validateAndComplete(t.Context(), restore2) assert.Contains(t, restore2.Status.ValidationErrors[0], "Error in parsing resource modifiers provided in configmap") restore3 := &velerov1api.Restore{ @@ -939,7 +1106,7 @@ func TestValidateAndCompleteWithResourceModifierSpecified(t *testing.T) { } require.NoError(t, r.kbClient.Create(t.Context(), invalidOperatorCm)) - r.validateAndComplete(restore3) + r.validateAndComplete(t.Context(), restore3) assert.Contains(t, restore3.Status.ValidationErrors[0], "Validation error in resource modifiers provided in configmap") } diff --git a/pkg/controller/restore_finalizer_controller.go b/pkg/controller/restore_finalizer_controller.go index f82216bc3..4e02bb0ef 100644 --- a/pkg/controller/restore_finalizer_controller.go +++ b/pkg/controller/restore_finalizer_controller.go @@ -22,9 +22,9 @@ import ( "sync" "time" + "github.com/cockroachdb/errors" volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" storagev1api "k8s.io/api/storage/v1" @@ -301,8 +301,10 @@ func (ctx *finalizerContext) execute() (results.Result, results.Result) { pdpErrs := ctx.patchDynamicPVWithVolumeInfo() errs.Merge(&pdpErrs) - vgscWarnings := ctx.cleanupStubVGSC() - warnings.Merge(&vgscWarnings) + if ctx.hasVolumeGroupSnapshotHandles() { + vgscWarnings := ctx.cleanupStubVGSC() + warnings.Merge(&vgscWarnings) + } rehErrs := ctx.WaitRestoreExecHook() errs.Merge(&rehErrs) @@ -419,7 +421,16 @@ func (ctx *finalizerContext) patchDynamicPVWithVolumeInfo() (errs results.Result // patch PV's reclaim policy and label using the corresponding data stored in volume info if needPatch(pv, volInfo.PVInfo) { updatedPV := pv.DeepCopy() - updatedPV.Labels = volInfo.PVInfo.Labels + + if updatedPV.Labels == nil { + updatedPV.Labels = make(map[string]string) + } + for k, v := range volInfo.PVInfo.Labels { + if _, exists := updatedPV.Labels[k]; !exists { + updatedPV.Labels[k] = v + } + } + updatedPV.Spec.PersistentVolumeReclaimPolicy = corev1api.PersistentVolumeReclaimPolicy(volInfo.PVInfo.ReclaimPolicy) if err := kubeutil.PatchResource(pv, updatedPV, ctx.crClient); err != nil { return false, err @@ -449,6 +460,15 @@ func (ctx *finalizerContext) patchDynamicPVWithVolumeInfo() (errs results.Result return errs } +func (ctx *finalizerContext) hasVolumeGroupSnapshotHandles() bool { + for _, vi := range ctx.volumeInfo { + if vi.CSISnapshotInfo != nil && vi.CSISnapshotInfo.VolumeGroupSnapshotHandle != "" { + return true + } + } + return false +} + // cleanupStubVGSC deletes stub VolumeGroupSnapshotContent objects that were // created during restore to satisfy CSI controller validation. These stubs are // labeled with velero.io/restore-name for identification. @@ -542,13 +562,10 @@ func needPatch(newPV *corev1api.PersistentVolume, pvInfo *volume.PVInfo) bool { } newPVLabels, pvLabels := newPV.Labels, pvInfo.Labels - for k, v := range pvLabels { + for k := range pvLabels { if _, ok := newPVLabels[k]; !ok { return true } - if newPVLabels[k] != v { - return true - } } return false diff --git a/pkg/controller/restore_finalizer_controller_test.go b/pkg/controller/restore_finalizer_controller_test.go index 6fb5ba303..8f2618f9d 100644 --- a/pkg/controller/restore_finalizer_controller_test.go +++ b/pkg/controller/restore_finalizer_controller_test.go @@ -634,6 +634,87 @@ func Test_restoreFinalizerReconciler_finishProcessing(t *testing.T) { } } +func TestNeedPatch(t *testing.T) { + tests := []struct { + name string + newPV *corev1api.PersistentVolume + pvInfo *volume.PVInfo + expected bool + }{ + { + name: "reclaim policy differs", + newPV: builder.ForPersistentVolume("pv1"). + ReclaimPolicy(corev1api.PersistentVolumeReclaimDelete).Result(), + pvInfo: &volume.PVInfo{ + ReclaimPolicy: string(corev1api.PersistentVolumeReclaimRetain), + Labels: map[string]string{}, + }, + expected: true, + }, + { + name: "backup has label new PV does not", + newPV: builder.ForPersistentVolume("pv1"). + ObjectMeta(builder.WithLabels("existing", "val")). + ReclaimPolicy(corev1api.PersistentVolumeReclaimDelete).Result(), + pvInfo: &volume.PVInfo{ + ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), + Labels: map[string]string{"existing": "val", "missing": "val"}, + }, + expected: true, + }, + { + name: "same labels same values", + newPV: builder.ForPersistentVolume("pv1"). + ObjectMeta(builder.WithLabels("key", "val")). + ReclaimPolicy(corev1api.PersistentVolumeReclaimDelete).Result(), + pvInfo: &volume.PVInfo{ + ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), + Labels: map[string]string{"key": "val"}, + }, + expected: false, + }, + { + name: "same label key different values", + newPV: builder.ForPersistentVolume("pv1"). + ObjectMeta(builder.WithLabels("topology.kubernetes.io/zone", "us-west-2a")). + ReclaimPolicy(corev1api.PersistentVolumeReclaimDelete).Result(), + pvInfo: &volume.PVInfo{ + ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), + Labels: map[string]string{"topology.kubernetes.io/zone": "us-east-1a"}, + }, + expected: false, + }, + { + name: "new PV has labels backup does not", + newPV: builder.ForPersistentVolume("pv1"). + ObjectMeta(builder.WithLabels("provisioner-label", "val")). + ReclaimPolicy(corev1api.PersistentVolumeReclaimDelete).Result(), + pvInfo: &volume.PVInfo{ + ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), + Labels: map[string]string{}, + }, + expected: false, + }, + { + name: "both labels nil", + newPV: builder.ForPersistentVolume("pv1"). + ReclaimPolicy(corev1api.PersistentVolumeReclaimDelete).Result(), + pvInfo: &volume.PVInfo{ + ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), + Labels: nil, + }, + expected: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := needPatch(tc.newPV, tc.pvInfo) + assert.Equal(t, tc.expected, result) + }) + } +} + func TestRestoreOperationList(t *testing.T) { var empty []*itemoperation.RestoreOperation tests := []struct { @@ -743,6 +824,83 @@ func TestRestoreOperationList(t *testing.T) { } } +func TestHasVolumeGroupSnapshotHandles(t *testing.T) { + tests := []struct { + name string + volumeInfo []*volume.BackupVolumeInfo + expected bool + }{ + { + name: "nil volumeInfo", + volumeInfo: nil, + expected: false, + }, + { + name: "empty volumeInfo", + volumeInfo: []*volume.BackupVolumeInfo{}, + expected: false, + }, + { + name: "no CSISnapshotInfo", + volumeInfo: []*volume.BackupVolumeInfo{ + {PVCName: "pvc-1", BackupMethod: volume.NativeSnapshot}, + }, + expected: false, + }, + { + name: "CSISnapshotInfo with empty VolumeGroupSnapshotHandle", + volumeInfo: []*volume.BackupVolumeInfo{ + { + PVCName: "pvc-1", + BackupMethod: volume.CSISnapshot, + CSISnapshotInfo: &volume.CSISnapshotInfo{ + SnapshotHandle: "snap-1", + }, + }, + }, + expected: false, + }, + { + name: "one volume with VolumeGroupSnapshotHandle", + volumeInfo: []*volume.BackupVolumeInfo{ + { + PVCName: "pvc-1", + BackupMethod: volume.CSISnapshot, + CSISnapshotInfo: &volume.CSISnapshotInfo{ + SnapshotHandle: "snap-1", + VolumeGroupSnapshotHandle: "vgs-handle-1", + }, + }, + }, + expected: true, + }, + { + name: "mixed volumes only one with VolumeGroupSnapshotHandle", + volumeInfo: []*volume.BackupVolumeInfo{ + {PVCName: "pvc-1", BackupMethod: volume.NativeSnapshot}, + { + PVCName: "pvc-2", + BackupMethod: volume.CSISnapshot, + CSISnapshotInfo: &volume.CSISnapshotInfo{ + SnapshotHandle: "snap-2", + VolumeGroupSnapshotHandle: "vgs-handle-2", + }, + }, + }, + expected: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := &finalizerContext{ + volumeInfo: tc.volumeInfo, + } + assert.Equal(t, tc.expected, ctx.hasVolumeGroupSnapshotHandles()) + }) + } +} + func TestCleanupStubVGSC(t *testing.T) { snapshotHandle1 := "snap-handle-1" snapshotHandle2 := "snap-handle-2" diff --git a/pkg/controller/restore_operations_controller.go b/pkg/controller/restore_operations_controller.go index 0539e21a4..301e5f438 100644 --- a/pkg/controller/restore_operations_controller.go +++ b/pkg/controller/restore_operations_controller.go @@ -20,7 +20,7 @@ import ( "context" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/controller/schedule_controller.go b/pkg/controller/schedule_controller.go index 443b3c08b..d71c86ca4 100644 --- a/pkg/controller/schedule_controller.go +++ b/pkg/controller/schedule_controller.go @@ -21,8 +21,8 @@ import ( "fmt" "time" - "github.com/pkg/errors" - cron "github.com/robfig/cron/v3" + "github.com/cockroachdb/errors" + cron "github.com/netresearch/go-cron" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/controller/schedule_controller_test.go b/pkg/controller/schedule_controller_test.go index f4585763c..85b87474a 100644 --- a/pkg/controller/schedule_controller_test.go +++ b/pkg/controller/schedule_controller_test.go @@ -20,14 +20,14 @@ import ( "testing" "time" - cron "github.com/robfig/cron/v3" + cron "github.com/netresearch/go-cron" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" testclocks "k8s.io/utils/clock/testing" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -94,7 +94,7 @@ func TestReconcileOfSchedule(t *testing.T) { }, { name: "schedule with phase New and SkipImmediately gets validated and does not trigger a backup", - schedule: newScheduleBuilder(velerov1.SchedulePhaseNew).CronSchedule("@every 5m").SkipImmediately(pointer.Bool(true)).Result(), + schedule: newScheduleBuilder(velerov1.SchedulePhaseNew).CronSchedule("@every 5m").SkipImmediately(ptr.To(true)).Result(), fakeClockTime: "2017-01-01 12:00:00", expectedPhase: string(velerov1.SchedulePhaseEnabled), expectedLastSkipped: "2017-01-01 12:00:00", @@ -123,7 +123,7 @@ func TestReconcileOfSchedule(t *testing.T) { }, { name: "schedule that's already run but has SkippedImmediately=false gets LastBackup updated", - schedule: newScheduleBuilder(velerov1.SchedulePhaseEnabled).CronSchedule("@every 5m").LastBackupTime("2000-01-01 00:00:00").SkipImmediately(pointer.Bool(false)).Result(), + schedule: newScheduleBuilder(velerov1.SchedulePhaseEnabled).CronSchedule("@every 5m").LastBackupTime("2000-01-01 00:00:00").SkipImmediately(ptr.To(false)).Result(), fakeClockTime: "2017-01-01 12:00:00", expectedBackupCreate: builder.ForBackup("ns", "name-20170101120000").ObjectMeta(builder.WithLabels(velerov1.ScheduleNameLabel, "name")).Result(), expectedLastBackup: "2017-01-01 12:00:00", @@ -138,7 +138,7 @@ func TestReconcileOfSchedule(t *testing.T) { }, { name: "schedule that's already run but has SkippedImmediately=true do not get LastBackup updated", - schedule: newScheduleBuilder(velerov1.SchedulePhaseEnabled).CronSchedule("@every 5m").LastBackupTime("2000-01-01 00:00:00").SkipImmediately(pointer.Bool(true)).Result(), + schedule: newScheduleBuilder(velerov1.SchedulePhaseEnabled).CronSchedule("@every 5m").LastBackupTime("2000-01-01 00:00:00").SkipImmediately(ptr.To(true)).Result(), fakeClockTime: "2017-01-01 12:00:00", expectedLastBackup: "2000-01-01 00:00:00", expectedLastSkipped: "2017-01-01 12:00:00", @@ -216,7 +216,7 @@ func TestReconcileOfSchedule(t *testing.T) { // we expect reconcile to flip SkipImmediately to false if it's true or the server is configured to skip immediately and the schedule doesn't have it set if scheduleb4reconcile.Spec.SkipImmediately != nil && *scheduleb4reconcile.Spec.SkipImmediately || test.reconcilerSkipImmediately && scheduleb4reconcile.Spec.SkipImmediately == nil { - assert.Equal(t, schedule.Spec.SkipImmediately, pointer.Bool(false)) + assert.Equal(t, schedule.Spec.SkipImmediately, ptr.To(false)) } backups := &velerov1.BackupList{} diff --git a/pkg/controller/server_status_request_controller.go b/pkg/controller/server_status_request_controller.go index 3fb1af80b..c779f6f1b 100644 --- a/pkg/controller/server_status_request_controller.go +++ b/pkg/controller/server_status_request_controller.go @@ -20,7 +20,7 @@ import ( "context" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/datamover/backup_micro_service.go b/pkg/datamover/backup_micro_service.go index 4edd24c97..08a005217 100644 --- a/pkg/datamover/backup_micro_service.go +++ b/pkg/datamover/backup_micro_service.go @@ -21,7 +21,7 @@ import ( "encoding/json" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" @@ -67,6 +67,10 @@ type BackupMicroService struct { duInformer cache.Informer duHandler cachetool.ResourceEventHandlerRegistration nodeName string + + changeID string + volumeID string + snapshotID string } type dataPathResult struct { @@ -76,7 +80,7 @@ type dataPathResult struct { func NewBackupMicroService(ctx context.Context, client client.Client, kubeClient kubernetes.Interface, dataUploadName string, namespace string, nodeName string, sourceTargetPath datapath.AccessPoint, dataPathMgr *datapath.Manager, repoEnsurer *repository.Ensurer, cred *credentials.CredentialGetter, - duInformer cache.Informer, log logrus.FieldLogger) *BackupMicroService { + duInformer cache.Informer, changeID string, volumeID string, snapshotID string, log logrus.FieldLogger) *BackupMicroService { return &BackupMicroService{ ctx: ctx, client: client, @@ -91,6 +95,9 @@ func NewBackupMicroService(ctx context.Context, client client.Client, kubeClient nodeName: nodeName, resultSignal: make(chan dataPathResult), duInformer: duInformer, + changeID: changeID, + volumeID: volumeID, + snapshotID: snapshotID, } } @@ -200,6 +207,9 @@ func (r *BackupMicroService) RunCancelableDataPath(ctx context.Context) (string, ParentSnapshot: "", ForceFull: false, Tags: tags, + VolumeID: r.volumeID, + ChangeID: r.changeID, + SnapshotID: r.snapshotID, }); err != nil { return "", errors.Wrap(err, "error starting data path backup") } diff --git a/pkg/datamover/backup_micro_service_test.go b/pkg/datamover/backup_micro_service_test.go index 79b4a834a..e6291244b 100644 --- a/pkg/datamover/backup_micro_service_test.go +++ b/pkg/datamover/backup_micro_service_test.go @@ -23,27 +23,22 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/runtime" - - "github.com/vmware-tanzu/velero/pkg/builder" - "github.com/vmware-tanzu/velero/pkg/datapath" - "github.com/vmware-tanzu/velero/pkg/uploader" - - velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - + kbclient "sigs.k8s.io/controller-runtime/pkg/client" clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" - velerotest "github.com/vmware-tanzu/velero/pkg/test" - - kbclient "sigs.k8s.io/controller-runtime/pkg/client" - + "github.com/vmware-tanzu/velero/pkg/builder" + "github.com/vmware-tanzu/velero/pkg/datapath" datapathmockes "github.com/vmware-tanzu/velero/pkg/datapath/mocks" + velerotest "github.com/vmware-tanzu/velero/pkg/test" + "github.com/vmware-tanzu/velero/pkg/uploader" ) type backupMsTestHelper struct { @@ -294,7 +289,10 @@ func TestCancelDataUpload(t *testing.T) { func TestRunCancelableDataPath(t *testing.T) { dataUploadName := "fake-data-upload" du := builder.ForDataUpload(velerov1api.DefaultNamespace, dataUploadName).Phase(velerov2alpha1api.DataUploadPhaseNew).Result() - duInProgress := builder.ForDataUpload(velerov1api.DefaultNamespace, dataUploadName).Phase(velerov2alpha1api.DataUploadPhaseInProgress).Result() + duInProgress := builder.ForDataUpload(velerov1api.DefaultNamespace, dataUploadName).Phase(velerov2alpha1api.DataUploadPhaseInProgress).CSISnapshot( + &velerov2alpha1api.CSISnapshotSpec{ + VolumeSnapshot: "fake-snapshot", + }).Result() ctxTimeout, cancel := context.WithTimeout(t.Context(), time.Second) tests := []struct { diff --git a/pkg/datamover/dataupload_delete_action.go b/pkg/datamover/dataupload_delete_action.go index 46c62a1f1..a50d0fce2 100644 --- a/pkg/datamover/dataupload_delete_action.go +++ b/pkg/datamover/dataupload_delete_action.go @@ -5,7 +5,7 @@ import ( "encoding/json" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -17,6 +17,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/plugin/velero" repotypes "github.com/vmware-tanzu/velero/pkg/repository/types" + datamoverutil "github.com/vmware-tanzu/velero/pkg/util/datamover" ) type DataUploadDeleteAction struct { @@ -88,7 +89,7 @@ func (d *DataUploadDeleteAction) Execute(input *velero.DeleteItemActionExecuteIn // generate the configmap which is to be created and used as a way to communicate the snapshot info to the backup deletion controller func genConfigmap(bak *velerov1.Backup, du velerov2alpha1.DataUpload) *corev1api.ConfigMap { - if !IsBuiltInDataMover(du.Spec.DataMover) || du.Status.SnapshotID == "" { + if !datamoverutil.IsBuiltInDataMover(du.Spec.DataMover) || du.Status.SnapshotID == "" { return nil } snapshot := repotypes.SnapshotIdentifier{ diff --git a/pkg/datamover/restore_micro_service.go b/pkg/datamover/restore_micro_service.go index e1c95ee3d..d918667f9 100644 --- a/pkg/datamover/restore_micro_service.go +++ b/pkg/datamover/restore_micro_service.go @@ -20,7 +20,7 @@ import ( "context" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/datamover/restore_micro_service_test.go b/pkg/datamover/restore_micro_service_test.go index bce4d94b7..33e22eab3 100644 --- a/pkg/datamover/restore_micro_service_test.go +++ b/pkg/datamover/restore_micro_service_test.go @@ -22,7 +22,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/datamover/util.go b/pkg/datamover/util.go index 7e37695b6..ed66d497a 100644 --- a/pkg/datamover/util.go +++ b/pkg/datamover/util.go @@ -16,25 +16,20 @@ limitations under the License. package datamover -import "fmt" +import ( + "fmt" -const ( - DataMoverTypeVeleroFs string = "velero-fs" - DataMoverTypeVeleroBlock string = "velero-block" + datamoverutil "github.com/vmware-tanzu/velero/pkg/util/datamover" ) func GetUploaderType(dataMover string) string { - if dataMover == "" || dataMover == "velero" { + if datamoverutil.IsBuiltInDataMover(dataMover) { return "kopia" } else { return dataMover } } -func IsBuiltInDataMover(dataMover string) bool { - return dataMover == "" || dataMover == "velero" -} - func GetRealSource(sourceNamespace string, pvcName string) string { return fmt.Sprintf("%s/%s", sourceNamespace, pvcName) } diff --git a/pkg/datamover/util_test.go b/pkg/datamover/util_test.go index 80e2f4e16..d44f3c307 100644 --- a/pkg/datamover/util_test.go +++ b/pkg/datamover/util_test.go @@ -6,35 +6,6 @@ import ( "github.com/stretchr/testify/assert" ) -func TestIsBuiltInUploader(t *testing.T) { - testcases := []struct { - name string - dataMover string - want bool - }{ - { - name: "empty dataMover is builtin", - dataMover: "", - want: true, - }, - { - name: "velero dataMover is builtin", - dataMover: "velero", - want: true, - }, - { - name: "kopia dataMover is not builtin", - dataMover: "kopia", - want: false, - }, - } - for _, tc := range testcases { - t.Run(tc.name, func(tt *testing.T) { - assert.Equal(tt, tc.want, IsBuiltInDataMover(tc.dataMover)) - }) - } -} - func TestGetUploaderType(t *testing.T) { testcases := []struct { name string diff --git a/pkg/datapath/data_path.go b/pkg/datapath/data_path.go index 68521c04b..6cef1af26 100644 --- a/pkg/datapath/data_path.go +++ b/pkg/datapath/data_path.go @@ -20,12 +20,13 @@ import ( "context" "sync" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/cbtservice" "github.com/vmware-tanzu/velero/pkg/repository" repokey "github.com/vmware-tanzu/velero/pkg/repository/keys" repoProvider "github.com/vmware-tanzu/velero/pkg/repository/provider" @@ -53,6 +54,9 @@ type BackupStartParam struct { ParentSnapshot string ForceFull bool Tags map[string]string + VolumeID string + ChangeID string + SnapshotID string } type generalDataPath struct { @@ -182,8 +186,24 @@ func (dp *generalDataPath) StartBackup(source AccessPoint, uploaderConfig map[st dp.wgDataPath.Done() }() - snapshotID, emptySnapshot, totalBytes, incrementalBytes, err := dp.uploaderProv.RunBackup(dp.ctx, source.ByPath, backupParam.RealSource, backupParam.Tags, backupParam.ForceFull, - backupParam.ParentSnapshot, provider.CBTParam{}, source.VolMode, uploaderConfig, dp) + snapshotID, emptySnapshot, totalBytes, incrementalBytes, err := dp.uploaderProv.RunBackup( + dp.ctx, + source.ByPath, + backupParam.RealSource, + backupParam.Tags, + backupParam.ForceFull, + backupParam.ParentSnapshot, + provider.CBTParam{ + Source: cbtservice.SourceInfo{ + Snapshot: backupParam.SnapshotID, + VolumeID: backupParam.VolumeID, + ChangeID: backupParam.ChangeID, + }, + }, + source.VolMode, + uploaderConfig, + dp, + ) if err == provider.ErrorCanceled { dp.callbacks.OnCancelled(context.Background(), dp.namespace, dp.jobName) diff --git a/pkg/datapath/data_path_test.go b/pkg/datapath/data_path_test.go index c063347c7..65d7f9b65 100644 --- a/pkg/datapath/data_path_test.go +++ b/pkg/datapath/data_path_test.go @@ -20,7 +20,7 @@ import ( "context" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" diff --git a/pkg/datapath/manager.go b/pkg/datapath/manager.go index 79e692b08..4722a81a4 100644 --- a/pkg/datapath/manager.go +++ b/pkg/datapath/manager.go @@ -20,7 +20,7 @@ import ( "context" "sync" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/pkg/datapath/micro_service_watcher.go b/pkg/datapath/micro_service_watcher.go index 665b84b81..3e8ace651 100644 --- a/pkg/datapath/micro_service_watcher.go +++ b/pkg/datapath/micro_service_watcher.go @@ -24,7 +24,7 @@ import ( "sync" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/discovery/helper.go b/pkg/discovery/helper.go index 11c2d623b..74e6f5f8a 100644 --- a/pkg/discovery/helper.go +++ b/pkg/discovery/helper.go @@ -21,7 +21,7 @@ import ( "strings" "sync" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -171,7 +171,7 @@ func (h *helper) Refresh() error { } h.resources = discovery.FilteredBy( - And(filterByVerbs, skipSubresource), + And(h.filterByVerbsWithLogging, h.skipSubresourceWithLogging), serverResources, ) @@ -266,11 +266,38 @@ func filterByVerbs(groupVersion string, r *metav1.APIResource) bool { return discovery.SupportsAllVerbs{Verbs: []string{"list", "create", "get", "delete"}}.Match(groupVersion, r) } +func (h *helper) filterByVerbsWithLogging(groupVersion string, r *metav1.APIResource) bool { + if filterByVerbs(groupVersion, r) { + return true + } + + h.logger.WithFields(logrus.Fields{ + "groupVersion": groupVersion, + "resource": r.Name, + "verbs": r.Verbs, + }).Info("Skipping resource because it does not support required verbs") + + return false +} + func skipSubresource(_ string, r *metav1.APIResource) bool { // if we have a slash, then this is a subresource and we shouldn't include it. return !strings.Contains(r.Name, "/") } +func (h *helper) skipSubresourceWithLogging(groupVersion string, r *metav1.APIResource) bool { + if skipSubresource(groupVersion, r) { + return true + } + + h.logger.WithFields(logrus.Fields{ + "groupVersion": groupVersion, + "resource": r.Name, + }).Info("Skipping subresource") + + return false +} + // sortResources sources resources by moving extensions to the end of the slice. The order of all // the other resources is preserved. func sortResources(resources []*metav1.APIResourceList) { diff --git a/pkg/exposer/csi_snapshot.go b/pkg/exposer/csi_snapshot.go index 33dafff99..ed510c798 100644 --- a/pkg/exposer/csi_snapshot.go +++ b/pkg/exposer/csi_snapshot.go @@ -20,11 +20,12 @@ import ( "context" "fmt" "maps" + "strings" "time" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -34,12 +35,12 @@ import ( "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/vmware-tanzu/velero/pkg/datamover" "github.com/vmware-tanzu/velero/pkg/nodeagent" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/csi" + "github.com/vmware-tanzu/velero/pkg/util/datamover" "github.com/vmware-tanzu/velero/pkg/util/kube" ) @@ -110,6 +111,12 @@ type CSISnapshotExposeWaitParam struct { NodeName string } +type cbtInfo struct { + changeID string + volumeID string + snapshotID string +} + // NewCSISnapshotExposer create a new instance of CSI snapshot exposer func NewCSISnapshotExposer(kubeClient kubernetes.Interface, csiSnapshotClient snapshotter.SnapshotV1Interface, log logrus.FieldLogger) SnapshotExposer { return &csiSnapshotExposer{ @@ -256,6 +263,14 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O affinity := kube.GetLoadAffinityByStorageClass(csiExposeParam.Affinity, backupPVCStorageClass, curLog) + var cbtInfo cbtInfo + if csiExposeParam.DataMover == datamover.DataMoverTypeVeleroBlock { + cbtInfo, err = e.getCBTInfo(ctx, backupVS, backupVSC, csiExposeParam.SourcePVName) + if err != nil { + return errors.Wrap(err, "error to get CBT info") + } + } + backupPod, err := e.createBackupPod( ctx, ownerObject, @@ -273,6 +288,7 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O intoleratableNodes, volumeTopology, csiExposeParam.SnapshotMetadataServiceConfigs, + &cbtInfo, ) if err != nil { return errors.Wrap(err, "error to create backup pod") @@ -289,6 +305,49 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O return nil } +func (e *csiSnapshotExposer) getCBTInfo(ctx context.Context, vs *snapshotv1api.VolumeSnapshot, vsc *snapshotv1api.VolumeSnapshotContent, sourcePVName string) (cbtInfo, error) { + cbtInfo := cbtInfo{} + if vs == nil || vsc == nil { + return cbtInfo, errors.New("vs or vsc is nil") + } + + cbtInfo.snapshotID = vs.Name + + if vs.Annotations != nil && + (vs.Annotations[util.VSphereCNSChangeIDAnno] != "" || + vs.Annotations[util.VSphereCNSSnapshotAnno] != "") { + cbtInfo.changeID = vs.Annotations[util.VSphereCNSChangeIDAnno] + + splitSnapshotAnno := strings.Split(vs.Annotations[util.VSphereCNSSnapshotAnno], "+") + if len(splitSnapshotAnno) >= 2 { + cbtInfo.volumeID = splitSnapshotAnno[0] + } + + e.log.Debugf("volumeID %s and changeID %s are read from VKS annotations.", cbtInfo.volumeID, cbtInfo.changeID) + } else { + pv, err := e.kubeClient.CoreV1().PersistentVolumes().Get(ctx, sourcePVName, metav1.GetOptions{}) + if err != nil { + return cbtInfo, fmt.Errorf("failed to get pv %s: %w", sourcePVName, err) + } + + if vsc.Status != nil && vsc.Status.SnapshotHandle != nil { + cbtInfo.changeID = *vsc.Status.SnapshotHandle + } + + if pv.Spec.CSI != nil && pv.Spec.CSI.VolumeHandle != "" { + cbtInfo.volumeID = pv.Spec.CSI.VolumeHandle + } + + e.log.Debugf("volumeID %s and changeID %s are read from PV and VS's handles.", cbtInfo.volumeID, cbtInfo.changeID) + } + + if cbtInfo.volumeID == "" { + return cbtInfo, fmt.Errorf("volumeID must not be empty for CBT") + } + + return cbtInfo, nil +} + func (e *csiSnapshotExposer) GetExposed(ctx context.Context, ownerObject corev1api.ObjectReference, timeout time.Duration, param any) (*ExposeResult, error) { exposeWaitParam := param.(*CSISnapshotExposeWaitParam) @@ -618,6 +677,7 @@ func (e *csiSnapshotExposer) createBackupPod( intoleratableNodes []string, volumeTopology *corev1api.NodeSelector, csiSnapshotMetadataServiceConfigs *velerotypes.CSISnapshotMetadataService, + cbtInfo *cbtInfo, ) (*corev1api.Pod, error) { podName := ownerObject.Name @@ -670,6 +730,12 @@ func (e *csiSnapshotExposer) createBackupPod( fmt.Sprintf("--resource-timeout=%s", operationTimeout.String()), } + if cbtInfo != nil { + args = append(args, fmt.Sprintf("--change-id=%s", cbtInfo.changeID)) + args = append(args, fmt.Sprintf("--volume-id=%s", cbtInfo.volumeID)) + args = append(args, fmt.Sprintf("--snapshot-id=%s", cbtInfo.snapshotID)) + } + args = append(args, podInfo.logFormatArgs...) args = append(args, podInfo.logLevelArgs...) diff --git a/pkg/exposer/csi_snapshot_priority_test.go b/pkg/exposer/csi_snapshot_priority_test.go index 8c3086f76..f05ab6007 100644 --- a/pkg/exposer/csi_snapshot_priority_test.go +++ b/pkg/exposer/csi_snapshot_priority_test.go @@ -156,6 +156,7 @@ func TestCreateBackupPodWithPriorityClass(t *testing.T) { nil, nil, nil, + nil, ) require.NoError(t, err, tc.description) @@ -243,6 +244,7 @@ func TestCreateBackupPodWithMissingConfigMap(t *testing.T) { nil, nil, nil, + nil, ) // Should succeed even when config map is missing diff --git a/pkg/exposer/csi_snapshot_test.go b/pkg/exposer/csi_snapshot_test.go index 8e00654f9..e5a7aa9a7 100644 --- a/pkg/exposer/csi_snapshot_test.go +++ b/pkg/exposer/csi_snapshot_test.go @@ -17,24 +17,29 @@ limitations under the License. package exposer import ( + "context" "fmt" "maps" + "strings" "testing" "time" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" snapshotFake "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/fake" - "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" + storagev1api "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" + kubefake "k8s.io/client-go/kubernetes/fake" clientTesting "k8s.io/client-go/testing" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -42,9 +47,8 @@ import ( velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util" "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/datamover" "github.com/vmware-tanzu/velero/pkg/util/kube" - - storagev1api "k8s.io/api/storage/v1" ) type reactor struct { @@ -191,6 +195,19 @@ func TestExpose(t *testing.T) { }, } + sourcePV := &corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-pv", + }, + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeSource: corev1api.PersistentVolumeSource{ + CSI: &corev1api.CSIPersistentVolumeSource{ + VolumeHandle: "csi-volume-handle", + }, + }, + }, + } + tests := []struct { name string snapshotClientObj []runtime.Object @@ -1015,6 +1032,46 @@ func TestExpose(t *testing.T) { }, expectedPVCAnnotation: map[string]string{util.VSphereCNSFastCloneAnno: "true"}, }, + { + name: "block data mover success", + ownerBackup: backup, + exposeParam: CSISnapshotExposeParam{ + SnapshotName: "fake-vs", + SourceNamespace: "fake-ns", + AccessMode: AccessModeFileSystem, + OperationTimeout: time.Millisecond, + ExposeTimeout: time.Millisecond, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", + DataMover: datamover.DataMoverTypeVeleroBlock, + }, + snapshotClientObj: []runtime.Object{ + vsObject, + vscObj, + }, + kubeClientObj: []runtime.Object{ + daemonSet, + scObj, + sourcePV, + }, + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, + }, + }, } for _, test := range tests { @@ -1413,7 +1470,7 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { Kind: backup.Kind, Name: backup.Name, UID: backup.UID, - Controller: pointer.BoolPtr(true), + Controller: ptr.To(true), }, }, }, @@ -1424,7 +1481,7 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { VolumeMode: &volumeMode, DataSource: dataSource, DataSourceRef: nil, - StorageClassName: pointer.String("fake-storage-class"), + StorageClassName: ptr.To("fake-storage-class"), Resources: corev1api.VolumeResourceRequirements{ Requests: corev1api.ResourceList{ corev1api.ResourceStorage: resource.MustParse("1Gi"), @@ -1444,7 +1501,7 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { Kind: backup.Kind, Name: backup.Name, UID: backup.UID, - Controller: pointer.BoolPtr(true), + Controller: ptr.To(true), }, }, }, @@ -1455,7 +1512,7 @@ func Test_csiSnapshotExposer_createBackupPVC(t *testing.T) { VolumeMode: &volumeMode, DataSource: dataSource, DataSourceRef: nil, - StorageClassName: pointer.String("fake-storage-class"), + StorageClassName: ptr.To("fake-storage-class"), Resources: corev1api.VolumeResourceRequirements{ Requests: corev1api.ResourceList{ corev1api.ResourceStorage: resource.MustParse("1Gi"), @@ -1994,3 +2051,150 @@ end diagnose CSI exposer`, }) } } + +func TestGetCBTInfo(t *testing.T) { + handle := "snapshot-handle-1" + + tests := []struct { + name string + vs *snapshotv1api.VolumeSnapshot + vsc *snapshotv1api.VolumeSnapshotContent + pv *corev1api.PersistentVolume + sourcePVName string + want cbtInfo + wantErrSubstr string + }{ + { + name: "return error when vs is nil", + vs: nil, + vsc: &snapshotv1api.VolumeSnapshotContent{}, + sourcePVName: "pv-1", + wantErrSubstr: "vs or vsc is nil", + }, + { + name: "use annotations when change-id and snapshot annotation exist", + vs: &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-anno", + Annotations: map[string]string{ + util.VSphereCNSChangeIDAnno: "change-id-1", + util.VSphereCNSSnapshotAnno: "volume-id-1+snapshot-id-1", + }, + }, + }, + vsc: &snapshotv1api.VolumeSnapshotContent{}, + sourcePVName: "pv-ignored", + want: cbtInfo{ + changeID: "change-id-1", + volumeID: "volume-id-1", + snapshotID: "vs-anno", + }, + }, + { + name: "fallback to pv and vsc snapshot handle", + vs: &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{Name: "vs-fallback"}, + }, + vsc: &snapshotv1api.VolumeSnapshotContent{ + Status: &snapshotv1api.VolumeSnapshotContentStatus{ + SnapshotHandle: &handle, + }, + }, + pv: &corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "pv-1"}, + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeSource: corev1api.PersistentVolumeSource{ + CSI: &corev1api.CSIPersistentVolumeSource{ + VolumeHandle: "csi-volume-handle-1", + }, + }, + }, + }, + sourcePVName: "pv-1", + want: cbtInfo{ + changeID: "snapshot-handle-1", + volumeID: "csi-volume-handle-1", + snapshotID: "vs-fallback", + }, + }, + { + name: "return error when pv not found in fallback path", + vs: &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{Name: "vs-no-pv"}, + }, + vsc: &snapshotv1api.VolumeSnapshotContent{}, + sourcePVName: "pv-not-found", + wantErrSubstr: "failed to get pv pv-not-found", + }, + { + name: "return error when pv has no csi volume handle", + vs: &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{Name: "vs-no-volume-handle"}, + }, + vsc: &snapshotv1api.VolumeSnapshotContent{}, + pv: &corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "pv-no-handle"}, + Spec: corev1api.PersistentVolumeSpec{}, + }, + sourcePVName: "pv-no-handle", + wantErrSubstr: "volumeID must not be empty for CBT", + }, + { + name: "return error when snapshot annotation is invalid", + vs: &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-no-volume-handle", + Annotations: map[string]string{ + util.VSphereCNSChangeIDAnno: "change-id-1", + util.VSphereCNSSnapshotAnno: "volume-id-1:snapshot-id-1", + }, + }, + }, + vsc: &snapshotv1api.VolumeSnapshotContent{}, + pv: &corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "pv-1"}, + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeSource: corev1api.PersistentVolumeSource{ + CSI: &corev1api.CSIPersistentVolumeSource{ + VolumeHandle: "csi-volume-handle-1", + }, + }, + }, + }, + sourcePVName: "pv-1", + wantErrSubstr: "volumeID must not be empty for CBT", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var objs []runtime.Object + if tc.pv != nil { + objs = append(objs, tc.pv) + } + exposer := &csiSnapshotExposer{ + kubeClient: kubefake.NewSimpleClientset(objs...), + log: logrus.StandardLogger(), + } + + got, err := exposer.getCBTInfo(context.Background(), tc.vs, tc.vsc, tc.sourcePVName) + + if tc.wantErrSubstr != "" { + if err == nil { + t.Fatalf("expected error containing %q, got nil", tc.wantErrSubstr) + } + if !strings.Contains(err.Error(), tc.wantErrSubstr) { + t.Fatalf("expected error containing %q, got %q", tc.wantErrSubstr, err.Error()) + } + return + } + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.changeID != tc.want.changeID || got.volumeID != tc.want.volumeID || got.snapshotID != tc.want.snapshotID { + t.Fatalf("unexpected cbtInfo, want %+v, got %+v", tc.want, got) + } + }) + } +} diff --git a/pkg/exposer/generic_restore.go b/pkg/exposer/generic_restore.go index b711e7364..0f4b9c5b4 100644 --- a/pkg/exposer/generic_restore.go +++ b/pkg/exposer/generic_restore.go @@ -21,7 +21,8 @@ import ( "fmt" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" + "github.com/google/uuid" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -33,6 +34,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/nodeagent" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/datamover" "github.com/vmware-tanzu/velero/pkg/util/kube" ) @@ -79,6 +81,24 @@ type GenericRestoreExposeParam struct { // CacheVolume specifies the info for cache volumes CacheVolume *CacheConfigs + + // DataMover is the data mover type, e.g., velero-fs, velero-block + DataMover string +} + +// GenericRestoreRebindVolumeParam define the input param for Generic Restore Rebind Volume +type GenericRestoreRebindVolumeParam struct { + // TargetPVCName is the target volume name to be restored + TargetPVCName string + + // TargetNamespace is the namespace of the volume to be restored + TargetNamespace string + + // OperationTimeout specifies the time wait for resources operations in Expose + OperationTimeout time.Duration + + // TargetFSType is the file system type of the target volume + TargetFSType string } // GenericRestoreExposer is the interfaces for a generic restore exposer @@ -101,7 +121,7 @@ type GenericRestoreExposer interface { DiagnoseExpose(context.Context, corev1api.ObjectReference) string // RebindVolume unexposes the restored PV and rebind it to the target PVC - RebindVolume(context.Context, corev1api.ObjectReference, string, string, time.Duration) error + RebindVolume(context.Context, corev1api.ObjectReference, GenericRestoreRebindVolumeParam) error // CleanUp cleans up any objects generated during the restore expose CleanUp(context.Context, corev1api.ObjectReference) @@ -176,10 +196,23 @@ func (e *genericRestoreExposer) Expose(ctx context.Context, ownerObject corev1ap } } + restorePVC, err := e.createRestorePVC(ctx, ownerObject, targetPVC, selectedNode, param.DataMover) + if err != nil { + return errors.Wrap(err, "error to create restore pvc") + } + + curLog.WithField("pvc name", restorePVC.Name).Info("Restore PVC is created") + + defer func() { + if err != nil { + kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), restorePVC.Name, restorePVC.Namespace, 0, curLog) + } + }() + restorePod, err := e.createRestorePod( ctx, ownerObject, - targetPVC, + restorePVC, param.OperationTimeout, param.HostingPodLabels, param.HostingPodAnnotations, @@ -203,19 +236,6 @@ func (e *genericRestoreExposer) Expose(ctx context.Context, ownerObject corev1ap } }() - restorePVC, err := e.createRestorePVC(ctx, ownerObject, targetPVC, selectedNode) - if err != nil { - return errors.Wrap(err, "error to create restore pvc") - } - - curLog.WithField("pvc name", restorePVC.Name).Info("Restore PVC is created") - - defer func() { - if err != nil { - kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), restorePVC.Name, restorePVC.Namespace, 0, curLog) - } - }() - return nil } @@ -379,26 +399,123 @@ func (e *genericRestoreExposer) CleanUp(ctx context.Context, ownerObject corev1a kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), cachePVCName, ownerObject.Namespace, 0, e.log) } -func (e *genericRestoreExposer) RebindVolume(ctx context.Context, ownerObject corev1api.ObjectReference, targetPVCName string, targetNamespace string, timeout time.Duration) error { - restorePodName := ownerObject.Name +func (e *genericRestoreExposer) RebindVolume(ctx context.Context, ownerObject corev1api.ObjectReference, param GenericRestoreRebindVolumeParam) error { restorePVCName := ownerObject.Name curLog := e.log.WithFields(logrus.Fields{ "owner": ownerObject.Name, - "target PVC": targetPVCName, - "target namespace": targetNamespace, + "target PVC": param.TargetPVCName, + "target namespace": param.TargetNamespace, }) - targetPVC, err := e.kubeClient.CoreV1().PersistentVolumeClaims(targetNamespace).Get(ctx, targetPVCName, metav1.GetOptions{}) + targetPVC, err := e.kubeClient.CoreV1().PersistentVolumeClaims(param.TargetNamespace).Get(ctx, param.TargetPVCName, metav1.GetOptions{}) if err != nil { - return errors.Wrapf(err, "error to get target PVC %s/%s", targetNamespace, targetPVCName) + return errors.Wrapf(err, "error to get target PVC %s/%s", param.TargetNamespace, param.TargetPVCName) } - restorePV, err := kube.WaitPVCBound(ctx, e.kubeClient.CoreV1(), e.kubeClient.CoreV1(), restorePVCName, ownerObject.Namespace, timeout) + restorePV, err := kube.WaitPVCBound(ctx, e.kubeClient.CoreV1(), e.kubeClient.CoreV1(), restorePVCName, ownerObject.Namespace, param.OperationTimeout) if err != nil { return errors.Wrapf(err, "error to get PV from restore PVC %s", restorePVCName) } + if kube.GetVolumeModeByPVC(targetPVC) != kube.GetVolumeModeByPV(restorePV) { + return e.rebindVolumeChangeMode(ctx, ownerObject, param, targetPVC, restorePV, curLog) + } else { + return e.rebindVolumeSameMode(ctx, ownerObject, param, targetPVC, restorePV, curLog) + } +} + +func (e *genericRestoreExposer) rebindVolumeChangeMode(ctx context.Context, ownerObject corev1api.ObjectReference, param GenericRestoreRebindVolumeParam, targetPVC *corev1api.PersistentVolumeClaim, restorePV *corev1api.PersistentVolume, curLog logrus.FieldLogger) error { + restorePodName := ownerObject.Name + restorePVCName := ownerObject.Name + + orgReclaim := restorePV.Spec.PersistentVolumeReclaimPolicy + + curLog.WithField("restore PV", restorePV.Name).Info("Restore PV is retrieved") + + retained, err := kube.SetPVReclaimPolicy(ctx, e.kubeClient.CoreV1(), restorePV, corev1api.PersistentVolumeReclaimRetain) + if err != nil { + return errors.Wrapf(err, "error to retain PV %s", restorePV.Name) + } + + curLog.WithField("restore PV", restorePV.Name).WithField("retained", (retained != nil)).Info("Restore PV is retained") + + var rebindPV *corev1api.PersistentVolume + + defer func() { + if retained != nil { + curLog.WithField("retained PV", retained.Name).Info("Deleting retained PV on error") + kube.DeletePVIfAny(ctx, e.kubeClient.CoreV1(), retained.Name, curLog) + } + + if rebindPV != nil { + curLog.WithField("rebind PV", rebindPV.Name).Info("Deleting rebind PV on error") + kube.DeletePVIfAny(ctx, e.kubeClient.CoreV1(), rebindPV.Name, curLog) + } + }() + + if retained != nil { + restorePV = retained + } + + err = kube.EnsureDeletePod(ctx, e.kubeClient.CoreV1(), restorePodName, ownerObject.Namespace, param.OperationTimeout) + if err != nil { + return errors.Wrapf(err, "error to delete restore pod %s", restorePodName) + } + + err = kube.WaitVolumeDetached(ctx, e.kubeClient.StorageV1(), restorePV.Name, param.OperationTimeout) + if err != nil { + return errors.Wrapf(err, "error waiting for restore PV %s to detach", restorePV.Name) + } + + curLog.WithField("restore PV", restorePV.Name).Info("Restore PV is detached") + + err = kube.EnsureDeletePVC(ctx, e.kubeClient.CoreV1(), restorePVCName, ownerObject.Namespace, param.OperationTimeout) + if err != nil { + return errors.Wrapf(err, "error to delete restore PVC %s", restorePVCName) + } + + curLog.WithField("restore PVC", restorePVCName).Info("Restore PVC is deleted") + + err = kube.EnsureDeletePV(ctx, e.kubeClient.CoreV1(), restorePV.Name, param.OperationTimeout) + if err != nil { + return errors.Wrapf(err, "error deleting restore PV %s", restorePV.Name) + } + + curLog.WithField("restore PV", restorePV.Name).Info("Restore PV is deleted") + + retained = nil + + rebindPV, err = kube.RebindPV(ctx, e.kubeClient.CoreV1(), uuid.NewString(), restorePV, targetPVC, orgReclaim, param.TargetFSType) + if err != nil { + return errors.Wrapf(err, "error rebinding PV for target PVC %s", param.TargetPVCName) + } + + curLog.WithField("rebind PV", rebindPV.Name).Info("Rebind PV is created") + + _, err = kube.RebindPVC(ctx, e.kubeClient.CoreV1(), targetPVC, rebindPV.Name) + if err != nil { + return errors.Wrapf(err, "error to rebind target PVC %s/%s to %s", targetPVC.Namespace, targetPVC.Name, rebindPV.Name) + } + + curLog.WithField("rebind PV", rebindPV.Name).Info("Target PVC is rebound to rebind PV") + + _, err = kube.WaitPVBound(ctx, e.kubeClient.CoreV1(), rebindPV.Name, targetPVC.Name, targetPVC.Namespace, param.OperationTimeout) + if err != nil { + return errors.Wrapf(err, "error to wait rebind PV ready, rebind PV %s", rebindPV.Name) + } + + curLog.WithField("rebind PV", rebindPV.Name).Info("Rebind PV is ready") + + rebindPV = nil + + return nil +} + +func (e *genericRestoreExposer) rebindVolumeSameMode(ctx context.Context, ownerObject corev1api.ObjectReference, param GenericRestoreRebindVolumeParam, targetPVC *corev1api.PersistentVolumeClaim, restorePV *corev1api.PersistentVolume, curLog logrus.FieldLogger) error { + restorePodName := ownerObject.Name + restorePVCName := ownerObject.Name + orgReclaim := restorePV.Spec.PersistentVolumeReclaimPolicy curLog.WithField("restore PV", restorePV.Name).Info("Restore PV is retrieved") @@ -421,12 +538,19 @@ func (e *genericRestoreExposer) RebindVolume(ctx context.Context, ownerObject co restorePV = retained } - err = kube.EnsureDeletePod(ctx, e.kubeClient.CoreV1(), restorePodName, ownerObject.Namespace, timeout) + err = kube.EnsureDeletePod(ctx, e.kubeClient.CoreV1(), restorePodName, ownerObject.Namespace, param.OperationTimeout) if err != nil { return errors.Wrapf(err, "error to delete restore pod %s", restorePodName) } - err = kube.EnsureDeletePVC(ctx, e.kubeClient.CoreV1(), restorePVCName, ownerObject.Namespace, timeout) + err = kube.WaitVolumeDetached(ctx, e.kubeClient.StorageV1(), restorePV.Name, param.OperationTimeout) + if err != nil { + return errors.Wrapf(err, "error waiting for restore PV %s to detach", restorePV.Name) + } + + curLog.WithField("restore PV", restorePV.Name).Info("Restore PV is detached") + + err = kube.EnsureDeletePVC(ctx, e.kubeClient.CoreV1(), restorePVCName, ownerObject.Namespace, param.OperationTimeout) if err != nil { return errors.Wrapf(err, "error to delete restore PVC %s", restorePVCName) } @@ -453,7 +577,7 @@ func (e *genericRestoreExposer) RebindVolume(ctx context.Context, ownerObject co curLog.WithField("restore PV", restorePV.Name).Info("Restore PV is rebound") - restorePV, err = kube.WaitPVBound(ctx, e.kubeClient.CoreV1(), restorePV.Name, targetPVC.Name, targetPVC.Namespace, timeout) + restorePV, err = kube.WaitPVBound(ctx, e.kubeClient.CoreV1(), restorePV.Name, targetPVC.Name, targetPVC.Namespace, param.OperationTimeout) if err != nil { return errors.Wrapf(err, "error to wait restore PV bound, restore PV %s", restorePVName) } @@ -682,7 +806,7 @@ func (e *genericRestoreExposer) createRestorePod( return e.kubeClient.CoreV1().Pods(ownerObject.Namespace).Create(ctx, pod, metav1.CreateOptions{}) } -func (e *genericRestoreExposer) createRestorePVC(ctx context.Context, ownerObject corev1api.ObjectReference, targetPVC *corev1api.PersistentVolumeClaim, selectedNode string) (*corev1api.PersistentVolumeClaim, error) { +func (e *genericRestoreExposer) createRestorePVC(ctx context.Context, ownerObject corev1api.ObjectReference, targetPVC *corev1api.PersistentVolumeClaim, selectedNode string, dataMover string) (*corev1api.PersistentVolumeClaim, error) { restorePVCName := ownerObject.Name pvcObj := &corev1api.PersistentVolumeClaim{ @@ -715,5 +839,13 @@ func (e *genericRestoreExposer) createRestorePVC(ctx context.Context, ownerObjec } } + if dataMover == datamover.DataMoverTypeVeleroBlock { + if pvcObj.Spec.VolumeMode == nil { + pvcObj.Spec.VolumeMode = new(corev1api.PersistentVolumeMode) + } + + *pvcObj.Spec.VolumeMode = corev1api.PersistentVolumeBlock + } + return e.kubeClient.CoreV1().PersistentVolumeClaims(pvcObj.Namespace).Create(ctx, pvcObj, metav1.CreateOptions{}) } diff --git a/pkg/exposer/generic_restore_test.go b/pkg/exposer/generic_restore_test.go index 799719a50..b65863318 100644 --- a/pkg/exposer/generic_restore_test.go +++ b/pkg/exposer/generic_restore_test.go @@ -20,7 +20,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" @@ -34,6 +34,7 @@ import ( velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" + "github.com/vmware-tanzu/velero/pkg/util/datamover" "github.com/vmware-tanzu/velero/pkg/util/kube" ) @@ -61,6 +62,18 @@ func TestRestoreExpose(t *testing.T) { }, } + modeFilesystem := corev1api.PersistentVolumeFilesystem + targetPVCObjWithVolumeMode := &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "fake-ns", + Name: "fake-target-pvc", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + StorageClassName: &scName, + VolumeMode: &modeFilesystem, + }, + } + storageClass := &storagev1api.StorageClass{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-sc", @@ -107,6 +120,7 @@ func TestRestoreExpose(t *testing.T) { targetNamespace string kubeReactors []reactor cacheVolume *CacheConfigs + dataMover string expectBackupPod bool expectBackupPVC bool expectCachePVC bool @@ -236,6 +250,34 @@ func TestRestoreExpose(t *testing.T) { expectBackupPVC: true, expectCachePVC: true, }, + { + name: "succeed with velero-block data mover", + targetPVCName: "fake-target-pvc", + targetNamespace: "fake-ns", + ownerRestore: restore, + kubeClientObj: []runtime.Object{ + targetPVCObj, + daemonSet, + storageClass, + }, + dataMover: datamover.DataMoverTypeVeleroBlock, + expectBackupPod: true, + expectBackupPVC: true, + }, + { + name: "succeed with velero-block data mover and existing volume mode", + targetPVCName: "fake-target-pvc", + targetNamespace: "fake-ns", + ownerRestore: restore, + kubeClientObj: []runtime.Object{ + targetPVCObjWithVolumeMode, + daemonSet, + storageClass, + }, + dataMover: datamover.DataMoverTypeVeleroBlock, + expectBackupPod: true, + expectBackupPVC: true, + }, } for _, test := range tests { @@ -273,6 +315,7 @@ func TestRestoreExpose(t *testing.T) { ExposeTimeout: time.Millisecond, LoadAffinity: nil, CacheVolume: test.cacheVolume, + DataMover: test.dataMover, }, ) @@ -289,9 +332,13 @@ func TestRestoreExpose(t *testing.T) { require.True(t, apierrors.IsNotFound(err)) } - _, err = exposer.kubeClient.CoreV1().PersistentVolumeClaims(ownerObject.Namespace).Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) + pvc, err := exposer.kubeClient.CoreV1().PersistentVolumeClaims(ownerObject.Namespace).Get(t.Context(), ownerObject.Name, metav1.GetOptions{}) if test.expectBackupPVC { require.NoError(t, err) + if test.dataMover == datamover.DataMoverTypeVeleroBlock { + require.NotNil(t, pvc.Spec.VolumeMode) + require.Equal(t, corev1api.PersistentVolumeBlock, *pvc.Spec.VolumeMode) + } } else { require.True(t, apierrors.IsNotFound(err)) } @@ -319,11 +366,27 @@ func TestRebindVolume(t *testing.T) { }, } - targetPVCObj := &corev1api.PersistentVolumeClaim{ + modeFilesystem := corev1api.PersistentVolumeFilesystem + modeBlock := corev1api.PersistentVolumeBlock + + targetPVCObjChangeMode := &corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: "fake-ns", Name: "fake-target-pvc", }, + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeMode: &modeBlock, + }, + } + + targetPVCObjSameMode := &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "fake-ns", + Name: "fake-target-pvc", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeMode: &modeFilesystem, + }, } restorePVCObj := &corev1api.PersistentVolumeClaim{ @@ -342,6 +405,7 @@ func TestRebindVolume(t *testing.T) { }, Spec: corev1api.PersistentVolumeSpec{ PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, + VolumeMode: &modeFilesystem, }, } @@ -352,8 +416,6 @@ func TestRebindVolume(t *testing.T) { }, } - hookCount := 0 - tests := []struct { name string kubeClientObj []runtime.Object @@ -376,17 +438,17 @@ func TestRebindVolume(t *testing.T) { targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ - targetPVCObj, + targetPVCObjSameMode, }, err: "error to get PV from restore PVC fake-restore: error to wait for rediness of PVC: error to get pvc velero/fake-restore: persistentvolumeclaims \"fake-restore\" not found", }, { - name: "retain target pv fail", + name: "[change mode] retain target pv fail", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ - targetPVCObj, + targetPVCObjChangeMode, restorePVCObj, restorePVObj, }, @@ -402,12 +464,12 @@ func TestRebindVolume(t *testing.T) { err: "error to retain PV fake-restore-pv: error patching PV: fake-patch-error", }, { - name: "delete restore pod fail", + name: "[change mode] delete restore pod fail", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ - targetPVCObj, + targetPVCObjChangeMode, restorePVCObj, restorePVObj, restorePod, @@ -424,12 +486,12 @@ func TestRebindVolume(t *testing.T) { err: "error to delete restore pod fake-restore: error to delete pod fake-restore: fake-delete-error", }, { - name: "delete restore pvc fail", + name: "[change mode] delete restore pvc fail", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ - targetPVCObj, + targetPVCObjChangeMode, restorePVCObj, restorePVObj, restorePod, @@ -446,12 +508,204 @@ func TestRebindVolume(t *testing.T) { err: "error to delete restore PVC fake-restore: error to delete pvc fake-restore: fake-delete-error", }, { - name: "rebind target pvc fail", + name: "[change mode] wait volume detached fail", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ - targetPVCObj, + targetPVCObjChangeMode, + restorePVCObj, + restorePVObj, + restorePod, + }, + kubeReactors: []reactor{ + { + verb: "list", + resource: "volumeattachments", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, errors.New("fake-list-error") + }, + }, + }, + err: "error waiting for restore PV fake-restore-pv to detach: error listing volumeattachment: error listing volumeattachment: fake-list-error", + }, + { + name: "[change mode] rebind pv fail", + targetPVCName: "fake-target-pvc", + targetNamespace: "fake-ns", + ownerRestore: restore, + kubeClientObj: []runtime.Object{ + targetPVCObjChangeMode, + restorePVCObj, + restorePVObj, + restorePod, + }, + kubeReactors: []reactor{ + { + verb: "create", + resource: "persistentvolumes", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, errors.New("fake-create-error") + }, + }, + }, + err: "error rebinding PV for target PVC fake-target-pvc: fake-create-error", + }, + { + name: "[change mode] delete retained pv fail", + targetPVCName: "fake-target-pvc", + targetNamespace: "fake-ns", + ownerRestore: restore, + kubeClientObj: []runtime.Object{ + targetPVCObjChangeMode, + restorePVCObj, + restorePVObj, + restorePod, + }, + kubeReactors: []reactor{ + { + verb: "delete", + resource: "persistentvolumes", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + // we want it to fail on the PV deletion but not the pod/pvc deletions + if action.(clientTesting.DeleteAction).GetName() == "fake-restore-pv" { + return true, nil, errors.New("fake-delete-error") + } + return false, nil, nil + }, + }, + }, + err: "error deleting restore PV fake-restore-pv: error to delete pv fake-restore-pv: fake-delete-error", + }, + { + name: "[change mode] rebind target pvc fail", + targetPVCName: "fake-target-pvc", + targetNamespace: "fake-ns", + ownerRestore: restore, + kubeClientObj: []runtime.Object{ + targetPVCObjChangeMode, + restorePVCObj, + restorePVObj, + restorePod, + }, + kubeReactors: []reactor{ + { + verb: "patch", + resource: "persistentvolumeclaims", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, errors.New("fake-patch-error") + }, + }, + }, + err: "error to rebind target PVC fake-ns/fake-target-pvc to", + }, + { + name: "[change mode] wait rebind PV ready fail", + targetPVCName: "fake-target-pvc", + targetNamespace: "fake-ns", + ownerRestore: restore, + kubeClientObj: []runtime.Object{ + targetPVCObjChangeMode, + restorePVCObj, + restorePVObj, + restorePod, + }, + err: "error to wait rebind PV ready, rebind PV", + }, + { + name: "[same mode] retain target pv fail", + targetPVCName: "fake-target-pvc", + targetNamespace: "fake-ns", + ownerRestore: restore, + kubeClientObj: []runtime.Object{ + targetPVCObjSameMode, + restorePVCObj, + restorePVObj, + }, + kubeReactors: []reactor{ + { + verb: "patch", + resource: "persistentvolumes", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, errors.New("fake-patch-error") + }, + }, + }, + err: "error to retain PV fake-restore-pv: error patching PV: fake-patch-error", + }, + { + name: "[same mode] delete restore pod fail", + targetPVCName: "fake-target-pvc", + targetNamespace: "fake-ns", + ownerRestore: restore, + kubeClientObj: []runtime.Object{ + targetPVCObjSameMode, + restorePVCObj, + restorePVObj, + restorePod, + }, + kubeReactors: []reactor{ + { + verb: "delete", + resource: "pods", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, errors.New("fake-delete-error") + }, + }, + }, + err: "error to delete restore pod fake-restore: error to delete pod fake-restore: fake-delete-error", + }, + { + name: "[same mode] delete restore pvc fail", + targetPVCName: "fake-target-pvc", + targetNamespace: "fake-ns", + ownerRestore: restore, + kubeClientObj: []runtime.Object{ + targetPVCObjSameMode, + restorePVCObj, + restorePVObj, + restorePod, + }, + kubeReactors: []reactor{ + { + verb: "delete", + resource: "persistentvolumeclaims", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, errors.New("fake-delete-error") + }, + }, + }, + err: "error to delete restore PVC fake-restore: error to delete pvc fake-restore: fake-delete-error", + }, + { + name: "[same mode] wait volume detached fail", + targetPVCName: "fake-target-pvc", + targetNamespace: "fake-ns", + ownerRestore: restore, + kubeClientObj: []runtime.Object{ + targetPVCObjSameMode, + restorePVCObj, + restorePVObj, + restorePod, + }, + kubeReactors: []reactor{ + { + verb: "list", + resource: "volumeattachments", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, errors.New("fake-list-error") + }, + }, + }, + err: "error waiting for restore PV fake-restore-pv to detach: error listing volumeattachment: error listing volumeattachment: fake-list-error", + }, + { + name: "[same mode] rebind target pvc fail", + targetPVCName: "fake-target-pvc", + targetNamespace: "fake-ns", + ownerRestore: restore, + kubeClientObj: []runtime.Object{ + targetPVCObjSameMode, restorePVCObj, restorePVObj, restorePod, @@ -468,12 +722,12 @@ func TestRebindVolume(t *testing.T) { err: "error to rebind target PVC fake-ns/fake-target-pvc to fake-restore-pv: error patching PVC: fake-patch-error", }, { - name: "reset pv binding fail", + name: "[same mode] reset pv binding fail", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ - targetPVCObj, + targetPVCObjSameMode, restorePVCObj, restorePVObj, restorePod, @@ -483,24 +737,25 @@ func TestRebindVolume(t *testing.T) { verb: "patch", resource: "persistentvolumes", reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { - if hookCount == 0 { - hookCount++ - return false, nil, nil - } else { - return true, nil, errors.New("fake-patch-error") + // we need it to succeed on set reclaim policy, but fail on reset binding + patchAction := action.(clientTesting.PatchAction) + patchString := string(patchAction.GetPatch()) + if patchString != `{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}` { + return true, nil, errors.New("fake-patch-error-reset") } + return false, nil, nil }, }, }, - err: "error to reset binding info for restore PV fake-restore-pv: error patching PV: fake-patch-error", + err: "error to reset binding info for restore PV fake-restore-pv: error patching PV: fake-patch-error-reset", }, { - name: "wait restore PV bound fail", + name: "[same mode] wait restore PV bound fail", targetPVCName: "fake-target-pvc", targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ - targetPVCObj, + targetPVCObjSameMode, restorePVCObj, restorePVObj, restorePod, @@ -533,14 +788,19 @@ func TestRebindVolume(t *testing.T) { } } - hookCount = 0 - - err := exposer.RebindVolume(t.Context(), ownerObject, test.targetPVCName, test.targetNamespace, time.Millisecond) - assert.EqualError(t, err, test.err) + err := exposer.RebindVolume(t.Context(), ownerObject, GenericRestoreRebindVolumeParam{ + TargetPVCName: test.targetPVCName, + TargetNamespace: test.targetNamespace, + OperationTimeout: time.Millisecond, + }) + if test.err != "" { + assert.ErrorContains(t, err, test.err) + } else { + assert.NoError(t, err) + } }) } } - func TestRestorePeekExpose(t *testing.T) { restore := &velerov1.Restore{ TypeMeta: metav1.TypeMeta{ diff --git a/pkg/exposer/host_path.go b/pkg/exposer/host_path.go index e51178711..db1dff908 100644 --- a/pkg/exposer/host_path.go +++ b/pkg/exposer/host_path.go @@ -21,7 +21,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" diff --git a/pkg/exposer/host_path_test.go b/pkg/exposer/host_path_test.go index e751afe0d..4c34aed02 100644 --- a/pkg/exposer/host_path_test.go +++ b/pkg/exposer/host_path_test.go @@ -21,7 +21,7 @@ import ( "fmt" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/pkg/exposer/image.go b/pkg/exposer/image.go index 58658e1b7..2157d8175 100644 --- a/pkg/exposer/image.go +++ b/pkg/exposer/image.go @@ -20,7 +20,7 @@ import ( "context" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" diff --git a/pkg/exposer/mocks/GenericRestoreExposer.go b/pkg/exposer/mocks/GenericRestoreExposer.go index 7daae6d6d..a1d8943d4 100644 --- a/pkg/exposer/mocks/GenericRestoreExposer.go +++ b/pkg/exposer/mocks/GenericRestoreExposer.go @@ -14,13 +14,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// NewMockGenericRestoreExposer creates a new instance of MockGenericRestoreExposer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// NewGenericRestoreExposer creates a new instance of GenericRestoreExposer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. -func NewMockGenericRestoreExposer(t interface { +func NewGenericRestoreExposer(t interface { mock.TestingT Cleanup(func()) -}) *MockGenericRestoreExposer { - mock := &MockGenericRestoreExposer{} +}) *GenericRestoreExposer { + mock := &GenericRestoreExposer{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) @@ -28,38 +28,38 @@ func NewMockGenericRestoreExposer(t interface { return mock } -// MockGenericRestoreExposer is an autogenerated mock type for the GenericRestoreExposer type -type MockGenericRestoreExposer struct { +// GenericRestoreExposer is an autogenerated mock type for the GenericRestoreExposer type +type GenericRestoreExposer struct { mock.Mock } -type MockGenericRestoreExposer_Expecter struct { +type GenericRestoreExposer_Expecter struct { mock *mock.Mock } -func (_m *MockGenericRestoreExposer) EXPECT() *MockGenericRestoreExposer_Expecter { - return &MockGenericRestoreExposer_Expecter{mock: &_m.Mock} +func (_m *GenericRestoreExposer) EXPECT() *GenericRestoreExposer_Expecter { + return &GenericRestoreExposer_Expecter{mock: &_m.Mock} } -// CleanUp provides a mock function for the type MockGenericRestoreExposer -func (_mock *MockGenericRestoreExposer) CleanUp(context1 context.Context, objectReference v1.ObjectReference) { +// CleanUp provides a mock function for the type GenericRestoreExposer +func (_mock *GenericRestoreExposer) CleanUp(context1 context.Context, objectReference v1.ObjectReference) { _mock.Called(context1, objectReference) return } -// MockGenericRestoreExposer_CleanUp_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CleanUp' -type MockGenericRestoreExposer_CleanUp_Call struct { +// GenericRestoreExposer_CleanUp_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CleanUp' +type GenericRestoreExposer_CleanUp_Call struct { *mock.Call } // CleanUp is a helper method to define mock.On call // - context1 context.Context // - objectReference v1.ObjectReference -func (_e *MockGenericRestoreExposer_Expecter) CleanUp(context1 interface{}, objectReference interface{}) *MockGenericRestoreExposer_CleanUp_Call { - return &MockGenericRestoreExposer_CleanUp_Call{Call: _e.mock.On("CleanUp", context1, objectReference)} +func (_e *GenericRestoreExposer_Expecter) CleanUp(context1 interface{}, objectReference interface{}) *GenericRestoreExposer_CleanUp_Call { + return &GenericRestoreExposer_CleanUp_Call{Call: _e.mock.On("CleanUp", context1, objectReference)} } -func (_c *MockGenericRestoreExposer_CleanUp_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference)) *MockGenericRestoreExposer_CleanUp_Call { +func (_c *GenericRestoreExposer_CleanUp_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference)) *GenericRestoreExposer_CleanUp_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -77,18 +77,18 @@ func (_c *MockGenericRestoreExposer_CleanUp_Call) Run(run func(context1 context. return _c } -func (_c *MockGenericRestoreExposer_CleanUp_Call) Return() *MockGenericRestoreExposer_CleanUp_Call { +func (_c *GenericRestoreExposer_CleanUp_Call) Return() *GenericRestoreExposer_CleanUp_Call { _c.Call.Return() return _c } -func (_c *MockGenericRestoreExposer_CleanUp_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference)) *MockGenericRestoreExposer_CleanUp_Call { +func (_c *GenericRestoreExposer_CleanUp_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference)) *GenericRestoreExposer_CleanUp_Call { _c.Run(run) return _c } -// DiagnoseExpose provides a mock function for the type MockGenericRestoreExposer -func (_mock *MockGenericRestoreExposer) DiagnoseExpose(context1 context.Context, objectReference v1.ObjectReference) string { +// DiagnoseExpose provides a mock function for the type GenericRestoreExposer +func (_mock *GenericRestoreExposer) DiagnoseExpose(context1 context.Context, objectReference v1.ObjectReference) string { ret := _mock.Called(context1, objectReference) if len(ret) == 0 { @@ -104,19 +104,19 @@ func (_mock *MockGenericRestoreExposer) DiagnoseExpose(context1 context.Context, return r0 } -// MockGenericRestoreExposer_DiagnoseExpose_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DiagnoseExpose' -type MockGenericRestoreExposer_DiagnoseExpose_Call struct { +// GenericRestoreExposer_DiagnoseExpose_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DiagnoseExpose' +type GenericRestoreExposer_DiagnoseExpose_Call struct { *mock.Call } // DiagnoseExpose is a helper method to define mock.On call // - context1 context.Context // - objectReference v1.ObjectReference -func (_e *MockGenericRestoreExposer_Expecter) DiagnoseExpose(context1 interface{}, objectReference interface{}) *MockGenericRestoreExposer_DiagnoseExpose_Call { - return &MockGenericRestoreExposer_DiagnoseExpose_Call{Call: _e.mock.On("DiagnoseExpose", context1, objectReference)} +func (_e *GenericRestoreExposer_Expecter) DiagnoseExpose(context1 interface{}, objectReference interface{}) *GenericRestoreExposer_DiagnoseExpose_Call { + return &GenericRestoreExposer_DiagnoseExpose_Call{Call: _e.mock.On("DiagnoseExpose", context1, objectReference)} } -func (_c *MockGenericRestoreExposer_DiagnoseExpose_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference)) *MockGenericRestoreExposer_DiagnoseExpose_Call { +func (_c *GenericRestoreExposer_DiagnoseExpose_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference)) *GenericRestoreExposer_DiagnoseExpose_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -134,18 +134,18 @@ func (_c *MockGenericRestoreExposer_DiagnoseExpose_Call) Run(run func(context1 c return _c } -func (_c *MockGenericRestoreExposer_DiagnoseExpose_Call) Return(s string) *MockGenericRestoreExposer_DiagnoseExpose_Call { +func (_c *GenericRestoreExposer_DiagnoseExpose_Call) Return(s string) *GenericRestoreExposer_DiagnoseExpose_Call { _c.Call.Return(s) return _c } -func (_c *MockGenericRestoreExposer_DiagnoseExpose_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference) string) *MockGenericRestoreExposer_DiagnoseExpose_Call { +func (_c *GenericRestoreExposer_DiagnoseExpose_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference) string) *GenericRestoreExposer_DiagnoseExpose_Call { _c.Call.Return(run) return _c } -// Expose provides a mock function for the type MockGenericRestoreExposer -func (_mock *MockGenericRestoreExposer) Expose(context1 context.Context, objectReference v1.ObjectReference, genericRestoreExposeParam exposer.GenericRestoreExposeParam) error { +// Expose provides a mock function for the type GenericRestoreExposer +func (_mock *GenericRestoreExposer) Expose(context1 context.Context, objectReference v1.ObjectReference, genericRestoreExposeParam exposer.GenericRestoreExposeParam) error { ret := _mock.Called(context1, objectReference, genericRestoreExposeParam) if len(ret) == 0 { @@ -161,8 +161,8 @@ func (_mock *MockGenericRestoreExposer) Expose(context1 context.Context, objectR return r0 } -// MockGenericRestoreExposer_Expose_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Expose' -type MockGenericRestoreExposer_Expose_Call struct { +// GenericRestoreExposer_Expose_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Expose' +type GenericRestoreExposer_Expose_Call struct { *mock.Call } @@ -170,11 +170,11 @@ type MockGenericRestoreExposer_Expose_Call struct { // - context1 context.Context // - objectReference v1.ObjectReference // - genericRestoreExposeParam exposer.GenericRestoreExposeParam -func (_e *MockGenericRestoreExposer_Expecter) Expose(context1 interface{}, objectReference interface{}, genericRestoreExposeParam interface{}) *MockGenericRestoreExposer_Expose_Call { - return &MockGenericRestoreExposer_Expose_Call{Call: _e.mock.On("Expose", context1, objectReference, genericRestoreExposeParam)} +func (_e *GenericRestoreExposer_Expecter) Expose(context1 interface{}, objectReference interface{}, genericRestoreExposeParam interface{}) *GenericRestoreExposer_Expose_Call { + return &GenericRestoreExposer_Expose_Call{Call: _e.mock.On("Expose", context1, objectReference, genericRestoreExposeParam)} } -func (_c *MockGenericRestoreExposer_Expose_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference, genericRestoreExposeParam exposer.GenericRestoreExposeParam)) *MockGenericRestoreExposer_Expose_Call { +func (_c *GenericRestoreExposer_Expose_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference, genericRestoreExposeParam exposer.GenericRestoreExposeParam)) *GenericRestoreExposer_Expose_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -197,18 +197,18 @@ func (_c *MockGenericRestoreExposer_Expose_Call) Run(run func(context1 context.C return _c } -func (_c *MockGenericRestoreExposer_Expose_Call) Return(err error) *MockGenericRestoreExposer_Expose_Call { +func (_c *GenericRestoreExposer_Expose_Call) Return(err error) *GenericRestoreExposer_Expose_Call { _c.Call.Return(err) return _c } -func (_c *MockGenericRestoreExposer_Expose_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference, genericRestoreExposeParam exposer.GenericRestoreExposeParam) error) *MockGenericRestoreExposer_Expose_Call { +func (_c *GenericRestoreExposer_Expose_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference, genericRestoreExposeParam exposer.GenericRestoreExposeParam) error) *GenericRestoreExposer_Expose_Call { _c.Call.Return(run) return _c } -// GetExposed provides a mock function for the type MockGenericRestoreExposer -func (_mock *MockGenericRestoreExposer) GetExposed(context1 context.Context, objectReference v1.ObjectReference, client1 client.Client, s string, duration time.Duration) (*exposer.ExposeResult, error) { +// GetExposed provides a mock function for the type GenericRestoreExposer +func (_mock *GenericRestoreExposer) GetExposed(context1 context.Context, objectReference v1.ObjectReference, client1 client.Client, s string, duration time.Duration) (*exposer.ExposeResult, error) { ret := _mock.Called(context1, objectReference, client1, s, duration) if len(ret) == 0 { @@ -235,8 +235,8 @@ func (_mock *MockGenericRestoreExposer) GetExposed(context1 context.Context, obj return r0, r1 } -// MockGenericRestoreExposer_GetExposed_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetExposed' -type MockGenericRestoreExposer_GetExposed_Call struct { +// GenericRestoreExposer_GetExposed_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetExposed' +type GenericRestoreExposer_GetExposed_Call struct { *mock.Call } @@ -246,11 +246,11 @@ type MockGenericRestoreExposer_GetExposed_Call struct { // - client1 client.Client // - s string // - duration time.Duration -func (_e *MockGenericRestoreExposer_Expecter) GetExposed(context1 interface{}, objectReference interface{}, client1 interface{}, s interface{}, duration interface{}) *MockGenericRestoreExposer_GetExposed_Call { - return &MockGenericRestoreExposer_GetExposed_Call{Call: _e.mock.On("GetExposed", context1, objectReference, client1, s, duration)} +func (_e *GenericRestoreExposer_Expecter) GetExposed(context1 interface{}, objectReference interface{}, client1 interface{}, s interface{}, duration interface{}) *GenericRestoreExposer_GetExposed_Call { + return &GenericRestoreExposer_GetExposed_Call{Call: _e.mock.On("GetExposed", context1, objectReference, client1, s, duration)} } -func (_c *MockGenericRestoreExposer_GetExposed_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference, client1 client.Client, s string, duration time.Duration)) *MockGenericRestoreExposer_GetExposed_Call { +func (_c *GenericRestoreExposer_GetExposed_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference, client1 client.Client, s string, duration time.Duration)) *GenericRestoreExposer_GetExposed_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -283,18 +283,18 @@ func (_c *MockGenericRestoreExposer_GetExposed_Call) Run(run func(context1 conte return _c } -func (_c *MockGenericRestoreExposer_GetExposed_Call) Return(exposeResult *exposer.ExposeResult, err error) *MockGenericRestoreExposer_GetExposed_Call { +func (_c *GenericRestoreExposer_GetExposed_Call) Return(exposeResult *exposer.ExposeResult, err error) *GenericRestoreExposer_GetExposed_Call { _c.Call.Return(exposeResult, err) return _c } -func (_c *MockGenericRestoreExposer_GetExposed_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference, client1 client.Client, s string, duration time.Duration) (*exposer.ExposeResult, error)) *MockGenericRestoreExposer_GetExposed_Call { +func (_c *GenericRestoreExposer_GetExposed_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference, client1 client.Client, s string, duration time.Duration) (*exposer.ExposeResult, error)) *GenericRestoreExposer_GetExposed_Call { _c.Call.Return(run) return _c } -// PeekExposed provides a mock function for the type MockGenericRestoreExposer -func (_mock *MockGenericRestoreExposer) PeekExposed(context1 context.Context, objectReference v1.ObjectReference) error { +// PeekExposed provides a mock function for the type GenericRestoreExposer +func (_mock *GenericRestoreExposer) PeekExposed(context1 context.Context, objectReference v1.ObjectReference) error { ret := _mock.Called(context1, objectReference) if len(ret) == 0 { @@ -310,19 +310,19 @@ func (_mock *MockGenericRestoreExposer) PeekExposed(context1 context.Context, ob return r0 } -// MockGenericRestoreExposer_PeekExposed_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PeekExposed' -type MockGenericRestoreExposer_PeekExposed_Call struct { +// GenericRestoreExposer_PeekExposed_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PeekExposed' +type GenericRestoreExposer_PeekExposed_Call struct { *mock.Call } // PeekExposed is a helper method to define mock.On call // - context1 context.Context // - objectReference v1.ObjectReference -func (_e *MockGenericRestoreExposer_Expecter) PeekExposed(context1 interface{}, objectReference interface{}) *MockGenericRestoreExposer_PeekExposed_Call { - return &MockGenericRestoreExposer_PeekExposed_Call{Call: _e.mock.On("PeekExposed", context1, objectReference)} +func (_e *GenericRestoreExposer_Expecter) PeekExposed(context1 interface{}, objectReference interface{}) *GenericRestoreExposer_PeekExposed_Call { + return &GenericRestoreExposer_PeekExposed_Call{Call: _e.mock.On("PeekExposed", context1, objectReference)} } -func (_c *MockGenericRestoreExposer_PeekExposed_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference)) *MockGenericRestoreExposer_PeekExposed_Call { +func (_c *GenericRestoreExposer_PeekExposed_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference)) *GenericRestoreExposer_PeekExposed_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -340,49 +340,47 @@ func (_c *MockGenericRestoreExposer_PeekExposed_Call) Run(run func(context1 cont return _c } -func (_c *MockGenericRestoreExposer_PeekExposed_Call) Return(err error) *MockGenericRestoreExposer_PeekExposed_Call { +func (_c *GenericRestoreExposer_PeekExposed_Call) Return(err error) *GenericRestoreExposer_PeekExposed_Call { _c.Call.Return(err) return _c } -func (_c *MockGenericRestoreExposer_PeekExposed_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference) error) *MockGenericRestoreExposer_PeekExposed_Call { +func (_c *GenericRestoreExposer_PeekExposed_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference) error) *GenericRestoreExposer_PeekExposed_Call { _c.Call.Return(run) return _c } -// RebindVolume provides a mock function for the type MockGenericRestoreExposer -func (_mock *MockGenericRestoreExposer) RebindVolume(context1 context.Context, objectReference v1.ObjectReference, s string, s1 string, duration time.Duration) error { - ret := _mock.Called(context1, objectReference, s, s1, duration) +// RebindVolume provides a mock function for the type GenericRestoreExposer +func (_mock *GenericRestoreExposer) RebindVolume(context1 context.Context, objectReference v1.ObjectReference, genericRestoreRebindVolumeParam exposer.GenericRestoreRebindVolumeParam) error { + ret := _mock.Called(context1, objectReference, genericRestoreRebindVolumeParam) if len(ret) == 0 { panic("no return value specified for RebindVolume") } var r0 error - if returnFunc, ok := ret.Get(0).(func(context.Context, v1.ObjectReference, string, string, time.Duration) error); ok { - r0 = returnFunc(context1, objectReference, s, s1, duration) + if returnFunc, ok := ret.Get(0).(func(context.Context, v1.ObjectReference, exposer.GenericRestoreRebindVolumeParam) error); ok { + r0 = returnFunc(context1, objectReference, genericRestoreRebindVolumeParam) } else { r0 = ret.Error(0) } return r0 } -// MockGenericRestoreExposer_RebindVolume_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RebindVolume' -type MockGenericRestoreExposer_RebindVolume_Call struct { +// GenericRestoreExposer_RebindVolume_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RebindVolume' +type GenericRestoreExposer_RebindVolume_Call struct { *mock.Call } // RebindVolume is a helper method to define mock.On call // - context1 context.Context // - objectReference v1.ObjectReference -// - s string -// - s1 string -// - duration time.Duration -func (_e *MockGenericRestoreExposer_Expecter) RebindVolume(context1 interface{}, objectReference interface{}, s interface{}, s1 interface{}, duration interface{}) *MockGenericRestoreExposer_RebindVolume_Call { - return &MockGenericRestoreExposer_RebindVolume_Call{Call: _e.mock.On("RebindVolume", context1, objectReference, s, s1, duration)} +// - genericRestoreRebindVolumeParam exposer.GenericRestoreRebindVolumeParam +func (_e *GenericRestoreExposer_Expecter) RebindVolume(context1 interface{}, objectReference interface{}, genericRestoreRebindVolumeParam interface{}) *GenericRestoreExposer_RebindVolume_Call { + return &GenericRestoreExposer_RebindVolume_Call{Call: _e.mock.On("RebindVolume", context1, objectReference, genericRestoreRebindVolumeParam)} } -func (_c *MockGenericRestoreExposer_RebindVolume_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference, s string, s1 string, duration time.Duration)) *MockGenericRestoreExposer_RebindVolume_Call { +func (_c *GenericRestoreExposer_RebindVolume_Call) Run(run func(context1 context.Context, objectReference v1.ObjectReference, genericRestoreRebindVolumeParam exposer.GenericRestoreRebindVolumeParam)) *GenericRestoreExposer_RebindVolume_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -392,35 +390,25 @@ func (_c *MockGenericRestoreExposer_RebindVolume_Call) Run(run func(context1 con if args[1] != nil { arg1 = args[1].(v1.ObjectReference) } - var arg2 string + var arg2 exposer.GenericRestoreRebindVolumeParam if args[2] != nil { - arg2 = args[2].(string) - } - var arg3 string - if args[3] != nil { - arg3 = args[3].(string) - } - var arg4 time.Duration - if args[4] != nil { - arg4 = args[4].(time.Duration) + arg2 = args[2].(exposer.GenericRestoreRebindVolumeParam) } run( arg0, arg1, arg2, - arg3, - arg4, ) }) return _c } -func (_c *MockGenericRestoreExposer_RebindVolume_Call) Return(err error) *MockGenericRestoreExposer_RebindVolume_Call { +func (_c *GenericRestoreExposer_RebindVolume_Call) Return(err error) *GenericRestoreExposer_RebindVolume_Call { _c.Call.Return(err) return _c } -func (_c *MockGenericRestoreExposer_RebindVolume_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference, s string, s1 string, duration time.Duration) error) *MockGenericRestoreExposer_RebindVolume_Call { +func (_c *GenericRestoreExposer_RebindVolume_Call) RunAndReturn(run func(context1 context.Context, objectReference v1.ObjectReference, genericRestoreRebindVolumeParam exposer.GenericRestoreRebindVolumeParam) error) *GenericRestoreExposer_RebindVolume_Call { _c.Call.Return(run) return _c } diff --git a/pkg/exposer/pod_volume.go b/pkg/exposer/pod_volume.go index aeb6f1903..0526b2c5e 100644 --- a/pkg/exposer/pod_volume.go +++ b/pkg/exposer/pod_volume.go @@ -21,7 +21,7 @@ import ( "fmt" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/exposer/vgdp_counter.go b/pkg/exposer/vgdp_counter.go index cf6737c14..1f9850085 100644 --- a/pkg/exposer/vgdp_counter.go +++ b/pkg/exposer/vgdp_counter.go @@ -4,7 +4,7 @@ import ( "context" "sync/atomic" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" diff --git a/pkg/install/daemonset.go b/pkg/install/daemonset.go index 10d2764e6..190e785d8 100644 --- a/pkg/install/daemonset.go +++ b/pkg/install/daemonset.go @@ -18,7 +18,7 @@ package install import ( "fmt" - "path/filepath" + "path" "strings" appsv1api "k8s.io/api/apps/v1" @@ -68,8 +68,8 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1api.DaemonSet if c.forWindows { dsName = "node-agent-windows" } - hostPodsVolumePath := filepath.Join(c.kubeletRootDir, "pods") - hostPluginsVolumePath := filepath.Join(c.kubeletRootDir, "plugins") + hostPodsVolumePath := path.Join(strings.ReplaceAll(c.kubeletRootDir, "\\", "/"), "pods") + hostPluginsVolumePath := path.Join(strings.ReplaceAll(c.kubeletRootDir, "\\", "/"), "plugins") volumes := []corev1api.Volume{} volumeMounts := []corev1api.VolumeMount{} if !c.nodeAgentDisableHostPath { diff --git a/pkg/install/daemonset_test.go b/pkg/install/daemonset_test.go index 0f4de11bd..6cab7f063 100644 --- a/pkg/install/daemonset_test.go +++ b/pkg/install/daemonset_test.go @@ -86,6 +86,10 @@ func TestDaemonSet(t *testing.T) { assert.Equal(t, "/data/test/kubelet/pods", ds.Spec.Template.Spec.Volumes[0].HostPath.Path) assert.Equal(t, "/data/test/kubelet/plugins", ds.Spec.Template.Spec.Volumes[1].HostPath.Path) + ds = DaemonSet("velero", WithKubeletRootDir(`C:\var\lib\kubelet`)) + assert.Equal(t, "C:/var/lib/kubelet/pods", ds.Spec.Template.Spec.Volumes[0].HostPath.Path) + assert.Equal(t, "C:/var/lib/kubelet/plugins", ds.Spec.Template.Spec.Volumes[1].HostPath.Path) + ds = DaemonSet("velero", WithNodeAgentDisableHostPath(true)) assert.Len(t, ds.Spec.Template.Spec.Volumes, 1) assert.Len(t, ds.Spec.Template.Spec.Containers[0].VolumeMounts, 1) diff --git a/pkg/install/deployment.go b/pkg/install/deployment.go index 7af17bc53..4ce4b5a4f 100644 --- a/pkg/install/deployment.go +++ b/pkg/install/deployment.go @@ -50,6 +50,7 @@ type podTemplateConfig struct { serviceAccountName string uploaderType string defaultSnapshotMoveData bool + csiSnapshotEarlyFrequentPolling bool privilegedNodeAgent bool disableInformerCache bool scheduleSkipImmediately bool @@ -166,6 +167,12 @@ func WithDefaultSnapshotMoveData(b bool) podTemplateOption { } } +func WithCSISnapshotEarlyFrequentPolling(b bool) podTemplateOption { + return func(c *podTemplateConfig) { + c.csiSnapshotEarlyFrequentPolling = b + } +} + func WithDisableInformerCache(b bool) podTemplateOption { return func(c *podTemplateConfig) { c.disableInformerCache = b @@ -489,6 +496,15 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1api.Deployme }...) } + if c.csiSnapshotEarlyFrequentPolling { + deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, []corev1api.EnvVar{ + { + Name: "CSI_SNAPSHOT_EARLY_FREQUENT_POLLING", + Value: "true", + }, + }...) + } + deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, c.envVars...) if len(c.plugins) > 0 { diff --git a/pkg/install/install.go b/pkg/install/install.go index b60c4f9aa..4584c3902 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -23,7 +23,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" diff --git a/pkg/install/resources.go b/pkg/install/resources.go index 5c7534774..c4ec6f1bc 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -263,6 +263,7 @@ type VeleroOptions struct { DefaultVolumesToFsBackup bool UploaderType string DefaultSnapshotMoveData bool + CSISnapshotEarlyFrequentPolling bool DisableInformerCache bool ScheduleSkipImmediately bool PodResources kube.PodResources @@ -390,6 +391,10 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { deployOpts = append(deployOpts, WithDefaultSnapshotMoveData(true)) } + if o.CSISnapshotEarlyFrequentPolling { + deployOpts = append(deployOpts, WithCSISnapshotEarlyFrequentPolling(true)) + } + if o.DisableInformerCache { deployOpts = append(deployOpts, WithDisableInformerCache(true)) } diff --git a/pkg/itemblock/actions/pod_action.go b/pkg/itemblock/actions/pod_action.go index 2596e78a2..6e9955d4e 100644 --- a/pkg/itemblock/actions/pod_action.go +++ b/pkg/itemblock/actions/pod_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/itemblock/actions/pvc_action.go b/pkg/itemblock/actions/pvc_action.go index 6777ef566..996fcf3d7 100644 --- a/pkg/itemblock/actions/pvc_action.go +++ b/pkg/itemblock/actions/pvc_action.go @@ -19,7 +19,7 @@ package actions import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/itemblock/actions/service_account_action.go b/pkg/itemblock/actions/service_account_action.go index 91cdbbe59..d94b33f15 100644 --- a/pkg/itemblock/actions/service_account_action.go +++ b/pkg/itemblock/actions/service_account_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/itemoperationmap/backup_operation_map.go b/pkg/itemoperationmap/backup_operation_map.go index 47cdcac81..49dfbecc8 100644 --- a/pkg/itemoperationmap/backup_operation_map.go +++ b/pkg/itemoperationmap/backup_operation_map.go @@ -20,7 +20,7 @@ import ( "bytes" "sync" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/persistence" diff --git a/pkg/itemoperationmap/restore_operation_map.go b/pkg/itemoperationmap/restore_operation_map.go index 4256591bc..2586d7bb3 100644 --- a/pkg/itemoperationmap/restore_operation_map.go +++ b/pkg/itemoperationmap/restore_operation_map.go @@ -20,7 +20,7 @@ import ( "bytes" "sync" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/persistence" diff --git a/pkg/nodeagent/node_agent.go b/pkg/nodeagent/node_agent.go index 87efb896a..61720c99d 100644 --- a/pkg/nodeagent/node_agent.go +++ b/pkg/nodeagent/node_agent.go @@ -21,7 +21,7 @@ import ( "encoding/json" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/nodeagent/node_agent_test.go b/pkg/nodeagent/node_agent_test.go index 168e91de1..36b154a75 100644 --- a/pkg/nodeagent/node_agent_test.go +++ b/pkg/nodeagent/node_agent_test.go @@ -19,7 +19,7 @@ package nodeagent import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" diff --git a/pkg/persistence/object_store.go b/pkg/persistence/object_store.go index fd441e943..440ca8756 100644 --- a/pkg/persistence/object_store.go +++ b/pkg/persistence/object_store.go @@ -25,7 +25,7 @@ import ( snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/runtime/serializer" kerrors "k8s.io/apimachinery/pkg/util/errors" diff --git a/pkg/plugin/clientmgmt/backupitemaction/v1/restartable_backup_item_action.go b/pkg/plugin/clientmgmt/backupitemaction/v1/restartable_backup_item_action.go index 4bc28e487..0fe8fb583 100644 --- a/pkg/plugin/clientmgmt/backupitemaction/v1/restartable_backup_item_action.go +++ b/pkg/plugin/clientmgmt/backupitemaction/v1/restartable_backup_item_action.go @@ -17,7 +17,7 @@ limitations under the License. package v1 import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/runtime" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" diff --git a/pkg/plugin/clientmgmt/backupitemaction/v1/restartable_backup_item_action_test.go b/pkg/plugin/clientmgmt/backupitemaction/v1/restartable_backup_item_action_test.go index 8dc113df5..c1abcc471 100644 --- a/pkg/plugin/clientmgmt/backupitemaction/v1/restartable_backup_item_action_test.go +++ b/pkg/plugin/clientmgmt/backupitemaction/v1/restartable_backup_item_action_test.go @@ -19,7 +19,7 @@ package v1 import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action.go b/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action.go index 84ddb54b2..890aef75a 100644 --- a/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action.go +++ b/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action.go @@ -17,7 +17,7 @@ limitations under the License. package v2 import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/runtime" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" diff --git a/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action_test.go b/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action_test.go index bd1ee0ec2..d0400f760 100644 --- a/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action_test.go +++ b/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action_test.go @@ -19,7 +19,7 @@ package v2 import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/clientmgmt/itemblockaction/v1/restartable_item_block_action.go b/pkg/plugin/clientmgmt/itemblockaction/v1/restartable_item_block_action.go index 83742978f..49d72251f 100644 --- a/pkg/plugin/clientmgmt/itemblockaction/v1/restartable_item_block_action.go +++ b/pkg/plugin/clientmgmt/itemblockaction/v1/restartable_item_block_action.go @@ -17,7 +17,7 @@ limitations under the License. package v1 import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/runtime" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" diff --git a/pkg/plugin/clientmgmt/itemblockaction/v1/restartable_item_block_action_test.go b/pkg/plugin/clientmgmt/itemblockaction/v1/restartable_item_block_action_test.go index 04dd60652..99ac00016 100644 --- a/pkg/plugin/clientmgmt/itemblockaction/v1/restartable_item_block_action_test.go +++ b/pkg/plugin/clientmgmt/itemblockaction/v1/restartable_item_block_action_test.go @@ -19,7 +19,7 @@ package v1 import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/clientmgmt/manager_test.go b/pkg/plugin/clientmgmt/manager_test.go index 7576c42c5..95f911ffc 100644 --- a/pkg/plugin/clientmgmt/manager_test.go +++ b/pkg/plugin/clientmgmt/manager_test.go @@ -20,7 +20,7 @@ import ( "fmt" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/plugin/clientmgmt/process/process.go b/pkg/plugin/clientmgmt/process/process.go index 4d20a1784..8d496cd7d 100644 --- a/pkg/plugin/clientmgmt/process/process.go +++ b/pkg/plugin/clientmgmt/process/process.go @@ -17,8 +17,8 @@ limitations under the License. package process import ( + "github.com/cockroachdb/errors" plugin "github.com/hashicorp/go-plugin" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" diff --git a/pkg/plugin/clientmgmt/process/process_test.go b/pkg/plugin/clientmgmt/process/process_test.go index e67de8db9..455ad954e 100644 --- a/pkg/plugin/clientmgmt/process/process_test.go +++ b/pkg/plugin/clientmgmt/process/process_test.go @@ -18,7 +18,7 @@ package process import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" diff --git a/pkg/plugin/clientmgmt/process/registry.go b/pkg/plugin/clientmgmt/process/registry.go index 744048690..f667fa5d3 100644 --- a/pkg/plugin/clientmgmt/process/registry.go +++ b/pkg/plugin/clientmgmt/process/registry.go @@ -22,7 +22,7 @@ import ( "path/filepath" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/plugin/framework" diff --git a/pkg/plugin/clientmgmt/process/restartable_process.go b/pkg/plugin/clientmgmt/process/restartable_process.go index e285f82da..7f3053fa0 100644 --- a/pkg/plugin/clientmgmt/process/restartable_process.go +++ b/pkg/plugin/clientmgmt/process/restartable_process.go @@ -19,7 +19,7 @@ package process import ( "sync" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" ) diff --git a/pkg/plugin/clientmgmt/restartable_delete_item_action.go b/pkg/plugin/clientmgmt/restartable_delete_item_action.go index b566ede6f..7d9897240 100644 --- a/pkg/plugin/clientmgmt/restartable_delete_item_action.go +++ b/pkg/plugin/clientmgmt/restartable_delete_item_action.go @@ -17,7 +17,7 @@ limitations under the License. package clientmgmt import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" diff --git a/pkg/plugin/clientmgmt/restartable_delete_item_action_test.go b/pkg/plugin/clientmgmt/restartable_delete_item_action_test.go index 52edf7ffb..e41dc43fe 100644 --- a/pkg/plugin/clientmgmt/restartable_delete_item_action_test.go +++ b/pkg/plugin/clientmgmt/restartable_delete_item_action_test.go @@ -19,7 +19,7 @@ package clientmgmt import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/clientmgmt/restartable_object_store.go b/pkg/plugin/clientmgmt/restartable_object_store.go index 6e66d4b3e..c5eeeb60d 100644 --- a/pkg/plugin/clientmgmt/restartable_object_store.go +++ b/pkg/plugin/clientmgmt/restartable_object_store.go @@ -20,7 +20,7 @@ import ( "io" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" diff --git a/pkg/plugin/clientmgmt/restartable_object_store_test.go b/pkg/plugin/clientmgmt/restartable_object_store_test.go index 0b25e02d8..e1f07fe9b 100644 --- a/pkg/plugin/clientmgmt/restartable_object_store_test.go +++ b/pkg/plugin/clientmgmt/restartable_object_store_test.go @@ -22,7 +22,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/pkg/plugin/clientmgmt/restoreitemaction/v1/restartable_restore_item_action.go b/pkg/plugin/clientmgmt/restoreitemaction/v1/restartable_restore_item_action.go index a03ccecd2..e5cefceb4 100644 --- a/pkg/plugin/clientmgmt/restoreitemaction/v1/restartable_restore_item_action.go +++ b/pkg/plugin/clientmgmt/restoreitemaction/v1/restartable_restore_item_action.go @@ -17,7 +17,7 @@ limitations under the License. package v1 import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" diff --git a/pkg/plugin/clientmgmt/restoreitemaction/v1/restartable_restore_item_action_test.go b/pkg/plugin/clientmgmt/restoreitemaction/v1/restartable_restore_item_action_test.go index 08239fc93..1fefdbddd 100644 --- a/pkg/plugin/clientmgmt/restoreitemaction/v1/restartable_restore_item_action_test.go +++ b/pkg/plugin/clientmgmt/restoreitemaction/v1/restartable_restore_item_action_test.go @@ -19,7 +19,7 @@ package v1 import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/clientmgmt/restoreitemaction/v2/restartable_restore_item_action.go b/pkg/plugin/clientmgmt/restoreitemaction/v2/restartable_restore_item_action.go index c23787cfb..75b3e1937 100644 --- a/pkg/plugin/clientmgmt/restoreitemaction/v2/restartable_restore_item_action.go +++ b/pkg/plugin/clientmgmt/restoreitemaction/v2/restartable_restore_item_action.go @@ -17,7 +17,7 @@ limitations under the License. package v2 import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" diff --git a/pkg/plugin/clientmgmt/restoreitemaction/v2/restartable_restore_item_action_test.go b/pkg/plugin/clientmgmt/restoreitemaction/v2/restartable_restore_item_action_test.go index af6521e43..685835dfc 100644 --- a/pkg/plugin/clientmgmt/restoreitemaction/v2/restartable_restore_item_action_test.go +++ b/pkg/plugin/clientmgmt/restoreitemaction/v2/restartable_restore_item_action_test.go @@ -19,7 +19,7 @@ package v2 import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/clientmgmt/volumesnapshotter/v1/restartable_volume_snapshotter.go b/pkg/plugin/clientmgmt/volumesnapshotter/v1/restartable_volume_snapshotter.go index 7aec39872..c2afdd747 100644 --- a/pkg/plugin/clientmgmt/volumesnapshotter/v1/restartable_volume_snapshotter.go +++ b/pkg/plugin/clientmgmt/volumesnapshotter/v1/restartable_volume_snapshotter.go @@ -17,7 +17,7 @@ limitations under the License. package v1 import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt/process" diff --git a/pkg/plugin/clientmgmt/volumesnapshotter/v1/restartable_volume_snapshotter_test.go b/pkg/plugin/clientmgmt/volumesnapshotter/v1/restartable_volume_snapshotter_test.go index 8a2efe04c..6675651e7 100644 --- a/pkg/plugin/clientmgmt/volumesnapshotter/v1/restartable_volume_snapshotter_test.go +++ b/pkg/plugin/clientmgmt/volumesnapshotter/v1/restartable_volume_snapshotter_test.go @@ -20,7 +20,7 @@ import ( "testing" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/framework/action_resolver.go b/pkg/plugin/framework/action_resolver.go index ac8a0b1d0..f2b883afe 100644 --- a/pkg/plugin/framework/action_resolver.go +++ b/pkg/plugin/framework/action_resolver.go @@ -17,7 +17,7 @@ limitations under the License. package framework import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/plugin/framework/backup_item_action_client.go b/pkg/plugin/framework/backup_item_action_client.go index 724737d01..a0ac831ab 100644 --- a/pkg/plugin/framework/backup_item_action_client.go +++ b/pkg/plugin/framework/backup_item_action_client.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/grpc" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/plugin/framework/backup_item_action_server.go b/pkg/plugin/framework/backup_item_action_server.go index 7c18b4ef6..c3d8c5980 100644 --- a/pkg/plugin/framework/backup_item_action_server.go +++ b/pkg/plugin/framework/backup_item_action_server.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" diff --git a/pkg/plugin/framework/backup_item_action_test.go b/pkg/plugin/framework/backup_item_action_test.go index 1472eb115..32fda1e78 100644 --- a/pkg/plugin/framework/backup_item_action_test.go +++ b/pkg/plugin/framework/backup_item_action_test.go @@ -20,7 +20,7 @@ import ( "encoding/json" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_client.go b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_client.go index 64695dbe3..cff46aee2 100644 --- a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_client.go +++ b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_client.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/grpc" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_server.go b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_server.go index f8c894eba..1106bbebb 100644 --- a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_server.go +++ b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_server.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/protobuf/types/known/emptypb" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_test.go b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_test.go index 502c37502..a60adb26f 100644 --- a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_test.go +++ b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_test.go @@ -20,7 +20,7 @@ import ( "encoding/json" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/framework/common/handle_panic.go b/pkg/plugin/framework/common/handle_panic.go index 697ff588c..3e69563be 100644 --- a/pkg/plugin/framework/common/handle_panic.go +++ b/pkg/plugin/framework/common/handle_panic.go @@ -19,7 +19,7 @@ package common import ( "runtime/debug" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/grpc/codes" ) diff --git a/pkg/plugin/framework/common/plugin_config.go b/pkg/plugin/framework/common/plugin_config.go index 82b914352..b248334e0 100644 --- a/pkg/plugin/framework/common/plugin_config.go +++ b/pkg/plugin/framework/common/plugin_config.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" diff --git a/pkg/plugin/framework/common/server_errors.go b/pkg/plugin/framework/common/server_errors.go index 60eff50f4..2295f7a7f 100644 --- a/pkg/plugin/framework/common/server_errors.go +++ b/pkg/plugin/framework/common/server_errors.go @@ -17,7 +17,7 @@ limitations under the License. package common import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors/errbase" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/protoadapt" @@ -81,5 +81,5 @@ func ErrorStack(err error) *proto.Stack { } type StackTracer interface { - StackTrace() errors.StackTrace + StackTrace() errbase.StackTrace } diff --git a/pkg/plugin/framework/common/server_mux.go b/pkg/plugin/framework/common/server_mux.go index 4eecdb8d2..ab13bf5aa 100644 --- a/pkg/plugin/framework/common/server_mux.go +++ b/pkg/plugin/framework/common/server_mux.go @@ -19,7 +19,7 @@ package common import ( "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation" diff --git a/pkg/plugin/framework/delete_item_action_client.go b/pkg/plugin/framework/delete_item_action_client.go index bec5088db..90822fb3c 100644 --- a/pkg/plugin/framework/delete_item_action_client.go +++ b/pkg/plugin/framework/delete_item_action_client.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/grpc" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" diff --git a/pkg/plugin/framework/delete_item_action_server.go b/pkg/plugin/framework/delete_item_action_server.go index 01abe8dc3..fbdf62486 100644 --- a/pkg/plugin/framework/delete_item_action_server.go +++ b/pkg/plugin/framework/delete_item_action_server.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" diff --git a/pkg/plugin/framework/itemblockaction/v1/item_block_action_client.go b/pkg/plugin/framework/itemblockaction/v1/item_block_action_client.go index aa597c4af..34d92e1e3 100644 --- a/pkg/plugin/framework/itemblockaction/v1/item_block_action_client.go +++ b/pkg/plugin/framework/itemblockaction/v1/item_block_action_client.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/grpc" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/plugin/framework/itemblockaction/v1/item_block_action_server.go b/pkg/plugin/framework/itemblockaction/v1/item_block_action_server.go index 2d940550c..fc6de7694 100644 --- a/pkg/plugin/framework/itemblockaction/v1/item_block_action_server.go +++ b/pkg/plugin/framework/itemblockaction/v1/item_block_action_server.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" diff --git a/pkg/plugin/framework/itemblockaction/v1/item_block_action_test.go b/pkg/plugin/framework/itemblockaction/v1/item_block_action_test.go index 6e2a0e4d5..5a09f4e47 100644 --- a/pkg/plugin/framework/itemblockaction/v1/item_block_action_test.go +++ b/pkg/plugin/framework/itemblockaction/v1/item_block_action_test.go @@ -20,7 +20,7 @@ import ( "encoding/json" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/plugin/framework/object_store_client.go b/pkg/plugin/framework/object_store_client.go index b59f3d1b0..474a0173c 100644 --- a/pkg/plugin/framework/object_store_client.go +++ b/pkg/plugin/framework/object_store_client.go @@ -22,7 +22,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/grpc" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" diff --git a/pkg/plugin/framework/object_store_server.go b/pkg/plugin/framework/object_store_server.go index fbed21ecf..ae79d7cf1 100644 --- a/pkg/plugin/framework/object_store_server.go +++ b/pkg/plugin/framework/object_store_server.go @@ -22,7 +22,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" diff --git a/pkg/plugin/framework/plugin_lister.go b/pkg/plugin/framework/plugin_lister.go index 6db81c66d..c3d6b89e9 100644 --- a/pkg/plugin/framework/plugin_lister.go +++ b/pkg/plugin/framework/plugin_lister.go @@ -19,8 +19,8 @@ package framework import ( "context" + "github.com/cockroachdb/errors" plugin "github.com/hashicorp/go-plugin" - "github.com/pkg/errors" "google.golang.org/grpc" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" diff --git a/pkg/plugin/framework/restore_item_action_client.go b/pkg/plugin/framework/restore_item_action_client.go index 3a5a633f3..ea08fb0d7 100644 --- a/pkg/plugin/framework/restore_item_action_client.go +++ b/pkg/plugin/framework/restore_item_action_client.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/grpc" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/plugin/framework/restore_item_action_server.go b/pkg/plugin/framework/restore_item_action_server.go index 175a941bd..94dd2a911 100644 --- a/pkg/plugin/framework/restore_item_action_server.go +++ b/pkg/plugin/framework/restore_item_action_server.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" diff --git a/pkg/plugin/framework/restoreitemaction/v2/restore_item_action_client.go b/pkg/plugin/framework/restoreitemaction/v2/restore_item_action_client.go index 5e2f01c37..cc36b950f 100644 --- a/pkg/plugin/framework/restoreitemaction/v2/restore_item_action_client.go +++ b/pkg/plugin/framework/restoreitemaction/v2/restore_item_action_client.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/grpc" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/plugin/framework/restoreitemaction/v2/restore_item_action_server.go b/pkg/plugin/framework/restoreitemaction/v2/restore_item_action_server.go index 115961656..ace2a33e1 100644 --- a/pkg/plugin/framework/restoreitemaction/v2/restore_item_action_server.go +++ b/pkg/plugin/framework/restoreitemaction/v2/restore_item_action_server.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/emptypb" "google.golang.org/protobuf/types/known/timestamppb" diff --git a/pkg/plugin/framework/validation.go b/pkg/plugin/framework/validation.go index ba8f39be1..bbebf3ca3 100644 --- a/pkg/plugin/framework/validation.go +++ b/pkg/plugin/framework/validation.go @@ -17,7 +17,7 @@ limitations under the License. package framework import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/util/sets" ) diff --git a/pkg/plugin/framework/volume_snapshotter_client.go b/pkg/plugin/framework/volume_snapshotter_client.go index f7af07ce4..78df70863 100644 --- a/pkg/plugin/framework/volume_snapshotter_client.go +++ b/pkg/plugin/framework/volume_snapshotter_client.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "google.golang.org/grpc" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/plugin/framework/volume_snapshotter_server.go b/pkg/plugin/framework/volume_snapshotter_server.go index de30c823f..152f9451e 100644 --- a/pkg/plugin/framework/volume_snapshotter_server.go +++ b/pkg/plugin/framework/volume_snapshotter_server.go @@ -21,7 +21,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" diff --git a/pkg/plugin/velero/backupitemaction/v2/backup_item_action.go b/pkg/plugin/velero/backupitemaction/v2/backup_item_action.go index 3c23802f2..323c8ef92 100644 --- a/pkg/plugin/velero/backupitemaction/v2/backup_item_action.go +++ b/pkg/plugin/velero/backupitemaction/v2/backup_item_action.go @@ -21,7 +21,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/velero" diff --git a/pkg/plugin/velero/restoreitemaction/v2/restore_item_action.go b/pkg/plugin/velero/restoreitemaction/v2/restore_item_action.go index dfc35428f..1b2cff318 100644 --- a/pkg/plugin/velero/restoreitemaction/v2/restore_item_action.go +++ b/pkg/plugin/velero/restoreitemaction/v2/restore_item_action.go @@ -19,7 +19,7 @@ package v2 import ( "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/plugin/velero" diff --git a/pkg/podexec/pod_command_executor.go b/pkg/podexec/pod_command_executor.go index 0dbc28e95..4ba4d4dc9 100644 --- a/pkg/podexec/pod_command_executor.go +++ b/pkg/podexec/pod_command_executor.go @@ -23,7 +23,7 @@ import ( "slices" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/podexec/pod_command_executor_test.go b/pkg/podexec/pod_command_executor_test.go index 3286ba42d..13b00877a 100644 --- a/pkg/podexec/pod_command_executor_test.go +++ b/pkg/podexec/pod_command_executor_test.go @@ -25,7 +25,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" diff --git a/pkg/podvolume/backup_micro_service.go b/pkg/podvolume/backup_micro_service.go index d393b2bcb..246221d25 100644 --- a/pkg/podvolume/backup_micro_service.go +++ b/pkg/podvolume/backup_micro_service.go @@ -21,7 +21,7 @@ import ( "encoding/json" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" diff --git a/pkg/podvolume/backup_micro_service_test.go b/pkg/podvolume/backup_micro_service_test.go index bec46f353..eac17e4de 100644 --- a/pkg/podvolume/backup_micro_service_test.go +++ b/pkg/podvolume/backup_micro_service_test.go @@ -23,7 +23,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/podvolume/backupper.go b/pkg/podvolume/backupper.go index 6b534d5ed..5864a2090 100644 --- a/pkg/podvolume/backupper.go +++ b/pkg/podvolume/backupper.go @@ -21,7 +21,7 @@ import ( "fmt" "sync" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -412,18 +412,21 @@ func (b *backupper) WaitAllPodVolumesProcessed(log logrus.FieldLogger) []*velero case <-b.ctx.Done(): log.Error("timed out waiting for all PodVolumeBackups to complete") case <-done: - for _, obj := range b.pvbIndexer.List() { - pvb, ok := obj.(*velerov1api.PodVolumeBackup) - if !ok { - log.Errorf("expected PodVolumeBackup, but got %T", obj) - continue - } - podVolumeBackups = append(podVolumeBackups, pvb) - if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed { - log.Errorf("pod volume backup failed: %s", pvb.Status.Message) - } else if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled { - log.Errorf("pod volume backup canceled: %s", pvb.Status.Message) - } + } + + // Collect tracked PVBs regardless of whether we timed out or completed normally. + // On timeout, already-completed PVBs must still be persisted so their data remains restorable. + for _, obj := range b.pvbIndexer.List() { + pvb, ok := obj.(*velerov1api.PodVolumeBackup) + if !ok { + log.Errorf("expected PodVolumeBackup, but got %T", obj) + continue + } + podVolumeBackups = append(podVolumeBackups, pvb) + if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed { + log.Errorf("pod volume backup failed: %s", pvb.Status.Message) + } else if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled { + log.Errorf("pod volume backup canceled: %s", pvb.Status.Message) } } return podVolumeBackups diff --git a/pkg/podvolume/backupper_factory.go b/pkg/podvolume/backupper_factory.go index f75f1d30b..0f70fe8c0 100644 --- a/pkg/podvolume/backupper_factory.go +++ b/pkg/podvolume/backupper_factory.go @@ -19,7 +19,7 @@ package podvolume import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/client-go/tools/cache" ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache" diff --git a/pkg/podvolume/backupper_test.go b/pkg/podvolume/backupper_test.go index e6042ede1..66ad9e5ae 100644 --- a/pkg/podvolume/backupper_test.go +++ b/pkg/podvolume/backupper_test.go @@ -757,16 +757,18 @@ func TestWaitAllPodVolumesProcessed(t *testing.T) { statusToBeUpdated *velerov1api.PodVolumeBackupStatus expectedErr string expectedPVBPhase velerov1api.PodVolumeBackupPhase + expectedPVBCount int }{ { name: "contains no pvb should report no error", ctx: timeoutCtx, }, { - name: "context canceled", - ctx: timeoutCtx, - pvb: pvb, - expectedErr: "timed out waiting for all PodVolumeBackups to complete", + name: "context canceled should still return tracked pvbs", + ctx: timeoutCtx, + pvb: pvb, + expectedErr: "timed out waiting for all PodVolumeBackups to complete", + expectedPVBCount: 1, }, { name: "failed pvbs", @@ -834,6 +836,10 @@ func TestWaitAllPodVolumesProcessed(t *testing.T) { assert.Nil(t, logHook.entry) } + if c.expectedPVBCount > 0 { + require.Len(t, pvbs, c.expectedPVBCount) + } + if c.expectedPVBPhase != "" { require.Len(t, pvbs, 1) assert.Equal(t, c.expectedPVBPhase, pvbs[0].Status.Phase) diff --git a/pkg/podvolume/restore_micro_service.go b/pkg/podvolume/restore_micro_service.go index 77e84eb04..24f001147 100644 --- a/pkg/podvolume/restore_micro_service.go +++ b/pkg/podvolume/restore_micro_service.go @@ -23,7 +23,7 @@ import ( "path/filepath" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/podvolume/restore_micro_service_test.go b/pkg/podvolume/restore_micro_service_test.go index 1d25e342f..007060160 100644 --- a/pkg/podvolume/restore_micro_service_test.go +++ b/pkg/podvolume/restore_micro_service_test.go @@ -24,7 +24,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/podvolume/restorer.go b/pkg/podvolume/restorer.go index ce662d5de..bac22298e 100644 --- a/pkg/podvolume/restorer.go +++ b/pkg/podvolume/restorer.go @@ -23,7 +23,7 @@ import ( "github.com/vmware-tanzu/velero/internal/volume" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/podvolume/restorer_factory.go b/pkg/podvolume/restorer_factory.go index 178d720c8..6a037b992 100644 --- a/pkg/podvolume/restorer_factory.go +++ b/pkg/podvolume/restorer_factory.go @@ -19,7 +19,7 @@ package podvolume import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" diff --git a/pkg/repository/backup_repo_op.go b/pkg/repository/backup_repo_op.go index 36356e7af..146418d0e 100644 --- a/pkg/repository/backup_repo_op.go +++ b/pkg/repository/backup_repo_op.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/pkg/repository/config/aws.go b/pkg/repository/config/aws.go index 76a2829a5..3c8e75b22 100644 --- a/pkg/repository/config/aws.go +++ b/pkg/repository/config/aws.go @@ -31,7 +31,7 @@ import ( awsconfig "github.com/aws/aws-sdk-go-v2/config" s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" ) // getS3CredentialsFunc is used to make testing more convenient diff --git a/pkg/repository/config/azure.go b/pkg/repository/config/azure.go index 6662d13c6..28724dda1 100644 --- a/pkg/repository/config/azure.go +++ b/pkg/repository/config/azure.go @@ -17,7 +17,7 @@ limitations under the License. package config import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/pkg/util/azure" ) diff --git a/pkg/repository/ensurer.go b/pkg/repository/ensurer.go index 91cdb0d9e..e3c20fdd8 100644 --- a/pkg/repository/ensurer.go +++ b/pkg/repository/ensurer.go @@ -21,7 +21,7 @@ import ( "sync" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/wait" diff --git a/pkg/repository/keys/keys.go b/pkg/repository/keys/keys.go index 21423afe0..a077d4153 100644 --- a/pkg/repository/keys/keys.go +++ b/pkg/repository/keys/keys.go @@ -20,7 +20,7 @@ package keys import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/repository/maintenance/maintenance.go b/pkg/repository/maintenance/maintenance.go index 747d89f52..33c3fb1f8 100644 --- a/pkg/repository/maintenance/maintenance.go +++ b/pkg/repository/maintenance/maintenance.go @@ -25,7 +25,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" appsv1api "k8s.io/api/apps/v1" batchv1api "k8s.io/api/batch/v1" @@ -34,6 +34,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" @@ -610,6 +611,18 @@ func buildJob( } if config != nil && len(config.PodLabels) > 0 { for k, v := range config.PodLabels { + if k == RepositoryNameLabel { + logger.Warnf("Skipping user-provided label with reserved key %q; this label is managed internally by Velero", k) + continue + } + if errs := validation.IsQualifiedName(k); len(errs) > 0 { + logger.Warnf("Skipping user-provided label with invalid key %q: %s", k, strings.Join(errs, "; ")) + continue + } + if errs := validation.IsValidLabelValue(v); len(errs) > 0 { + logger.Warnf("Skipping user-provided label %q with invalid value %q: %s", k, v, strings.Join(errs, "; ")) + continue + } podLabels[k] = v } } else { @@ -623,6 +636,10 @@ func buildJob( podAnnotations := map[string]string{} if config != nil && len(config.PodAnnotations) > 0 { for k, v := range config.PodAnnotations { + if errs := validation.IsQualifiedName(k); len(errs) > 0 { + logger.Warnf("Skipping user-provided annotation with invalid key %q: %s", k, strings.Join(errs, "; ")) + continue + } podAnnotations[k] = v } } else { diff --git a/pkg/repository/maintenance/maintenance_test.go b/pkg/repository/maintenance/maintenance_test.go index 97eee1148..05fce89e9 100644 --- a/pkg/repository/maintenance/maintenance_test.go +++ b/pkg/repository/maintenance/maintenance_test.go @@ -1224,6 +1224,274 @@ func TestBuildJob(t *testing.T) { }, }, }, + { + name: "Invalid label key is skipped", + m: &velerotypes.JobConfigs{ + PodResources: &kube.PodResources{ + CPURequest: "100m", + MemoryRequest: "128Mi", + CPULimit: "200m", + MemoryLimit: "256Mi", + }, + PodLabels: map[string]string{ + "valid-label": "valid-value", + "INVALID KEY!!": "some-value", + }, + }, + deploy: deploy2, + logLevel: logrus.InfoLevel, + logFormat: logging.NewFormatFlag(), + expectedJobName: "test-123-maintain-job", + expectedError: false, + expectedEnv: []corev1api.EnvVar{ + { + Name: "test-name", + Value: "test-value", + }, + }, + expectedEnvFrom: []corev1api.EnvFromSource{ + { + ConfigMapRef: &corev1api.ConfigMapEnvSource{ + LocalObjectReference: corev1api.LocalObjectReference{ + Name: "test-configmap", + }, + }, + }, + { + SecretRef: &corev1api.SecretEnvSource{ + LocalObjectReference: corev1api.LocalObjectReference{ + Name: "test-secret", + }, + }, + }, + }, + expectedPodLabel: map[string]string{ + RepositoryNameLabel: "test-123", + "valid-label": "valid-value", + }, + expectedSecurityContext: nil, + expectedPodSecurityContext: nil, + expectedImagePullSecrets: []corev1api.LocalObjectReference{ + { + Name: "imagePullSecret1", + }, + }, + }, + { + name: "Invalid label value is skipped", + m: &velerotypes.JobConfigs{ + PodResources: &kube.PodResources{ + CPURequest: "100m", + MemoryRequest: "128Mi", + CPULimit: "200m", + MemoryLimit: "256Mi", + }, + PodLabels: map[string]string{ + "valid-label": "valid-value", + "another-label": "this value has spaces and is invalid", + }, + }, + deploy: deploy2, + logLevel: logrus.InfoLevel, + logFormat: logging.NewFormatFlag(), + expectedJobName: "test-123-maintain-job", + expectedError: false, + expectedEnv: []corev1api.EnvVar{ + { + Name: "test-name", + Value: "test-value", + }, + }, + expectedEnvFrom: []corev1api.EnvFromSource{ + { + ConfigMapRef: &corev1api.ConfigMapEnvSource{ + LocalObjectReference: corev1api.LocalObjectReference{ + Name: "test-configmap", + }, + }, + }, + { + SecretRef: &corev1api.SecretEnvSource{ + LocalObjectReference: corev1api.LocalObjectReference{ + Name: "test-secret", + }, + }, + }, + }, + expectedPodLabel: map[string]string{ + RepositoryNameLabel: "test-123", + "valid-label": "valid-value", + }, + expectedSecurityContext: nil, + expectedPodSecurityContext: nil, + expectedImagePullSecrets: []corev1api.LocalObjectReference{ + { + Name: "imagePullSecret1", + }, + }, + }, + { + name: "Label value exceeding 63 characters is skipped", + m: &velerotypes.JobConfigs{ + PodResources: &kube.PodResources{ + CPURequest: "100m", + MemoryRequest: "128Mi", + CPULimit: "200m", + MemoryLimit: "256Mi", + }, + PodLabels: map[string]string{ + "valid-label": "valid-value", + "long-value-label": "this-value-is-way-too-long-for-a-kubernetes-label-value-and-exceeds-sixty-three-characters", + }, + }, + deploy: deploy2, + logLevel: logrus.InfoLevel, + logFormat: logging.NewFormatFlag(), + expectedJobName: "test-123-maintain-job", + expectedError: false, + expectedEnv: []corev1api.EnvVar{ + { + Name: "test-name", + Value: "test-value", + }, + }, + expectedEnvFrom: []corev1api.EnvFromSource{ + { + ConfigMapRef: &corev1api.ConfigMapEnvSource{ + LocalObjectReference: corev1api.LocalObjectReference{ + Name: "test-configmap", + }, + }, + }, + { + SecretRef: &corev1api.SecretEnvSource{ + LocalObjectReference: corev1api.LocalObjectReference{ + Name: "test-secret", + }, + }, + }, + }, + expectedPodLabel: map[string]string{ + RepositoryNameLabel: "test-123", + "valid-label": "valid-value", + }, + expectedSecurityContext: nil, + expectedPodSecurityContext: nil, + expectedImagePullSecrets: []corev1api.LocalObjectReference{ + { + Name: "imagePullSecret1", + }, + }, + }, + { + name: "User-provided label cannot overwrite RepositoryNameLabel", + m: &velerotypes.JobConfigs{ + PodResources: &kube.PodResources{ + CPURequest: "100m", + MemoryRequest: "128Mi", + CPULimit: "200m", + MemoryLimit: "256Mi", + }, + PodLabels: map[string]string{ + RepositoryNameLabel: "user-override-attempt", + "valid-label": "valid-value", + }, + }, + deploy: deploy2, + logLevel: logrus.InfoLevel, + logFormat: logging.NewFormatFlag(), + expectedJobName: "test-123-maintain-job", + expectedError: false, + expectedEnv: []corev1api.EnvVar{ + { + Name: "test-name", + Value: "test-value", + }, + }, + expectedEnvFrom: []corev1api.EnvFromSource{ + { + ConfigMapRef: &corev1api.ConfigMapEnvSource{ + LocalObjectReference: corev1api.LocalObjectReference{ + Name: "test-configmap", + }, + }, + }, + { + SecretRef: &corev1api.SecretEnvSource{ + LocalObjectReference: corev1api.LocalObjectReference{ + Name: "test-secret", + }, + }, + }, + }, + expectedPodLabel: map[string]string{ + RepositoryNameLabel: "test-123", + "valid-label": "valid-value", + }, + expectedSecurityContext: nil, + expectedPodSecurityContext: nil, + expectedImagePullSecrets: []corev1api.LocalObjectReference{ + { + Name: "imagePullSecret1", + }, + }, + }, + { + name: "Invalid annotation key is skipped", + m: &velerotypes.JobConfigs{ + PodResources: &kube.PodResources{ + CPURequest: "100m", + MemoryRequest: "128Mi", + CPULimit: "200m", + MemoryLimit: "256Mi", + }, + PodAnnotations: map[string]string{ + "valid-annotation": "any value is fine for annotations, even with spaces!", + "INVALID KEY ANNO!": "some-value", + }, + }, + deploy: deploy2, + logLevel: logrus.InfoLevel, + logFormat: logging.NewFormatFlag(), + expectedJobName: "test-123-maintain-job", + expectedError: false, + expectedEnv: []corev1api.EnvVar{ + { + Name: "test-name", + Value: "test-value", + }, + }, + expectedEnvFrom: []corev1api.EnvFromSource{ + { + ConfigMapRef: &corev1api.ConfigMapEnvSource{ + LocalObjectReference: corev1api.LocalObjectReference{ + Name: "test-configmap", + }, + }, + }, + { + SecretRef: &corev1api.SecretEnvSource{ + LocalObjectReference: corev1api.LocalObjectReference{ + Name: "test-secret", + }, + }, + }, + }, + expectedPodLabel: map[string]string{ + RepositoryNameLabel: "test-123", + "azure.workload.identity/use": "fake-label-value", + }, + expectedPodAnnotation: map[string]string{ + "valid-annotation": "any value is fine for annotations, even with spaces!", + }, + expectedSecurityContext: nil, + expectedPodSecurityContext: nil, + expectedImagePullSecrets: []corev1api.LocalObjectReference{ + { + Name: "imagePullSecret1", + }, + }, + }, } param := provider.RepoParam{ @@ -1245,10 +1513,14 @@ func TestBuildJob(t *testing.T) { }, } + defaultBackupRepo := param.BackupRepo + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { if tc.backupRepository != nil { param.BackupRepo = tc.backupRepository + } else { + param.BackupRepo = defaultBackupRepo } // Create a fake clientset with resources @@ -1328,6 +1600,10 @@ func TestBuildJob(t *testing.T) { assert.Equal(t, tc.expectedPodLabel, job.Spec.Template.Labels) + if tc.expectedPodAnnotation != nil { + assert.Equal(t, tc.expectedPodAnnotation, job.Spec.Template.Annotations) + } + assert.Equal(t, tc.expectedImagePullSecrets, job.Spec.Template.Spec.ImagePullSecrets) } }) diff --git a/pkg/repository/manager/manager.go b/pkg/repository/manager/manager.go index 4d03931c0..d34c97624 100644 --- a/pkg/repository/manager/manager.go +++ b/pkg/repository/manager/manager.go @@ -21,7 +21,7 @@ import ( "fmt" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/pkg/repository/provider/unified_repo.go b/pkg/repository/provider/unified_repo.go index 2af59f190..bfe1a2bd9 100644 --- a/pkg/repository/provider/unified_repo.go +++ b/pkg/repository/provider/unified_repo.go @@ -27,7 +27,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/internal/credentials" diff --git a/pkg/repository/udmrepo/kopialib/backend/file_system.go b/pkg/repository/udmrepo/kopialib/backend/file_system.go index f0999e832..e3bf9e0c4 100644 --- a/pkg/repository/udmrepo/kopialib/backend/file_system.go +++ b/pkg/repository/udmrepo/kopialib/backend/file_system.go @@ -23,9 +23,9 @@ import ( "github.com/sirupsen/logrus" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/blob/filesystem" - "github.com/pkg/errors" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/logging" diff --git a/pkg/repository/udmrepo/kopialib/backend/utils.go b/pkg/repository/udmrepo/kopialib/backend/utils.go index 62ba4c322..d61a07410 100644 --- a/pkg/repository/udmrepo/kopialib/backend/utils.go +++ b/pkg/repository/udmrepo/kopialib/backend/utils.go @@ -22,8 +22,8 @@ import ( "strconv" "time" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/repo/logging" - "github.com/pkg/errors" ) func mustHaveString(key string, flags map[string]string) (string, error) { diff --git a/pkg/repository/udmrepo/kopialib/lib_repo.go b/pkg/repository/udmrepo/kopialib/lib_repo.go index 12dd0f688..151bf1cb2 100644 --- a/pkg/repository/udmrepo/kopialib/lib_repo.go +++ b/pkg/repository/udmrepo/kopialib/lib_repo.go @@ -27,6 +27,7 @@ import ( "sync/atomic" "time" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/compression" @@ -38,13 +39,12 @@ import ( "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/snapshot/snapshotfs" "github.com/kopia/kopia/snapshot/snapshotmaintenance" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/kopia" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend" - "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/freelist" + "github.com/vmware-tanzu/velero/pkg/util/freelist" ) type kopiaRepoService struct { @@ -92,6 +92,7 @@ type kopiaObjectWriterEx struct { description string compressor compression.Name splitter string + zeroObject object.ID writeLock sync.Mutex asyncWritesSem chan struct{} asyncWritesGroup sync.WaitGroup @@ -479,6 +480,7 @@ func (kr *kopiaRepository) NewObjectWriter(ctx context.Context, opt udmrepo.Obje description: opt.Description, compressor: getCompressorForObject(opt), blockSize: fixedBlockSize, + zeroObject: object.EmptyID, splitter: fixedSplitter1M, asyncWritesSem: asyncWritesSem, asyncBuffer: asyncBuffer, @@ -985,9 +987,119 @@ func (kow *kopiaObjectWriterEx) writeObjectAsync(objName string, entryID int, p } } -// TODO add implementation in following PRs +func (kow *kopiaObjectWriterEx) writeZeroObject(objName string, entryID int) error { + if kow.zeroObject == object.EmptyID { + zeroBuffer := make([]byte, kow.blockSize) + objectID, err := kow.writeObject(objName, zeroBuffer) + if err != nil { + return err + } + + kow.zeroObject = objectID + } + + kow.entryLock.Lock() + kow.entries[entryID].Object = kow.zeroObject + kow.entryLock.Unlock() + + return nil +} + func (kow *kopiaObjectWriterEx) WriteAt(p []byte, offset int64) (int, error) { - return 0, errors.New("not implemented") + kow.writeLock.Lock() + defer kow.writeLock.Unlock() + + if kow.rawRepoWriter == nil { + return 0, errors.New("object writer is closed or not open") + } + + if err := kow.getWriteError(); err != nil { + return 0, errors.Wrapf(err, "error happened during writing object") + } + + if offset%kow.blockSize != 0 { + return 0, errors.Errorf("invalid offset %v", offset) + } + + length := len(p) + if int64(length)%kow.blockSize != 0 { + return 0, errors.Errorf("invalid length %v", length) + } + + kow.entryLock.Lock() + curPos := int64(len(kow.entries)) * kow.blockSize + kow.entryLock.Unlock() + + if offset < curPos { + return 0, errors.Errorf("cannot write back, cur pos %v", curPos) + } + + if offset > curPos && kow.parentEntries != nil { + startEntry := int(curPos / kow.blockSize) + endEntry := int(offset / kow.blockSize) + if startEntry < len(kow.parentEntries) { + if len(kow.parentEntries) < endEntry { + endEntry = len(kow.parentEntries) + } + + for i := startEntry; i < endEntry; i++ { + e := kow.parentEntries[i] + + if e.Start != int64(i)*kow.blockSize { + return 0, errors.Errorf("parent entry %v start %v does not match expected start %v", i, e.Start, int64(i)*kow.blockSize) + } + + if e.Length != kow.blockSize { + return 0, errors.Errorf("parent entry %v length %v does not match child block size %v", i, e.Length, kow.blockSize) + } + } + + kow.entryLock.Lock() + kow.entries = append(kow.entries, kow.parentEntries[startEntry:endEntry]...) + curPos = int64(len(kow.entries)) * kow.blockSize + kow.entryLock.Unlock() + } + } + + entryID := 0 + for curPos < offset { + kow.entryLock.Lock() + entryID = len(kow.entries) + kow.entries = append(kow.entries, object.IndirectObjectEntry{ + Start: curPos, + Length: kow.blockSize, + }) + kow.entryLock.Unlock() + + objName := fmt.Sprintf("%s-b%v", kow.description, entryID) + if err := kow.writeZeroObject(objName, entryID); err != nil { + return 0, errors.Wrapf(err, "error writing zero object for %s", objName) + } + + curPos += kow.blockSize + } + + if length == 0 { + return length, nil + } + + for curPos < offset+int64(length) { + kow.entryLock.Lock() + entryID = len(kow.entries) + kow.entries = append(kow.entries, object.IndirectObjectEntry{ + Start: curPos, + Length: kow.blockSize, + }) + kow.entryLock.Unlock() + + buffOffset := curPos - offset + objName := fmt.Sprintf("%s-b%v", kow.description, entryID) + kow.writeObjectAsync(objName, entryID, p[buffOffset:buffOffset+kow.blockSize]) + + curPos += kow.blockSize + } + + return length, nil } func (kow *kopiaObjectWriterEx) Checkpoint() (udmrepo.ID, error) { diff --git a/pkg/repository/udmrepo/kopialib/lib_repo_ex_test.go b/pkg/repository/udmrepo/kopialib/lib_repo_ex_test.go index 6d9c5fc98..3294063a6 100644 --- a/pkg/repository/udmrepo/kopialib/lib_repo_ex_test.go +++ b/pkg/repository/udmrepo/kopialib/lib_repo_ex_test.go @@ -1,3 +1,19 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package kopialib import ( @@ -15,8 +31,8 @@ import ( "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" repomocks "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/mocks" - "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/freelist" velerotest "github.com/vmware-tanzu/velero/pkg/test" + "github.com/vmware-tanzu/velero/pkg/util/freelist" ) type mockDirectRepository struct { @@ -216,6 +232,7 @@ func TestKopiaObjectWriterEx_Write(t *testing.T) { inputData []byte expectedErr string expectedLen int + verify func(t *testing.T, kow *kopiaObjectWriterEx) }{ { name: "writer is closed", @@ -254,6 +271,59 @@ func TestKopiaObjectWriterEx_Write(t *testing.T) { inputData: make([]byte, 1023), expectedErr: "invalid length 1023", }, + { + name: "write object returns nil writer", + setupWriter: func(t *testing.T) *kopiaObjectWriterEx { + t.Helper() + mockRepoWriter := repomocks.NewMockRepositoryWriter(t) + mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(nil) + + return &kopiaObjectWriterEx{ + ctx: context.Background(), + rawRepoWriter: mockRepoWriter, + blockSize: 1024, + logger: velerotest.NewLogger(), + } + }, + inputData: make([]byte, 1024), + expectedLen: 1024, + verify: func(t *testing.T, kow *kopiaObjectWriterEx) { + t.Helper() + err := kow.getWriteError() + require.Error(t, err) + assert.Contains(t, err.Error(), "error opening writer for -b0") + }, + }, + { + name: "write object result error", + setupWriter: func(t *testing.T) *kopiaObjectWriterEx { + t.Helper() + mockRepoWriter := repomocks.NewMockRepositoryWriter(t) + mockWriter := repomocks.NewWriter(t) + + mockWriter.On("Write", mock.Anything).Return(1024, nil) + mockWriter.On("Close").Return(nil) + + mockWriter.On("Result").Return(object.EmptyID, errors.New("simulated result error")) + + mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter) + + return &kopiaObjectWriterEx{ + ctx: context.Background(), + rawRepoWriter: mockRepoWriter, + blockSize: 1024, + logger: velerotest.NewLogger(), + } + }, + inputData: make([]byte, 1024), + expectedLen: 1024, + verify: func(t *testing.T, kow *kopiaObjectWriterEx) { + t.Helper() + err := kow.getWriteError() + require.Error(t, err) + assert.Contains(t, err.Error(), "simulated result error") + }, + }, { name: "success sync write", setupWriter: func(t *testing.T) *kopiaObjectWriterEx { @@ -350,6 +420,9 @@ func TestKopiaObjectWriterEx_Write(t *testing.T) { } else { require.NoError(t, err) assert.Equal(t, tc.expectedLen, l) + if tc.verify != nil { + tc.verify(t, kow) + } } }) } @@ -406,6 +479,48 @@ func TestKopiaObjectWriterEx_Result(t *testing.T) { }, expectedID: udmrepo.ID("IIabcdef"), }, + { + name: "write indirect object encoding failure", + setupWriter: func(t *testing.T) *kopiaObjectWriterEx { + t.Helper() + mockRepoWriter := repomocks.NewMockRepositoryWriter(t) + mockWriter := repomocks.NewWriter(t) + + mockWriter.On("Write", mock.Anything).Return(0, errors.New("json encoding failed")) + mockWriter.On("Close").Return(nil) + + mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter) + + return &kopiaObjectWriterEx{ + ctx: context.Background(), + rawRepoWriter: mockRepoWriter, + logger: velerotest.NewLogger(), + } + }, + expectedErr: "error to write indirect object: unable to write indirect object index: json encoding failed", + }, + { + name: "write indirect object result failure", + setupWriter: func(t *testing.T) *kopiaObjectWriterEx { + t.Helper() + mockRepoWriter := repomocks.NewMockRepositoryWriter(t) + mockWriter := repomocks.NewWriter(t) + + mockWriter.On("Write", mock.Anything).Return(100, nil) + mockWriter.On("Close").Return(nil) + + mockWriter.On("Result").Return(object.EmptyID, errors.New("result generation failed")) + + mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter) + + return &kopiaObjectWriterEx{ + ctx: context.Background(), + rawRepoWriter: mockRepoWriter, + logger: velerotest.NewLogger(), + } + }, + expectedErr: "error to write indirect object: result generation failed", + }, } for _, tc := range testCases { @@ -516,7 +631,6 @@ func TestKopiaObjectWriterEx_MultipleWrites(t *testing.T) { mockRepoWriter := repomocks.NewMockRepositoryWriter(t) mockWriter := repomocks.NewWriter(t) - // Since we are writing 3 blocks, Write should be called 3 times and Close 3 times mockWriter.On("Write", mock.Anything).Return(1024, nil) mockWriter.On("Close").Return(nil) @@ -532,12 +646,10 @@ func TestKopiaObjectWriterEx_MultipleWrites(t *testing.T) { logger: velerotest.NewLogger(), } - // Write 1st block l, err := kow.Write(make([]byte, 1024)) require.NoError(t, err) assert.Equal(t, 1024, l) - // Write 2nd and 3rd block l, err = kow.Write(make([]byte, 2048)) require.NoError(t, err) assert.Equal(t, 2048, l) @@ -548,3 +660,652 @@ func TestKopiaObjectWriterEx_MultipleWrites(t *testing.T) { assert.Equal(t, int64(1024), kow.entries[1].Start) assert.Equal(t, int64(2048), kow.entries[2].Start) } + +func TestKopiaObjectWriterEx_WriteAt(t *testing.T) { + testCases := []struct { + name string + setupWriter func(t *testing.T) *kopiaObjectWriterEx + inputData []byte + offset int64 + expectedErr string + expectedLen int + verify func(t *testing.T, kow *kopiaObjectWriterEx) + }{ + { + name: "writer is closed", + setupWriter: func(t *testing.T) *kopiaObjectWriterEx { + t.Helper() + return &kopiaObjectWriterEx{ + rawRepoWriter: nil, + } + }, + inputData: make([]byte, 1024), + offset: 0, + expectedErr: "object writer is closed or not open", + }, + { + name: "invalid offset", + setupWriter: func(t *testing.T) *kopiaObjectWriterEx { + t.Helper() + return &kopiaObjectWriterEx{ + rawRepoWriter: repomocks.NewMockRepositoryWriter(t), + blockSize: 1024, + } + }, + inputData: make([]byte, 1024), + offset: 1023, + expectedErr: "invalid offset 1023", + }, + { + name: "invalid length", + setupWriter: func(t *testing.T) *kopiaObjectWriterEx { + t.Helper() + return &kopiaObjectWriterEx{ + rawRepoWriter: repomocks.NewMockRepositoryWriter(t), + blockSize: 1024, + } + }, + inputData: make([]byte, 1023), + offset: 0, + expectedErr: "invalid length 1023", + }, + { + name: "cannot write back", + setupWriter: func(t *testing.T) *kopiaObjectWriterEx { + t.Helper() + return &kopiaObjectWriterEx{ + rawRepoWriter: repomocks.NewMockRepositoryWriter(t), + blockSize: 1024, + entries: []object.IndirectObjectEntry{ + {Start: 0, Length: 1024}, + }, + } + }, + inputData: make([]byte, 1024), + offset: 0, + expectedErr: "cannot write back, cur pos 1024", + }, + { + name: "success write at cur pos", + setupWriter: func(t *testing.T) *kopiaObjectWriterEx { + t.Helper() + mockRepoWriter := repomocks.NewMockRepositoryWriter(t) + mockWriter := repomocks.NewWriter(t) + + mockWriter.On("Write", mock.Anything).Return(1024, nil) + mockWriter.On("Close").Return(nil) + + id, _ := object.ParseID("I12345") + mockWriter.On("Result").Return(id, nil) + + mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter) + + return &kopiaObjectWriterEx{ + ctx: context.Background(), + rawRepoWriter: mockRepoWriter, + blockSize: 1024, + logger: velerotest.NewLogger(), + } + }, + inputData: make([]byte, 1024), + offset: 0, + expectedLen: 1024, + verify: func(t *testing.T, kow *kopiaObjectWriterEx) { + t.Helper() + assert.Len(t, kow.entries, 1) + assert.Equal(t, int64(0), kow.entries[0].Start) + }, + }, + { + name: "success write with gap filling zeros", + setupWriter: func(t *testing.T) *kopiaObjectWriterEx { + t.Helper() + mockRepoWriter := repomocks.NewMockRepositoryWriter(t) + mockWriter := repomocks.NewWriter(t) + + mockWriter.On("Write", mock.Anything).Return(1024, nil) + mockWriter.On("Close").Return(nil) + + id, _ := object.ParseID("I12345") + mockWriter.On("Result").Return(id, nil) + + mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter) + + return &kopiaObjectWriterEx{ + ctx: context.Background(), + rawRepoWriter: mockRepoWriter, + blockSize: 1024, + zeroObject: object.EmptyID, + logger: velerotest.NewLogger(), + } + }, + inputData: make([]byte, 1024), + offset: 1024, + expectedLen: 1024, + verify: func(t *testing.T, kow *kopiaObjectWriterEx) { + t.Helper() + assert.Len(t, kow.entries, 2) + assert.Equal(t, int64(0), kow.entries[0].Start) + id, _ := object.ParseID("I12345") + assert.Equal(t, id, kow.entries[0].Object) + assert.Equal(t, id, kow.zeroObject) + assert.Equal(t, int64(1024), kow.entries[1].Start) + }, + }, + { + name: "success write with gap filling from parent", + setupWriter: func(t *testing.T) *kopiaObjectWriterEx { + t.Helper() + mockRepoWriter := repomocks.NewMockRepositoryWriter(t) + mockWriter := repomocks.NewWriter(t) + + mockWriter.On("Write", mock.Anything).Return(1024, nil) + mockWriter.On("Close").Return(nil) + + id, _ := object.ParseID("I12345") + mockWriter.On("Result").Return(id, nil) + + mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter) + + parentID, _ := object.ParseID("Iparent") + return &kopiaObjectWriterEx{ + ctx: context.Background(), + rawRepoWriter: mockRepoWriter, + blockSize: 1024, + parentEntries: []object.IndirectObjectEntry{ + {Start: 0, Length: 1024, Object: parentID}, + }, + logger: velerotest.NewLogger(), + } + }, + inputData: make([]byte, 1024), + offset: 1024, + expectedLen: 1024, + verify: func(t *testing.T, kow *kopiaObjectWriterEx) { + t.Helper() + assert.Len(t, kow.entries, 2) + assert.Equal(t, int64(0), kow.entries[0].Start) + parentID, _ := object.ParseID("Iparent") + assert.Equal(t, parentID, kow.entries[0].Object) + assert.Equal(t, int64(1024), kow.entries[1].Start) + }, + }, + { + name: "success write zero length", + setupWriter: func(t *testing.T) *kopiaObjectWriterEx { + t.Helper() + mockRepoWriter := repomocks.NewMockRepositoryWriter(t) + return &kopiaObjectWriterEx{ + ctx: context.Background(), + rawRepoWriter: mockRepoWriter, + blockSize: 1024, + logger: velerotest.NewLogger(), + } + }, + inputData: []byte{}, + offset: 0, + expectedLen: 0, + verify: func(t *testing.T, kow *kopiaObjectWriterEx) { + t.Helper() + assert.Empty(t, kow.entries) + }, + }, + { + name: "gap filling with invalid parent entry length", + setupWriter: func(t *testing.T) *kopiaObjectWriterEx { + t.Helper() + mockRepoWriter := repomocks.NewMockRepositoryWriter(t) + return &kopiaObjectWriterEx{ + ctx: context.Background(), + rawRepoWriter: mockRepoWriter, + blockSize: 1024, + parentEntries: []object.IndirectObjectEntry{ + {Start: 0, Length: 512, Object: object.EmptyID}, + }, + logger: velerotest.NewLogger(), + } + }, + inputData: make([]byte, 1024), + offset: 1024, + expectedErr: "parent entry 0 length 512 does not match child block size 1024", + }, + { + name: "gap filling partially with parent and rest with zeros", + setupWriter: func(t *testing.T) *kopiaObjectWriterEx { + t.Helper() + mockRepoWriter := repomocks.NewMockRepositoryWriter(t) + mockWriter := repomocks.NewWriter(t) + + mockWriter.On("Write", mock.Anything).Return(1024, nil) + mockWriter.On("Close").Return(nil) + + id, _ := object.ParseID("I12345") + mockWriter.On("Result").Return(id, nil) + + mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter) + + parentID, _ := object.ParseID("Iparent") + return &kopiaObjectWriterEx{ + ctx: context.Background(), + rawRepoWriter: mockRepoWriter, + blockSize: 1024, + zeroObject: object.EmptyID, + parentEntries: []object.IndirectObjectEntry{ + {Start: 0, Length: 1024, Object: parentID}, + }, + logger: velerotest.NewLogger(), + } + }, + inputData: make([]byte, 1024), + offset: 2048, + expectedLen: 1024, + verify: func(t *testing.T, kow *kopiaObjectWriterEx) { + t.Helper() + assert.Len(t, kow.entries, 3) + assert.Equal(t, int64(0), kow.entries[0].Start) + + parentID, _ := object.ParseID("Iparent") + assert.Equal(t, parentID, kow.entries[0].Object) + + zeroID, _ := object.ParseID("I12345") + assert.Equal(t, int64(1024), kow.entries[1].Start) + assert.Equal(t, zeroID, kow.entries[1].Object) + + assert.Equal(t, int64(2048), kow.entries[2].Start) + }, + }, + { + name: "writeZeroObject failure", + setupWriter: func(t *testing.T) *kopiaObjectWriterEx { + t.Helper() + mockRepoWriter := repomocks.NewMockRepositoryWriter(t) + mockWriter := repomocks.NewWriter(t) + + mockWriter.On("Write", mock.Anything).Return(0, errors.New("simulated zero object write error")) + mockWriter.On("Close").Return(nil) + + mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter) + + return &kopiaObjectWriterEx{ + ctx: context.Background(), + rawRepoWriter: mockRepoWriter, + blockSize: 1024, + zeroObject: object.EmptyID, + logger: velerotest.NewLogger(), + } + }, + inputData: make([]byte, 1024), + offset: 1024, + expectedErr: "error writing zero object for -b0: error writing for -b0: simulated zero object write error", + }, + { + name: "writeObject short write", + setupWriter: func(t *testing.T) *kopiaObjectWriterEx { + t.Helper() + mockRepoWriter := repomocks.NewMockRepositoryWriter(t) + mockWriter := repomocks.NewWriter(t) + + mockWriter.On("Write", mock.Anything).Return(512, nil) + mockWriter.On("Close").Return(nil) + + mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter) + + return &kopiaObjectWriterEx{ + ctx: context.Background(), + rawRepoWriter: mockRepoWriter, + blockSize: 1024, + logger: velerotest.NewLogger(), + } + }, + inputData: make([]byte, 1024), + offset: 0, + expectedLen: 1024, + verify: func(t *testing.T, kow *kopiaObjectWriterEx) { + t.Helper() + err := kow.getWriteError() + require.Error(t, err) + assert.Contains(t, err.Error(), "short write for -b0") + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + kow := tc.setupWriter(t) + l, err := kow.WriteAt(tc.inputData, tc.offset) + + if kow.asyncWritesSem != nil { + kow.asyncWritesGroup.Wait() + } + + if tc.expectedErr != "" { + assert.EqualError(t, err, tc.expectedErr) + } else { + require.NoError(t, err) + assert.Equal(t, tc.expectedLen, l) + if tc.verify != nil { + tc.verify(t, kow) + } + } + }) + } +} + +func TestKopiaObjectWriterEx_MultipleWriteAt(t *testing.T) { + mockRepoWriter := repomocks.NewMockRepositoryWriter(t) + mockWriter := repomocks.NewWriter(t) + + mockWriter.On("Write", mock.Anything).Return(1024, nil) + mockWriter.On("Close").Return(nil) + + id, _ := object.ParseID("I12345") + mockWriter.On("Result").Return(id, nil) + + mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter) + + kow := &kopiaObjectWriterEx{ + ctx: context.Background(), + rawRepoWriter: mockRepoWriter, + blockSize: 1024, + zeroObject: object.EmptyID, + logger: velerotest.NewLogger(), + } + + l, err := kow.WriteAt(make([]byte, 1024), 0) + require.NoError(t, err) + assert.Equal(t, 1024, l) + + l, err = kow.WriteAt(make([]byte, 1024), 2048) + require.NoError(t, err) + assert.Equal(t, 1024, l) + + assert.Len(t, kow.entries, 3) + assert.Equal(t, int64(0), kow.entries[0].Start) + assert.Equal(t, int64(1024), kow.entries[1].Start) + assert.Equal(t, id, kow.entries[1].Object) + assert.Equal(t, int64(2048), kow.entries[2].Start) +} + +func TestKopiaObjectWriterEx_ConcurrentWriteAt(t *testing.T) { + mockRepoWriter := repomocks.NewMockRepositoryWriter(t) + mockWriter := repomocks.NewWriter(t) + + mockWriter.On("Write", mock.Anything).Return(1024, nil) + mockWriter.On("Close").Return(nil) + + id, _ := object.ParseID("I12345") + mockWriter.On("Result").Return(id, nil) + + mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter) + + kow := &kopiaObjectWriterEx{ + ctx: context.Background(), + rawRepoWriter: mockRepoWriter, + blockSize: 1024, + logger: velerotest.NewLogger(), + } + + numGoroutines := 10 + var wg sync.WaitGroup + + start := make(chan struct{}) + + for i := 0; i < numGoroutines; i++ { + wg.Add(1) + go func(offset int64) { + defer wg.Done() + <-start + + data := make([]byte, 1024) + _, err := kow.WriteAt(data, offset) + + if err != nil { + assert.Contains(t, err.Error(), "cannot write back") + } + }(int64(i * 1024)) + } + + close(start) + wg.Wait() + + assert.NotEmpty(t, kow.entries) +} + +type dummyObjectWriter struct { + writtenBytes int +} + +func (dw *dummyObjectWriter) Write(p []byte) (int, error) { + dw.writtenBytes += len(p) + return len(p), nil +} + +func (dw *dummyObjectWriter) Close() error { + return nil +} + +func (dw *dummyObjectWriter) Result() (object.ID, error) { + id, _ := object.ParseID("I12345") + return id, nil +} + +func (dw *dummyObjectWriter) Checkpoint() (object.ID, error) { + return dw.Result() +} + +type dummyRepoWriter struct { + repo.RepositoryWriter +} + +func (drw *dummyRepoWriter) NewObjectWriter(ctx context.Context, opt object.WriterOptions) object.Writer { + return &dummyObjectWriter{} +} + +func TestKopiaObjectWriterEx_LargeSequentialWrite(t *testing.T) { + mockRepoWriter := &dummyRepoWriter{} + + blockSize := int64(1 << 20) + + kow := &kopiaObjectWriterEx{ + ctx: context.Background(), + rawRepoWriter: mockRepoWriter, + blockSize: blockSize, + logger: velerotest.NewLogger(), + } + + data := make([]byte, blockSize) + blocks := 5120 + + for i := 0; i < blocks; i++ { + l, err := kow.Write(data) + require.NoError(t, err) + assert.Equal(t, int(blockSize), l) + } + + assert.Len(t, kow.entries, blocks) + assert.Equal(t, int64(blocks-1)*blockSize, kow.entries[blocks-1].Start) +} + +func TestKopiaObjectWriterEx_LargeSparseWriteAt(t *testing.T) { + mockRepoWriter := &dummyRepoWriter{} + + blockSize := int64(1 << 20) + + kow := &kopiaObjectWriterEx{ + ctx: context.Background(), + rawRepoWriter: mockRepoWriter, + blockSize: blockSize, + zeroObject: object.EmptyID, + logger: velerotest.NewLogger(), + } + + var offset int64 = 5 * 1024 * 1024 * 1024 + + data := make([]byte, blockSize) + l, err := kow.WriteAt(data, offset) + require.NoError(t, err) + assert.Equal(t, int(blockSize), l) + + expectedEntries := 5121 + assert.Len(t, kow.entries, expectedEntries) + assert.Equal(t, int64(0), kow.entries[0].Start) + assert.Equal(t, offset, kow.entries[expectedEntries-1].Start) +} + +func TestKopiaObjectWriterEx_MixedWriteAndWriteAt(t *testing.T) { + mockRepoWriter := repomocks.NewMockRepositoryWriter(t) + mockWriter := repomocks.NewWriter(t) + + blockSize := int64(1024) + + mockWriter.On("Write", mock.Anything).Return(int(blockSize), nil) + mockWriter.On("Close").Return(nil) + + id, _ := object.ParseID("I12345") + mockWriter.On("Result").Return(id, nil) + + mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter) + + kow := &kopiaObjectWriterEx{ + ctx: context.Background(), + rawRepoWriter: mockRepoWriter, + blockSize: blockSize, + zeroObject: object.EmptyID, + logger: velerotest.NewLogger(), + } + + // 1. Write 1 block sequentially + data1 := make([]byte, blockSize) + l, err := kow.Write(data1) + require.NoError(t, err) + assert.Equal(t, int(blockSize), l) + + // Entries: [0:1024] + assert.Len(t, kow.entries, 1) + assert.Equal(t, int64(0), kow.entries[0].Start) + + // 2. WriteAt with gap (offset = 2048). This creates a gap block at 1024 + data2 := make([]byte, blockSize) + l, err = kow.WriteAt(data2, 2048) + require.NoError(t, err) + assert.Equal(t, int(blockSize), l) + + // Entries should now be 3: [0:1024, 1024:2048(zero object), 2048:3072] + assert.Len(t, kow.entries, 3) + assert.Equal(t, int64(0), kow.entries[0].Start) + assert.Equal(t, int64(1024), kow.entries[1].Start) + assert.Equal(t, id, kow.entries[1].Object) // filled with zero block + assert.Equal(t, int64(2048), kow.entries[2].Start) + + // 3. Write another block sequentially. It should append at 3072. + data3 := make([]byte, blockSize) + l, err = kow.Write(data3) + require.NoError(t, err) + assert.Equal(t, int(blockSize), l) + + // Entries should now be 4: [0:1024, 1024:2048(zero object), 2048:3072, 3072:4096] + assert.Len(t, kow.entries, 4) + assert.Equal(t, int64(3072), kow.entries[3].Start) +} + +func TestKopiaObjectWriterEx_ConcurrentAsyncErrors(t *testing.T) { + mockRepoWriter := repomocks.NewMockRepositoryWriter(t) + mockWriter := repomocks.NewWriter(t) + + mockWriter.On("Write", mock.Anything).Return(0, errors.New("simulated async error")) + mockWriter.On("Close").Return(nil) + + mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter) + + sem := make(chan struct{}, 10) + buf := freelist.New(10*1024, 1024) + + kow := &kopiaObjectWriterEx{ + ctx: context.Background(), + rawRepoWriter: mockRepoWriter, + blockSize: 1024, + asyncWritesSem: sem, + asyncBuffer: buf, + logger: velerotest.NewLogger(), + } + + data := make([]byte, 1024) + + // Issue multiple writes so they all spawn async goroutines + // First few writes shouldn't fail immediately until getWriteError catches the asynchronous fault + for i := 0; i < 10; i++ { + l, err := kow.Write(data) + require.NoError(t, err) + assert.Equal(t, 1024, l) + } + + id, err := kow.Result() + + require.Error(t, err) + assert.Contains(t, err.Error(), "simulated async error") + assert.Equal(t, udmrepo.ID(""), id) +} + +func TestKopiaObjectWriterEx_ConcurrentWriteAndWriteAt(t *testing.T) { + mockRepoWriter := repomocks.NewMockRepositoryWriter(t) + mockWriter := repomocks.NewWriter(t) + + mockWriter.On("Write", mock.Anything).Return(1024, nil) + mockWriter.On("Close").Return(nil) + + id, _ := object.ParseID("I12345") + mockWriter.On("Result").Return(id, nil) + + mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter) + + kow := &kopiaObjectWriterEx{ + ctx: context.Background(), + rawRepoWriter: mockRepoWriter, + blockSize: 1024, + zeroObject: object.EmptyID, + logger: velerotest.NewLogger(), + } + + var wg sync.WaitGroup + start := make(chan struct{}) + + for i := 0; i < 5; i++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + l, err := kow.Write(make([]byte, 1024)) + require.NoError(t, err) + assert.Equal(t, 1024, l) + }() + } + + // Fire multiple sparse WriteAts alongside them + // Note: Because order is totally random and WriteAt strictly demands monotonic offsets, + // some will hit the legitimate "cannot write back" error, which we safely expect. + for i := 0; i < 5; i++ { + wg.Add(1) + go func(offset int64) { + defer wg.Done() + <-start + _, err := kow.WriteAt(make([]byte, 1024), offset) + if err != nil { + assert.Contains(t, err.Error(), "cannot write back") + } + }(int64(i * 2048)) + } + + close(start) + wg.Wait() + + // We only care that the locking effectively mitigated a panic or slice data corruption + assert.NotEmpty(t, kow.entries) +} + +func TestKopiaObjectWriterEx_Checkpoint(t *testing.T) { + kow := &kopiaObjectWriterEx{} + id, err := kow.Checkpoint() + require.Error(t, err) + assert.Equal(t, udmrepo.ID(""), id) + assert.Equal(t, "not supported", err.Error()) +} diff --git a/pkg/repository/udmrepo/kopialib/lib_repo_test.go b/pkg/repository/udmrepo/kopialib/lib_repo_test.go index 1776f9be0..370b82b9e 100644 --- a/pkg/repository/udmrepo/kopialib/lib_repo_test.go +++ b/pkg/repository/udmrepo/kopialib/lib_repo_test.go @@ -25,12 +25,12 @@ import ( "testing" "time" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/repo/object" "github.com/kopia/kopia/snapshot" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/repository/udmrepo/kopialib/repo_init.go b/pkg/repository/udmrepo/kopialib/repo_init.go index ade9039d7..4e62c9087 100644 --- a/pkg/repository/udmrepo/kopialib/repo_init.go +++ b/pkg/repository/udmrepo/kopialib/repo_init.go @@ -26,11 +26,11 @@ import ( "github.com/sirupsen/logrus" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/maintenance" - "github.com/pkg/errors" "github.com/vmware-tanzu/velero/pkg/kopia" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" diff --git a/pkg/repository/udmrepo/kopialib/repo_init_test.go b/pkg/repository/udmrepo/kopialib/repo_init_test.go index 3b8a52be2..c8b8e6aa1 100644 --- a/pkg/repository/udmrepo/kopialib/repo_init_test.go +++ b/pkg/repository/udmrepo/kopialib/repo_init_test.go @@ -38,7 +38,7 @@ import ( repomocks "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/mocks" storagemocks "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/kopialib/backend/mocks" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" ) type comparableError struct { diff --git a/pkg/repository/udmrepo/repo.go b/pkg/repository/udmrepo/repo.go index bd0e6eb19..76cdc5f1d 100644 --- a/pkg/repository/udmrepo/repo.go +++ b/pkg/repository/udmrepo/repo.go @@ -189,6 +189,7 @@ type BackupRepo interface { Close(ctx context.Context) error } +// ObjectReader is used to read data from an object in the backup repository. type ObjectReader interface { io.ReadCloser io.Seeker @@ -197,11 +198,16 @@ type ObjectReader interface { Length() int64 } +// ObjectWriter is used to write data to an object in the backup repository. +// The sequential and random write behavior is determined by the backup repository implementation, +// it may not exactly follow io.Writer or io.WriterAt, in terms of writer point movement, io pattern, etc. +// The uploaders should refer to the backup repository implementation to use the ObjectWriter correctly. type ObjectWriter interface { - io.WriteCloser + // Write writes data to the object in the sequential manner. + Write([]byte) (int, error) - // WriterAt is used in the cases that the object is not written sequentially - io.WriterAt + // WriterAt is used in the cases that the object is not written sequentially. + WriteAt([]byte, int64) (int, error) // Checkpoint is periodically called to preserve the state of data written to the repo so far. // Checkpoint returns a unified identifier that represent the current state. @@ -211,4 +217,7 @@ type ObjectWriter interface { // Result waits for the completion of the object write. // Result returns the object's unified identifier after the write completes. Result() (ID, error) + + // Close closes the object writer and releases all resources. + Close() error } diff --git a/pkg/restore/actions/add_pvc_from_pod_action.go b/pkg/restore/actions/add_pvc_from_pod_action.go index 3e88f796a..3dd92424e 100644 --- a/pkg/restore/actions/add_pvc_from_pod_action.go +++ b/pkg/restore/actions/add_pvc_from_pod_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/restore/actions/admissionwebhook_config_action.go b/pkg/restore/actions/admissionwebhook_config_action.go index 82599dc62..3291fff71 100644 --- a/pkg/restore/actions/admissionwebhook_config_action.go +++ b/pkg/restore/actions/admissionwebhook_config_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/restore/actions/change_image_name_action.go b/pkg/restore/actions/change_image_name_action.go index 828da40d6..69e9e5f33 100644 --- a/pkg/restore/actions/change_image_name_action.go +++ b/pkg/restore/actions/change_image_name_action.go @@ -21,7 +21,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/restore/actions/change_storageclass_action.go b/pkg/restore/actions/change_storageclass_action.go index f9f031fe3..bdee2aa13 100644 --- a/pkg/restore/actions/change_storageclass_action.go +++ b/pkg/restore/actions/change_storageclass_action.go @@ -19,7 +19,7 @@ package actions import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" diff --git a/pkg/restore/actions/change_storageclass_action_test.go b/pkg/restore/actions/change_storageclass_action_test.go index 13bbcdcc4..72cab80e7 100644 --- a/pkg/restore/actions/change_storageclass_action_test.go +++ b/pkg/restore/actions/change_storageclass_action_test.go @@ -19,7 +19,7 @@ package actions import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/pkg/restore/actions/clusterrolebinding_action.go b/pkg/restore/actions/clusterrolebinding_action.go index a11665c1a..edc2ed961 100644 --- a/pkg/restore/actions/clusterrolebinding_action.go +++ b/pkg/restore/actions/clusterrolebinding_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/actions/crd_v1_preserve_unknown_fields_action.go b/pkg/restore/actions/crd_v1_preserve_unknown_fields_action.go index 9e4cf7e2f..fda389de4 100644 --- a/pkg/restore/actions/crd_v1_preserve_unknown_fields_action.go +++ b/pkg/restore/actions/crd_v1_preserve_unknown_fields_action.go @@ -19,7 +19,7 @@ package actions import ( "encoding/json" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/actions/csi/pvc_action.go b/pkg/restore/actions/csi/pvc_action.go index dee23cf70..6dd98c6b6 100644 --- a/pkg/restore/actions/csi/pvc_action.go +++ b/pkg/restore/actions/csi/pvc_action.go @@ -23,7 +23,7 @@ import ( snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -427,6 +427,7 @@ func newDataDownload( TargetVolume: velerov2alpha1.TargetVolumeSpec{ PVC: pvc.Name, Namespace: newNamespace, + FSType: dataUploadResult.FSType, }, BackupStorageLocation: dataUploadResult.BackupStorageLocation, DataMover: dataUploadResult.DataMover, diff --git a/pkg/restore/actions/csi/volumesnapshot_action.go b/pkg/restore/actions/csi/volumesnapshot_action.go index dec33d4ef..da5d4d281 100644 --- a/pkg/restore/actions/csi/volumesnapshot_action.go +++ b/pkg/restore/actions/csi/volumesnapshot_action.go @@ -20,9 +20,9 @@ import ( "context" "fmt" + "github.com/cockroachdb/errors" volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/restore/actions/csi/volumesnapshotclass_action.go b/pkg/restore/actions/csi/volumesnapshotclass_action.go index c906a04b2..595e39b31 100644 --- a/pkg/restore/actions/csi/volumesnapshotclass_action.go +++ b/pkg/restore/actions/csi/volumesnapshotclass_action.go @@ -17,8 +17,8 @@ limitations under the License. package csi import ( + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/restore/actions/csi/volumesnapshotcontent_action.go b/pkg/restore/actions/csi/volumesnapshotcontent_action.go index 00a25c86f..dc18b9bb3 100644 --- a/pkg/restore/actions/csi/volumesnapshotcontent_action.go +++ b/pkg/restore/actions/csi/volumesnapshotcontent_action.go @@ -19,8 +19,8 @@ package csi import ( "context" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/restore/actions/dataupload_retrieve_action.go b/pkg/restore/actions/dataupload_retrieve_action.go index a7efdc5f7..77e4766f5 100644 --- a/pkg/restore/actions/dataupload_retrieve_action.go +++ b/pkg/restore/actions/dataupload_retrieve_action.go @@ -20,7 +20,7 @@ import ( "context" "encoding/json" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -80,6 +80,7 @@ func (d *DataUploadRetrieveAction) Execute(input *velero.RestoreItemActionExecut SourceNamespace: dataUpload.Spec.SourceNamespace, DataMoverResult: dataUpload.Status.DataMoverResult, NodeOS: dataUpload.Status.NodeOS, + FSType: dataUpload.Spec.SourceFSType, } jsonBytes, err := json.Marshal(dataUploadResult) diff --git a/pkg/restore/actions/init_restorehook_pod_action.go b/pkg/restore/actions/init_restorehook_pod_action.go index 7614ef085..f6fee4eeb 100644 --- a/pkg/restore/actions/init_restorehook_pod_action.go +++ b/pkg/restore/actions/init_restorehook_pod_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/actions/job_action.go b/pkg/restore/actions/job_action.go index 1eabc208c..5ee4b8c6a 100644 --- a/pkg/restore/actions/job_action.go +++ b/pkg/restore/actions/job_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" batchv1api "k8s.io/api/batch/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/actions/pod_action.go b/pkg/restore/actions/pod_action.go index a9db3ed7e..ca12c9031 100644 --- a/pkg/restore/actions/pod_action.go +++ b/pkg/restore/actions/pod_action.go @@ -19,7 +19,7 @@ package actions import ( "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/actions/pod_volume_restore_action.go b/pkg/restore/actions/pod_volume_restore_action.go index e26a53034..5f2b3db3e 100644 --- a/pkg/restore/actions/pod_volume_restore_action.go +++ b/pkg/restore/actions/pod_volume_restore_action.go @@ -23,7 +23,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/util/boolptr" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" diff --git a/pkg/restore/actions/pvc_action.go b/pkg/restore/actions/pvc_action.go index a4a63374d..b9422d20f 100644 --- a/pkg/restore/actions/pvc_action.go +++ b/pkg/restore/actions/pvc_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/actions/rolebinding_action.go b/pkg/restore/actions/rolebinding_action.go index 05e463587..ff63f3022 100644 --- a/pkg/restore/actions/rolebinding_action.go +++ b/pkg/restore/actions/rolebinding_action.go @@ -17,7 +17,7 @@ limitations under the License. package actions import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/actions/secret_action.go b/pkg/restore/actions/secret_action.go index 2ec9fb4ff..6517045ca 100644 --- a/pkg/restore/actions/secret_action.go +++ b/pkg/restore/actions/secret_action.go @@ -21,7 +21,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/actions/service_account_action.go b/pkg/restore/actions/service_account_action.go index 429c21949..fb410d384 100644 --- a/pkg/restore/actions/service_account_action.go +++ b/pkg/restore/actions/service_account_action.go @@ -19,7 +19,7 @@ package actions import ( "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/actions/service_action.go b/pkg/restore/actions/service_action.go index 47147a31f..1dd712d9f 100644 --- a/pkg/restore/actions/service_action.go +++ b/pkg/restore/actions/service_action.go @@ -21,7 +21,7 @@ import ( "fmt" "strconv" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/merge_service_account.go b/pkg/restore/merge_service_account.go index 7abaa7ee2..6d6ea38cd 100644 --- a/pkg/restore/merge_service_account.go +++ b/pkg/restore/merge_service_account.go @@ -19,8 +19,8 @@ package restore import ( "encoding/json" + "github.com/cockroachdb/errors" jsonpatch "github.com/evanphx/json-patch/v5" - "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/pkg/restore/prioritize_group_version.go b/pkg/restore/prioritize_group_version.go index 5d7ab15d5..8801a3b0d 100644 --- a/pkg/restore/prioritize_group_version.go +++ b/pkg/restore/prioritize_group_version.go @@ -21,7 +21,7 @@ import ( "sort" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/restore/pv_restorer.go b/pkg/restore/pv_restorer.go index 53fbd0126..cc851f72e 100644 --- a/pkg/restore/pv_restorer.go +++ b/pkg/restore/pv_restorer.go @@ -19,7 +19,7 @@ package restore import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/restore/pv_restorer_test.go b/pkg/restore/pv_restorer_test.go index 09c6dd0ad..2f40a9a93 100644 --- a/pkg/restore/pv_restorer_test.go +++ b/pkg/restore/pv_restorer_test.go @@ -21,7 +21,7 @@ import ( "github.com/sirupsen/logrus" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" diff --git a/pkg/restore/request.go b/pkg/restore/request.go index 239d65df9..57ab6f119 100644 --- a/pkg/restore/request.go +++ b/pkg/restore/request.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/velero/internal/resourcemodifiers" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/itemoperation" @@ -61,6 +62,7 @@ type Request struct { RestoredItems map[itemKey]restoredItemStatus itemOperationsList *[]*itemoperation.RestoreOperation ResourceModifiers *resourcemodifiers.ResourceModifiers + ResPolicies *resourcepolicies.Policies DisableInformerCache bool CSIVolumeSnapshots []*snapshotv1api.VolumeSnapshot BackupVolumeInfoMap map[string]volume.BackupVolumeInfo diff --git a/pkg/restore/restore.go b/pkg/restore/restore.go index 5287eec21..a71fc4b23 100644 --- a/pkg/restore/restore.go +++ b/pkg/restore/restore.go @@ -31,9 +31,10 @@ import ( "sync" "time" + "github.com/cockroachdb/errors" + "github.com/gobwas/glob" "github.com/google/uuid" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -55,6 +56,7 @@ import ( "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/internal/hook" "github.com/vmware-tanzu/velero/internal/resourcemodifiers" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/archive" @@ -237,6 +239,43 @@ func (kr *kubernetesRestorer) RestoreWithResolvers( Includes(req.Restore.Spec.IncludedNamespaces...). Excludes(req.Restore.Spec.ExcludedNamespaces...) + var clusterScopedFilterMap map[string]*resolvedResourceFilter + var namespacedFilterMap map[string]*resolvedNamespaceFilter + var namespacedFilterPatterns []namespacedFilterPattern + + if req.ResPolicies != nil { + if kr.discoveryHelper == nil { + return results.Result{}, results.Result{Velero: []string{"failed to resolve namespace filter policies: discovery client unavailable"}} + } + + // Resolve clusterScopedFilterPolicy + csPolicy := req.ResPolicies.GetClusterScopedFilterPolicy() + if csPolicy != nil { + clusterScopedFilterMap, err = resolveRestoreClusterScopedFilterPolicy( + csPolicy, + kr.discoveryHelper, + req.Log, + ) + if err != nil { + return results.Result{}, results.Result{Velero: []string{err.Error()}} + } + } + + // Resolve namespacedFilterPolicies + nfPolicies := req.ResPolicies.GetNamespacedFilterPolicies() + if len(nfPolicies) > 0 { + namespacedFilterMap, namespacedFilterPatterns, err = resolveRestoreNamespacedFilterPolicies( + nfPolicies, + req.Restore.Spec.ExcludedResources, + kr.discoveryHelper, + req.Log, + ) + if err != nil { + return results.Result{}, results.Result{Velero: []string{err.Error()}} + } + } + } + resolvedActions, err := restoreItemActionResolver.ResolveActions(kr.discoveryHelper, kr.logger) if err != nil { return results.Result{}, results.Result{Velero: []string{err.Error()}} @@ -333,6 +372,10 @@ func (kr *kubernetesRestorer) RestoreWithResolvers( restoreVolumeInfoTracker: req.RestoreVolumeInfoTracker, hooksWaitExecutor: hooksWaitExecutor, resourceDeletionStatusTracker: req.ResourceDeletionStatusTracker, + clusterScopedFilterMap: clusterScopedFilterMap, + namespacedFilterMap: namespacedFilterMap, + namespacedFilterPatterns: namespacedFilterPatterns, + namespaceFilterCache: make(map[string]*resolvedNamespaceFilter), } return restoreCtx.execute() @@ -382,6 +425,247 @@ type restoreContext struct { restoreVolumeInfoTracker *volume.RestoreVolumeInfoTracker hooksWaitExecutor *hooksWaitExecutor resourceDeletionStatusTracker kube.ResourceDeletionStatusTracker + + // clusterScopedFilterMap holds resolved per-kind filters for cluster-scoped resources. + // Key is the resolved group-resource string. + clusterScopedFilterMap map[string]*resolvedResourceFilter + + // namespacedFilterMap holds resolved per-namespace filters. + // Key is either an exact namespace name or a glob pattern string. + namespacedFilterMap map[string]*resolvedNamespaceFilter + + // namespacedFilterPatterns preserves the order of patterns for first-match + // semantics and caches pre-compiled globs to avoid repeated compilation. + namespacedFilterPatterns []namespacedFilterPattern + + // namespaceFilterCache memoizes the resolved filter for a given namespace + // to avoid re-evaluating glob patterns on every call. + namespaceFilterCache map[string]*resolvedNamespaceFilter +} + +type resolvedResourceFilter struct { + labelSelector labels.Selector + orLabelSelectors []labels.Selector + nameIE *collections.IncludesExcludes + originalKinds []string +} + +type resolvedNamespaceFilter struct { + // resourceFilterMap is keyed by the resolved group-resource string + resourceFilterMap map[string]*resolvedResourceFilter + // catchAllFilter holds the resolved filter for a catch-all entry (empty kinds or ["*"]). + // nil when no catch-all entry is defined. + catchAllFilter *resolvedResourceFilter + // hasUnresolvedKinds is true if any kind in the policy failed discovery. + // This is used to bypass the fast-path skip so the peek-and-map fallback can run. + hasUnresolvedKinds bool +} + +// namespacedFilterPattern pairs a namespace pattern string with its pre-compiled +// glob so that getNamespaceFilter does not recompile on every call. +type namespacedFilterPattern struct { + pattern string + compiled glob.Glob // compiled once at restore start; nil for exact-match patterns +} + +func (ctx *restoreContext) getNamespaceFilter(namespace string) *resolvedNamespaceFilter { + if ctx.namespacedFilterMap == nil { + return nil + } + + // 1. Check the cache first + if filter, ok := ctx.namespaceFilterCache[namespace]; ok { + return filter + } + + // 2. Walk patterns in definition order (first-match semantics) + // Note: namespaceFilterCache is mutated below without synchronization. This is safe + // today because resource collection runs sequentially. If the restore loop is + // parallelized in the future, these map writes will need a lock to prevent data races. + for _, p := range ctx.namespacedFilterPatterns { + if p.compiled != nil { + if p.compiled.Match(namespace) { + filter := ctx.namespacedFilterMap[p.pattern] + ctx.namespaceFilterCache[namespace] = filter + return filter + } + } else if p.pattern == namespace { + filter := ctx.namespacedFilterMap[p.pattern] + ctx.namespaceFilterCache[namespace] = filter + return filter + } + } + + // 3. Cache the miss so we don't re-evaluate failed matches + ctx.namespaceFilterCache[namespace] = nil + return nil +} + +// resolveRestoreClusterScopedFilterPolicy resolves the cluster-scoped filter policy +// into a map keyed by group-resource string. Note: catch-all entries (empty or ["*"] kinds) +// are NOT supported in clusterScopedFilterPolicy — validation rejects them earlier. +// Cluster-scoped filtering is a refinement overlay; unlisted kinds fall back to global +// filters via the existing pipeline, so there is no catchAllFilter field on this map. +func resolveRestoreClusterScopedFilterPolicy( + policy *resourcepolicies.ClusterScopedFilterPolicy, + helper discovery.Helper, + log logrus.FieldLogger, +) (map[string]*resolvedResourceFilter, error) { + result := make(map[string]*resolvedResourceFilter) + for _, rf := range policy.ResourceFilters { + resolved, err := resolveResourceFilter(rf) + if err != nil { + return nil, err + } + for _, kind := range resolved.originalKinds { + gr, resource, err := helper.ResourceFor(schema.ParseGroupResource(kind).WithVersion("")) + + key := kind + if err != nil { + log.WithField("kind", kind).Warnf("Cannot resolve kind via discovery, using as-is") + } else { + if resource.Namespaced { + log.Warnf("kind %q in clusterScopedFilterPolicy is a namespace-scoped resource; it will never match in a cluster-scoped filter — did you mean namespacedFilterPolicies?", kind) + } + key = gr.GroupResource().String() + } + + if _, exists := result[key]; exists { + return nil, fmt.Errorf("ambiguous policy: duplicate kind %q detected", key) + } + + result[key] = resolved + } + } + return result, nil +} + +func resolveRestoreNamespacedFilterPolicies( + policies []resourcepolicies.NamespacedFilterPolicy, + excludedResources []string, + helper discovery.Helper, + log logrus.FieldLogger, +) (map[string]*resolvedNamespaceFilter, []namespacedFilterPattern, error) { + result := make(map[string]*resolvedNamespaceFilter) + var patternOrder []namespacedFilterPattern + + // Build a quick lookup map for globally excluded resources + globalExcludes := make(map[string]bool) + for _, ex := range excludedResources { + // We lowercase the excluded resources here because the kinds in the resource filters + // are lowercased during resolution, and we want to ensure case-insensitive matching. + globalExcludes[strings.ToLower(ex)] = true + } + + for _, policy := range policies { + rfMap := make(map[string]*resolvedResourceFilter) + var catchAll *resolvedResourceFilter + hasUnresolvedKinds := false + + for _, rf := range policy.ResourceFilters { + resolved, err := resolveResourceFilter(rf) + if err != nil { + return nil, nil, err + } + + if rf.IsCatchAll() { + catchAll = resolved + continue + } + + for _, kind := range resolved.originalKinds { + gr, resource, err := helper.ResourceFor( + schema.ParseGroupResource(kind).WithVersion(""), + ) + + key := kind + if err != nil { + log.WithField("kind", kind).Warnf( + "Cannot resolve kind via discovery, using as-is") + hasUnresolvedKinds = true + } else { + if !resource.Namespaced { + log.Warnf("kind %q in namespacedFilterPolicies is a cluster-scoped resource; it will never match in a namespace-scoped filter — did you mean clusterScopedFilterPolicy?", kind) + } + + if globalExcludes[kind] || globalExcludes[gr.GroupResource().String()] { + log.WithFields(logrus.Fields{ + "kind": kind, + "namespacePattern": strings.Join(policy.Namespaces, ","), + }).Warn("namespacedFilterPolicies entry lists a kind that is globally excluded by RestoreSpec.ExcludedResources; the per-namespace filter entry has no effect") + } + key = gr.GroupResource().String() + } + + if _, exists := rfMap[key]; exists { + return nil, nil, fmt.Errorf("ambiguous policy: duplicate kind %q detected", key) + } + + rfMap[key] = resolved + } + } + + nsFilter := &resolvedNamespaceFilter{ + resourceFilterMap: rfMap, + catchAllFilter: catchAll, + hasUnresolvedKinds: hasUnresolvedKinds, + } + for _, nsPattern := range policy.Namespaces { + result[nsPattern] = nsFilter + var compiled glob.Glob + if strings.ContainsAny(nsPattern, "*?[") { + var err error + compiled, err = glob.Compile(nsPattern) + if err != nil { + log.WithError(err).Warnf("Failed to compile namespace glob pattern %q, falling back to exact match", nsPattern) + } + } + patternOrder = append(patternOrder, namespacedFilterPattern{ + pattern: nsPattern, + compiled: compiled, + }) + } + } + return result, patternOrder, nil +} + +// resolveResourceFilter converts a ResourceFilter's label selectors and name patterns +// into their runtime representations. +func resolveResourceFilter( + rf resourcepolicies.ResourceFilter, +) (*resolvedResourceFilter, error) { + var selector labels.Selector + if len(rf.LabelSelector) > 0 { + var err error + selector, err = labels.ValidatedSelectorFromSet(labels.Set(rf.LabelSelector)) + if err != nil { + return nil, fmt.Errorf("invalid label selector in resource filter: %w", err) + } + } + var orSelectors []labels.Selector + for _, ols := range rf.OrLabelSelectors { + s, err := labels.ValidatedSelectorFromSet(labels.Set(ols)) + if err != nil { + return nil, fmt.Errorf("invalid OR label selector in resource filter: %w", err) + } + orSelectors = append(orSelectors, s) + } + var nameIE *collections.IncludesExcludes + if len(rf.Names) > 0 || len(rf.ExcludedNames) > 0 { + nameIE = collections.NewIncludesExcludes().Includes(rf.Names...).Excludes(rf.ExcludedNames...) + } + + normalizedKinds := make([]string, len(rf.Kinds)) + for i, k := range rf.Kinds { + normalizedKinds[i] = strings.ToLower(k) + } + + return &resolvedResourceFilter{ + labelSelector: selector, + orLabelSelectors: orSelectors, + nameIE: nameIE, + originalKinds: normalizedKinds, + }, nil } type resourceClientKey struct { @@ -467,7 +751,7 @@ func (ctx *restoreContext) execute() (results.Result, results.Result) { backupResources, err := archive.NewParser(ctx.log, ctx.fileSystem).Parse(ctx.restoreDir) // If ErrNotExist occurs, it implies that the backup to be restored includes zero items. // Need to add a warning about it and jump out of the function. - if errors.Cause(err) == archive.ErrNotExist { + if errors.Is(err, archive.ErrNotExist) { warnings.AddVeleroError(errors.Wrap(err, "zero items to be restored")) return warnings, errs } @@ -1128,6 +1412,10 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso // and should be excluded. Note that we're checking the object's namespace ( // via obj.GetNamespace()) instead of the namespace parameter, because we want // to check the *original* namespace, not the remapped one if it's been remapped. + // + // Note: Additional items intentionally bypass fine-grained resource filter policies + // (like per-namespace label/name selectors) to avoid breaking semantic dependencies, + // but they must still pass the global exclusions enforced below. if namespace != "" { if !ctx.namespaceIncludesExcludes.ShouldInclude(obj.GetNamespace()) && !ctx.resourceMustHave.Has(groupResource.String()) { restoreLogger.Info("Not restoring item because namespace is excluded") @@ -2059,6 +2347,9 @@ func hasPodVolumeBackup(unstructuredPV *unstructured.Unstructured, ctx *restoreC var found bool for _, pvb := range ctx.podVolumeBackups { + if pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseCompleted || pvb.Status.SnapshotID == "" { + continue + } if pvb.Spec.Pod.Namespace == pv.Spec.ClaimRef.Namespace && pvb.GetAnnotations()[configs.PVCNameAnnotation] == pv.Spec.ClaimRef.Name { found = true break @@ -2277,6 +2568,18 @@ func (ctx *restoreContext) getOrderedResourceCollection( continue } + // Per-namespace resource type check from restore filter policy + if namespace != "" && !ctx.resourceMustHave.Has(groupResource.String()) { + if nsFilter := ctx.getNamespaceFilter(namespace); nsFilter != nil { + _, kindListed := nsFilter.resourceFilterMap[groupResource.String()] + if !kindListed && nsFilter.catchAllFilter == nil && !nsFilter.hasUnresolvedKinds { + ctx.log.Infof("Skipping resource %s in namespace %s: not in resourceFilters", + resource, namespace) + continue + } + } + } + res, w, e := ctx.getSelectedRestoreableItems(groupResource.String(), namespace, items) warnings.Merge(&w) errs.Merge(&e) @@ -2330,6 +2633,88 @@ func (ctx *restoreContext) getSelectedRestoreableItems(resource string, original resourceForPath = filepath.Join(resource, cgv.Dir) } + var rf *resolvedResourceFilter + var useFilterPolicy bool + + if !ctx.resourceMustHave.Has(resource) { + if originalNamespace != "" { + // Namespace-scoped path + if nsFilter := ctx.getNamespaceFilter(originalNamespace); nsFilter != nil { + // Resolve effective filter: kind-specific takes precedence over catch-all + rf = nsFilter.resourceFilterMap[resource] + + // Peek-and-map logic for unresolvable kinds + if rf == nil && len(items) > 0 { + peekPath := archive.GetItemFilePath(ctx.restoreDir, resourceForPath, originalNamespace, items[0]) + // Ignore unmarshal errors during peek; the main restore loop will catch and report them + if obj, err := archive.Unmarshal(ctx.fileSystem, peekPath); err == nil { + actualKind := obj.GroupVersionKind().Kind + for _, filter := range nsFilter.resourceFilterMap { + for _, k := range filter.originalKinds { + if strings.EqualFold(k, actualKind) { + rf = filter + // Cache it for future lookups of this resource + // Note: resourceFilterMap is mutated in place without synchronization. + // This is safe today because resource collection runs sequentially. + // If parallelized in the future, this will need a lock to prevent data races. + nsFilter.resourceFilterMap[resource] = rf + break + } + } + if rf != nil { + break + } + } + } + } + + if rf == nil { + rf = nsFilter.catchAllFilter // may be nil if no catch-all + } + useFilterPolicy = true + + if rf == nil { + ctx.log.Infof("Skipping resource %s in namespace %s: not in resourceFilters", resource, originalNamespace) + return restorable, warnings, errs + } + } + } else if ctx.clusterScopedFilterMap != nil { + // Cluster-scoped path: only applies if kind is listed (refinement overlay) + if listedRF, ok := ctx.clusterScopedFilterMap[resource]; ok { + rf = listedRF + useFilterPolicy = true + } else if len(items) > 0 { + // Peek-and-map logic for unresolvable kinds. + // Note: Unlike the namespaced path, this fallback is always reachable + // because the main restore loop does not have a fast-path skip for + // unlisted cluster-scoped resources. + peekPath := archive.GetItemFilePath(ctx.restoreDir, resourceForPath, originalNamespace, items[0]) + // Ignore unmarshal errors during peek; the main restore loop will catch and report them + if obj, err := archive.Unmarshal(ctx.fileSystem, peekPath); err == nil { + actualKind := obj.GroupVersionKind().Kind + for _, filter := range ctx.clusterScopedFilterMap { + for _, k := range filter.originalKinds { + if strings.EqualFold(k, actualKind) { + rf = filter + // Cache it for future lookups of this resource + // Note: clusterScopedFilterMap is mutated in place without synchronization. + // This is safe today because resource collection runs sequentially. + // If parallelized in the future, this will need a lock to prevent data races. + ctx.clusterScopedFilterMap[resource] = rf + useFilterPolicy = true + break + } + } + if rf != nil { + break + } + } + } + } + // If kind not listed, fall through to global selectors below + } + } + for _, item := range items { itemPath := archive.GetItemFilePath(ctx.restoreDir, resourceForPath, originalNamespace, item) @@ -2347,29 +2732,58 @@ func (ctx *restoreContext) getSelectedRestoreableItems(resource string, original } if !ctx.resourceMustHave.Has(resource) { - if !ctx.selector.Matches(labels.Set(obj.GetLabels())) { - continue - } - - // Processing OrLabelSelectors when specified in the restore request. LabelSelectors as well as OrLabelSelectors - // cannot co-exist, only one of them can be specified - var skipItem = false - var skip = 0 - ctx.log.Debugf("orSelectors specified: %s for item: %s", ctx.OrSelectors, item) - for _, s := range ctx.OrSelectors { - if !s.Matches(labels.Set(obj.GetLabels())) { - skip++ + if useFilterPolicy { + if rf != nil { + // Per-kind label selector + if rf.labelSelector != nil && !rf.labelSelector.Matches(labels.Set(obj.GetLabels())) { + continue + } + // Per-kind OR label selectors + if len(rf.orLabelSelectors) > 0 { + matched := false + for _, s := range rf.orLabelSelectors { + if s.Matches(labels.Set(obj.GetLabels())) { + matched = true + break + } + } + if !matched { + ctx.log.Infof("Excluding item %s: no OR label selector matched (restore filter policy)", item) + continue + } + } + // Per-kind name filter + if rf.nameIE != nil && !rf.nameIE.ShouldInclude(obj.GetName()) { + ctx.log.Infof("Excluding item %s: name does not match restore filter policy", obj.GetName()) + continue + } + } + } else { + // Existing global selector logic + if !ctx.selector.Matches(labels.Set(obj.GetLabels())) { + continue } - if len(ctx.OrSelectors) == skip && skip > 0 { - ctx.log.Infof("setting skip flag to true for item: %s", item) - skipItem = true - } - } + // Processing OrLabelSelectors when specified in the restore request. LabelSelectors as well as OrLabelSelectors + // cannot co-exist, only one of them can be specified + var skipItem = false + var skip = 0 + ctx.log.Debugf("orSelectors specified: %s for item: %s", ctx.OrSelectors, item) + for _, s := range ctx.OrSelectors { + if !s.Matches(labels.Set(obj.GetLabels())) { + skip++ + } - if skipItem { - ctx.log.Infof("restore orSelector labels did not match, skipping restore of item: %s", skipItem, item) - continue + if len(ctx.OrSelectors) == skip && skip > 0 { + ctx.log.Infof("setting skip flag to true for item: %s", item) + skipItem = true + } + } + + if skipItem { + ctx.log.Infof("restore orSelector labels did not match, skipping restore of item: %s", skipItem, item) + continue + } } } diff --git a/pkg/restore/restore_policies_test.go b/pkg/restore/restore_policies_test.go new file mode 100644 index 000000000..42b8fb11f --- /dev/null +++ b/pkg/restore/restore_policies_test.go @@ -0,0 +1,332 @@ +package restore + +import ( + "io" + "strings" + "testing" + + "github.com/sirupsen/logrus" + logrustest "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/require" + corev1api "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/vmware-tanzu/velero/internal/resourcepolicies" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" + "github.com/vmware-tanzu/velero/pkg/test" +) + +func TestRestoreResourcePoliciesFiltering(t *testing.T) { + customKindRes := &test.APIResource{Group: "mygroup.io", Version: "v1", Name: "mycustomkinds", Kind: "MyCustomKind", Namespaced: true} + clusterCustomKindRes := &test.APIResource{Group: "mygroup.io", Version: "v1", Name: "myclustercustomkinds", Kind: "MyClusterCustomKind", Namespaced: false} + + tests := []struct { + name string + restore *velerov1api.Restore + backup *velerov1api.Backup + policyYAML string + apiResources []*test.APIResource + tarball io.Reader + want map[*test.APIResource][]string + }{ + { + name: "namespaced filter policy with exact namespace match", + restore: defaultRestore().Result(), + backup: defaultBackup().Result(), + policyYAML: `version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-1 + resourceFilters: + - kinds: + - pods + names: + - pod-1 +`, + tarball: test.NewTarWriter(t). + AddItems("pods", + builder.ForPod("ns-1", "pod-1").Result(), + builder.ForPod("ns-1", "pod-2").Result(), + builder.ForPod("ns-2", "pod-1").Result(), + ). + Done(), + apiResources: []*test.APIResource{ + test.Pods(), + }, + want: map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-1", "ns-2/pod-1"}, // ns-2 is not filtered, ns-1 only includes pod-1 + }, + }, + { + name: "namespaced filter policy with glob namespace match and first-match semantics", + restore: defaultRestore().Result(), + backup: defaultBackup().Result(), + policyYAML: `version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-* + resourceFilters: + - kinds: + - pods + names: + - pod-1 + - namespaces: + - ns-1 + resourceFilters: + - kinds: + - pods + names: + - pod-2 +`, + tarball: test.NewTarWriter(t). + AddItems("pods", + builder.ForPod("ns-1", "pod-1").Result(), + builder.ForPod("ns-1", "pod-2").Result(), + builder.ForPod("ns-2", "pod-1").Result(), + builder.ForPod("ns-2", "pod-2").Result(), + ). + Done(), + apiResources: []*test.APIResource{ + test.Pods(), + }, + want: map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-1", "ns-2/pod-1"}, + }, + }, + { + name: "cluster scoped filter policy", + restore: defaultRestore().Result(), + backup: defaultBackup().Result(), + policyYAML: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: + - persistentvolumes + names: + - pv-1 +`, + tarball: test.NewTarWriter(t). + AddItems("persistentvolumes", + builder.ForPersistentVolume("pv-1").Result(), + builder.ForPersistentVolume("pv-2").Result(), + ). + Done(), + apiResources: []*test.APIResource{ + test.PVs(), + }, + want: map[*test.APIResource][]string{ + test.PVs(): {"/pv-1"}, + }, + }, + { + name: "catch-all filter", + restore: defaultRestore().Result(), + backup: defaultBackup().Result(), + policyYAML: `version: v1 +namespacedFilterPolicies: + - namespaces: + - ns-1 + resourceFilters: + - kinds: + - '*' + labelSelector: + app: test +`, + tarball: test.NewTarWriter(t). + AddItems("pods", + builder.ForPod("ns-1", "pod-1").ObjectMeta(builder.WithLabels("app", "test")).Result(), + builder.ForPod("ns-1", "pod-2").Result(), + ). + AddItems("deployments.apps", + builder.ForDeployment("ns-1", "deploy-1").ObjectMeta(builder.WithLabels("app", "test")).Result(), + builder.ForDeployment("ns-1", "deploy-2").Result(), + ). + Done(), + apiResources: []*test.APIResource{ + test.Pods(), + test.Deployments(), + }, + want: map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-1"}, + test.Deployments(): {"ns-1/deploy-1"}, + }, + }, + { + name: "unresolved kind in namespaced filter policy is still restored via peek-and-map", + restore: defaultRestore().Result(), + backup: defaultBackup().Result(), + policyYAML: `version: v1 +namespacedFilterPolicies: + - namespaces: ["ns-1"] + resourceFilters: + - kinds: ["MyCustomKind"] +`, + tarball: test.NewTarWriter(t).AddItems("mycustomkinds.mygroup.io", + &unstructured.Unstructured{Object: map[string]any{"apiVersion": "mygroup.io/v1", "kind": "MyCustomKind", "metadata": map[string]any{"namespace": "ns-1", "name": "my-cr"}}}, + ).Done(), + apiResources: []*test.APIResource{ + customKindRes, + }, + want: map[*test.APIResource][]string{ + customKindRes: {"ns-1/my-cr"}, + }, + }, + { + name: "unresolved kind in cluster-scoped filter policy is still restored via peek-and-map", + restore: defaultRestore().Result(), + backup: defaultBackup().Result(), + policyYAML: `version: v1 +clusterScopedFilterPolicy: + resourceFilters: + - kinds: ["MyClusterCustomKind"] +`, + tarball: test.NewTarWriter(t).AddItems("myclustercustomkinds.mygroup.io", + &unstructured.Unstructured{Object: map[string]any{"apiVersion": "mygroup.io/v1", "kind": "MyClusterCustomKind", "metadata": map[string]any{"name": "my-cluster-cr"}}}, + ).Done(), + apiResources: []*test.APIResource{ + clusterCustomKindRes, + }, + want: map[*test.APIResource][]string{ + clusterCustomKindRes: {"/my-cluster-cr"}, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t) + + for _, r := range tc.apiResources { + h.DiscoveryClient.WithAPIResource(r) + } + require.NoError(t, h.restorer.discoveryHelper.Refresh()) + + var resPolicies *resourcepolicies.Policies + if tc.policyYAML != "" { + cm := &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-policies", + Namespace: "velero", + }, + Data: map[string]string{ + "policy.yaml": tc.policyYAML, + }, + } + client := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(cm).Build() + restore := tc.restore.DeepCopy() + restore.Namespace = "velero" + restore.Spec.ResourcePolicy = &corev1api.TypedLocalObjectReference{ + Kind: "configmap", + Name: "test-policies", + } + var err error + resPolicies, err = resourcepolicies.GetResourcePoliciesFromRestore(t.Context(), restore, client, logrus.New()) + require.NoError(t, err) + } + + data := &Request{ + Log: h.log, + Restore: tc.restore, + Backup: tc.backup, + PodVolumeBackups: nil, + VolumeSnapshots: nil, + BackupReader: tc.tarball, + ResPolicies: resPolicies, + } + warnings, errs := h.restorer.Restore( + data, + nil, // restoreItemActions + nil, // volume snapshotter getter + ) + + assertEmptyResults(t, warnings, errs) + assertAPIContents(t, h, tc.want) + }) + } +} + +func TestResolveRestoreNamespacedFilterPolicies_Validation(t *testing.T) { + log := logrus.New() + helper := test.NewFakeDiscoveryHelper(true, nil) + + policies := []resourcepolicies.NamespacedFilterPolicy{ + { + Namespaces: []string{"ns-1"}, + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"MyKind", "mykind"}, + }, + }, + }, + } + + _, _, err := resolveRestoreNamespacedFilterPolicies(policies, nil, helper, log) + require.Error(t, err) + require.Contains(t, err.Error(), "ambiguous policy: duplicate kind") +} + +func TestResolveRestoreClusterScopedFilterPolicy_Validation(t *testing.T) { + log := logrus.New() + helper := test.NewFakeDiscoveryHelper(true, nil) + + policy := &resourcepolicies.ClusterScopedFilterPolicy{ + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"MyKind", "mykind"}, + }, + }, + } + + _, err := resolveRestoreClusterScopedFilterPolicy(policy, helper, log) + require.Error(t, err) + require.Contains(t, err.Error(), "ambiguous policy: duplicate kind") +} + +func TestResolveRestoreNamespacedFilterPolicies_GlobalExcludesWarning(t *testing.T) { + log, hook := logrustest.NewNullLogger() + helper := test.NewFakeDiscoveryHelper(true, nil) + + policies := []resourcepolicies.NamespacedFilterPolicy{ + { + Namespaces: []string{"ns-1"}, + ResourceFilters: []resourcepolicies.ResourceFilter{ + { + Kinds: []string{"ConfigMaps"}, + }, + }, + }, + } + + excludedResources := []string{"ConfigMaps"} // Same case + _, _, err := resolveRestoreNamespacedFilterPolicies(policies, excludedResources, helper, log) + require.NoError(t, err) + + // Check if a warning was emitted + found := false + for _, entry := range hook.Entries { + if entry.Level == logrus.WarnLevel && strings.Contains(entry.Message, "namespacedFilterPolicies entry lists a kind that is globally excluded") { + found = true + break + } + } + require.True(t, found, "expected warning about globally excluded resource") + + hook.Reset() + + excludedResourcesDiffCase := []string{"configmaps"} // Different case + _, _, err = resolveRestoreNamespacedFilterPolicies(policies, excludedResourcesDiffCase, helper, log) + require.NoError(t, err) + + found = false + for _, entry := range hook.Entries { + if entry.Level == logrus.WarnLevel && strings.Contains(entry.Message, "namespacedFilterPolicies entry lists a kind that is globally excluded") { + found = true + break + } + } + require.True(t, found, "expected warning about globally excluded resource even if case differs") +} diff --git a/pkg/restore/restore_test.go b/pkg/restore/restore_test.go index 95b283cbe..fc4051387 100644 --- a/pkg/restore/restore_test.go +++ b/pkg/restore/restore_test.go @@ -25,11 +25,12 @@ import ( "testing" "time" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/collections" + "github.com/cockroachdb/errors" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -764,6 +765,10 @@ func TestRestoreResourceFiltering(t *testing.T) { } require.NoError(t, h.restorer.discoveryHelper.Refresh()) + // We need to fetch the policies using the actual function + resPolicies, err := resourcepolicies.GetResourcePoliciesFromRestore(t.Context(), tc.restore, h.restorer.kbClient, h.log) + require.NoError(t, err) + data := &Request{ Log: h.log, Restore: tc.restore, @@ -771,6 +776,7 @@ func TestRestoreResourceFiltering(t *testing.T) { PodVolumeBackups: nil, VolumeSnapshots: nil, BackupReader: tc.tarball, + ResPolicies: resPolicies, } warnings, errs := h.restorer.Restore( data, @@ -4240,3 +4246,84 @@ func TestDetermineRestoreStatus(t *testing.T) { }) } } + +func TestHasPodVolumeBackup(t *testing.T) { + pvUnstructured := func() *unstructured.Unstructured { + pv := &corev1api.PersistentVolume{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "PersistentVolume"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pv", + }, + Spec: corev1api.PersistentVolumeSpec{ + ClaimRef: &corev1api.ObjectReference{ + Namespace: "test-ns", + Name: "test-pvc", + }, + }, + } + obj, _ := runtime.DefaultUnstructuredConverter.ToUnstructured(pv) + return &unstructured.Unstructured{Object: obj} + } + + makePVB := func(phase velerov1api.PodVolumeBackupPhase, snapshotID string) *velerov1api.PodVolumeBackup { + return &velerov1api.PodVolumeBackup{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "velero.io/pvc-name": "test-pvc", + }, + }, + Spec: velerov1api.PodVolumeBackupSpec{ + Pod: corev1api.ObjectReference{ + Namespace: "test-ns", + }, + }, + Status: velerov1api.PodVolumeBackupStatus{ + Phase: phase, + SnapshotID: snapshotID, + }, + } + } + + tests := []struct { + name string + pvbs []*velerov1api.PodVolumeBackup + expected bool + }{ + { + name: "no pvbs", + pvbs: nil, + expected: false, + }, + { + name: "completed pvb with snapshot ID", + pvbs: []*velerov1api.PodVolumeBackup{makePVB(velerov1api.PodVolumeBackupPhaseCompleted, "snap-123")}, + expected: true, + }, + { + name: "in-progress pvb should not match", + pvbs: []*velerov1api.PodVolumeBackup{makePVB(velerov1api.PodVolumeBackupPhaseInProgress, "")}, + expected: false, + }, + { + name: "completed pvb with empty snapshot ID should not match", + pvbs: []*velerov1api.PodVolumeBackup{makePVB(velerov1api.PodVolumeBackupPhaseCompleted, "")}, + expected: false, + }, + { + name: "failed pvb should not match", + pvbs: []*velerov1api.PodVolumeBackup{makePVB(velerov1api.PodVolumeBackupPhaseFailed, "")}, + expected: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := &restoreContext{ + podVolumeBackups: tc.pvbs, + log: logrus.New(), + } + result := hasPodVolumeBackup(pvUnstructured(), ctx) + assert.Equal(t, tc.expected, result) + }) + } +} diff --git a/pkg/test/api_server.go b/pkg/test/api_server.go index dd5b0a07a..63975014a 100644 --- a/pkg/test/api_server.go +++ b/pkg/test/api_server.go @@ -56,6 +56,8 @@ func NewAPIServer(t *testing.T) *APIServer { {Group: "extensions", Version: "v1", Resource: "deployments"}: "ExtDeploymentsList", {Group: "velero.io", Version: "v1", Resource: "deployments"}: "VeleroDeploymentsList", {Group: "velero.io", Version: "v2alpha1", Resource: "datauploads"}: "DataUploadsList", + {Group: "mygroup.io", Version: "v1", Resource: "mycustomkinds"}: "MyCustomKindList", + {Group: "mygroup.io", Version: "v1", Resource: "myclustercustomkinds"}: "MyClusterCustomKindList", }) discoveryClient = &DiscoveryClient{FakeDiscovery: kubeClient.Discovery().(*discoveryfake.FakeDiscovery)} ) diff --git a/pkg/test/fake_mapper.go b/pkg/test/fake_mapper.go index 1686af815..529d989d5 100644 --- a/pkg/test/fake_mapper.go +++ b/pkg/test/fake_mapper.go @@ -17,7 +17,7 @@ limitations under the License. package test import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/uploader/block/dev_linux.go b/pkg/uploader/block/dev_linux.go new file mode 100644 index 000000000..297815390 --- /dev/null +++ b/pkg/uploader/block/dev_linux.go @@ -0,0 +1,78 @@ +//go:build linux +// +build linux + +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package block + +import ( + "os" + "path/filepath" + "syscall" + + "github.com/cockroachdb/errors" +) + +var lstatFunc = os.Lstat +var openFileFunc = os.OpenFile + +// openBlockDevice opens a block device for read/write, caller needs to close the returned handle +func openBlockDevice(path string, read bool) (*os.File, error) { + devPath, err := resolveSymlink(path) + if err != nil { + return nil, errors.Wrap(err, "resolveSymlink") + } + + fileInfo, err := lstatFunc(devPath) + if err != nil { + return nil, errors.Wrapf(err, "unable to get the device information %s", devPath) + } + + if (fileInfo.Sys().(*syscall.Stat_t).Mode & syscall.S_IFMT) != syscall.S_IFBLK { + return nil, errors.Errorf("path %s is not a block device", devPath) + } + + flag := os.O_RDWR + mode := os.FileMode(0666) + if read { + flag = os.O_RDONLY + mode = 0 + } + + device, err := openFileFunc(devPath, flag|syscall.O_DIRECT, mode) + if err != nil { + if os.IsPermission(err) || errors.Is(err, syscall.EPERM) { + return nil, errors.Wrapf(err, "no permission to open device %s with mode %v", devPath, mode) + } + return nil, errors.Wrapf(err, "unable to open device %s", devPath) + } + + return device, nil +} + +func resolveSymlink(path string) (string, error) { + st, err := os.Lstat(path) + if err != nil { + return "", errors.Wrap(err, "stat") + } + + if (st.Mode() & os.ModeSymlink) == 0 { + return path, nil + } + + return filepath.EvalSymlinks(path) +} diff --git a/pkg/uploader/block/dev_linux_test.go b/pkg/uploader/block/dev_linux_test.go new file mode 100644 index 000000000..42f0dd83e --- /dev/null +++ b/pkg/uploader/block/dev_linux_test.go @@ -0,0 +1,358 @@ +//go:build linux +// +build linux + +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package block + +import ( + "os" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeBlockDevFileInfo struct{} + +func (fakeBlockDevFileInfo) Name() string { return "fake-blk" } +func (fakeBlockDevFileInfo) Size() int64 { return 0 } +func (fakeBlockDevFileInfo) Mode() os.FileMode { return os.ModeDevice } +func (fakeBlockDevFileInfo) ModTime() time.Time { return time.Time{} } +func (fakeBlockDevFileInfo) IsDir() bool { return false } +func (fakeBlockDevFileInfo) Sys() any { + return &syscall.Stat_t{Mode: syscall.S_IFBLK} +} + +func TestResolveSymlink(t *testing.T) { + testCases := []struct { + name string + setupPath func(t *testing.T) string + expectError bool + errContains string + checkResult func(t *testing.T, input, result string) + }{ + { + name: "path does not exist returns error", + setupPath: func(t *testing.T) string { + t.Helper() + return filepath.Join(t.TempDir(), "nonexistent") + }, + expectError: true, + errContains: "stat", + }, + { + name: "regular file returns same path", + setupPath: func(t *testing.T) string { + t.Helper() + f, err := os.CreateTemp(t.TempDir(), "regular-*") + require.NoError(t, err) + f.Close() + return f.Name() + }, + checkResult: func(t *testing.T, input, result string) { + t.Helper() + assert.Equal(t, input, result) + }, + }, + { + name: "directory returns same path", + setupPath: func(t *testing.T) string { + t.Helper() + return t.TempDir() + }, + checkResult: func(t *testing.T, input, result string) { + t.Helper() + assert.Equal(t, input, result) + }, + }, + { + name: "symlink to existing file returns target real path", + setupPath: func(t *testing.T) string { + t.Helper() + dir := t.TempDir() + target, err := os.CreateTemp(dir, "target-*") + require.NoError(t, err) + target.Close() + linkPath := filepath.Join(dir, "link") + require.NoError(t, os.Symlink(target.Name(), linkPath)) + return linkPath + }, + checkResult: func(t *testing.T, input, result string) { + t.Helper() + assert.NotEqual(t, input, result) + fi, err := os.Lstat(result) + require.NoError(t, err) + assert.Zero(t, fi.Mode()&os.ModeSymlink) + }, + }, + { + name: "symlink to existing directory returns resolved path", + setupPath: func(t *testing.T) string { + t.Helper() + outer := t.TempDir() + inner := t.TempDir() + linkPath := filepath.Join(outer, "dirlink") + require.NoError(t, os.Symlink(inner, linkPath)) + return linkPath + }, + checkResult: func(t *testing.T, input, result string) { + t.Helper() + assert.NotEqual(t, input, result) + fi, err := os.Lstat(result) + require.NoError(t, err) + assert.True(t, fi.IsDir()) + }, + }, + { + name: "broken symlink — target does not exist — returns error", + setupPath: func(t *testing.T) string { + t.Helper() + dir := t.TempDir() + linkPath := filepath.Join(dir, "broken-link") + require.NoError(t, os.Symlink(filepath.Join(dir, "nonexistent-target"), linkPath)) + return linkPath + }, + expectError: true, + errContains: "no such file or directory", + }, + { + name: "chain of symlinks is fully resolved", + setupPath: func(t *testing.T) string { + t.Helper() + dir := t.TempDir() + // real → link1 → link2 (two-hop chain) + real, err := os.CreateTemp(dir, "real-*") + require.NoError(t, err) + real.Close() + link1 := filepath.Join(dir, "link1") + require.NoError(t, os.Symlink(real.Name(), link1)) + link2 := filepath.Join(dir, "link2") + require.NoError(t, os.Symlink(link1, link2)) + return link2 + }, + checkResult: func(t *testing.T, input, result string) { + t.Helper() + assert.NotEqual(t, input, result) + fi, err := os.Lstat(result) + require.NoError(t, err) + assert.Zero(t, fi.Mode()&os.ModeSymlink) + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + input := tc.setupPath(t) + result, err := resolveSymlink(input) + + if tc.expectError { + require.Error(t, err) + if tc.errContains != "" { + require.ErrorContains(t, err, tc.errContains) + } + assert.Empty(t, result) + } else { + require.NoError(t, err) + if tc.checkResult != nil { + tc.checkResult(t, input, result) + } + } + }) + } +} + +func TestOpenBlockDevice(t *testing.T) { + testCases := []struct { + name string + setupPath func(t *testing.T) string + read bool + expectError bool + errContains string + injectLstat func(string) (os.FileInfo, error) + injectOpenFile func(string, int, os.FileMode) (*os.File, error) + }{ + { + name: "path does not exist — resolveSymlink fails", + setupPath: func(t *testing.T) string { + t.Helper() + return filepath.Join(t.TempDir(), "nonexistent") + }, + read: true, + expectError: true, + errContains: "resolveSymlink", + }, + { + name: "regular file is not a block device — read mode", + setupPath: func(t *testing.T) string { + t.Helper() + f, err := os.CreateTemp(t.TempDir(), "regular-*") + require.NoError(t, err) + f.Close() + return f.Name() + }, + read: true, + expectError: true, + errContains: "is not a block device", + }, + { + name: "regular file is not a block device — write mode", + setupPath: func(t *testing.T) string { + t.Helper() + f, err := os.CreateTemp(t.TempDir(), "regular-*") + require.NoError(t, err) + f.Close() + return f.Name() + }, + read: false, + expectError: true, + errContains: "is not a block device", + }, + { + name: "directory is not a block device", + setupPath: func(t *testing.T) string { + t.Helper() + return t.TempDir() + }, + read: true, + expectError: true, + errContains: "is not a block device", + }, + { + name: "symlink to regular file is not a block device", + setupPath: func(t *testing.T) string { + t.Helper() + dir := t.TempDir() + target, err := os.CreateTemp(dir, "target-*") + require.NoError(t, err) + target.Close() + linkPath := filepath.Join(dir, "link") + require.NoError(t, os.Symlink(target.Name(), linkPath)) + return linkPath + }, + read: true, + expectError: true, + errContains: "is not a block device", + }, + { + name: "broken symlink — resolveSymlink fails", + setupPath: func(t *testing.T) string { + t.Helper() + dir := t.TempDir() + linkPath := filepath.Join(dir, "broken-link") + require.NoError(t, os.Symlink(filepath.Join(dir, "ghost"), linkPath)) + return linkPath + }, + read: true, + expectError: true, + errContains: "resolveSymlink", + }, + { + name: "EACCES from OpenFile — permission denied message", + setupPath: func(t *testing.T) string { + t.Helper() + f, err := os.CreateTemp(t.TempDir(), "blk-*") + require.NoError(t, err) + f.Close() + return f.Name() + }, + read: true, + expectError: true, + errContains: "no permission to open device", + injectLstat: func(_ string) (os.FileInfo, error) { + return fakeBlockDevFileInfo{}, nil + }, + injectOpenFile: func(name string, _ int, _ os.FileMode) (*os.File, error) { + t.Helper() + return nil, &os.PathError{Op: "open", Path: name, Err: syscall.EACCES} + }, + }, + { + name: "EPERM from OpenFile — permission denied message", + setupPath: func(t *testing.T) string { + t.Helper() + f, err := os.CreateTemp(t.TempDir(), "blk-*") + require.NoError(t, err) + f.Close() + return f.Name() + }, + read: false, + expectError: true, + errContains: "no permission to open device", + injectLstat: func(_ string) (os.FileInfo, error) { + return fakeBlockDevFileInfo{}, nil + }, + injectOpenFile: func(name string, _ int, _ os.FileMode) (*os.File, error) { + return nil, &os.PathError{Op: "open", Path: name, Err: syscall.EPERM} + }, + }, + { + name: "generic OpenFile error — unable to open device message", + setupPath: func(t *testing.T) string { + t.Helper() + f, err := os.CreateTemp(t.TempDir(), "blk-*") + require.NoError(t, err) + f.Close() + return f.Name() + }, + read: true, + expectError: true, + errContains: "unable to open device", + injectLstat: func(_ string) (os.FileInfo, error) { + t.Helper() + return fakeBlockDevFileInfo{}, nil + }, + injectOpenFile: func(name string, _ int, _ os.FileMode) (*os.File, error) { + t.Helper() + return nil, &os.PathError{Op: "open", Path: name, Err: syscall.EIO} + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Cleanup(func() { + lstatFunc = os.Lstat + openFileFunc = os.OpenFile + }) + if tc.injectLstat != nil { + lstatFunc = tc.injectLstat + } + if tc.injectOpenFile != nil { + openFileFunc = tc.injectOpenFile + } + + path := tc.setupPath(t) + f, err := openBlockDevice(path, tc.read) + + if tc.expectError { + require.Error(t, err) + if tc.errContains != "" { + require.ErrorContains(t, err, tc.errContains) + } + assert.Nil(t, f) + } else { + require.NoError(t, err) + require.NotNil(t, f) + f.Close() + } + }) + } +} diff --git a/pkg/uploader/block/dev_other.go b/pkg/uploader/block/dev_other.go new file mode 100644 index 000000000..c8a55cab2 --- /dev/null +++ b/pkg/uploader/block/dev_other.go @@ -0,0 +1,29 @@ +//go:build !linux +// +build !linux + +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package block + +import ( + "fmt" + "os" +) + +func openBlockDevice(_ string, _ bool) (*os.File, error) { + return nil, fmt.Errorf("block mode is not supported for non-linux platforms") +} diff --git a/pkg/uploader/block/snapshot.go b/pkg/uploader/block/snapshot.go new file mode 100644 index 000000000..e30f5c1bb --- /dev/null +++ b/pkg/uploader/block/snapshot.go @@ -0,0 +1,276 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package block + +import ( + "context" + "io" + "maps" + "path/filepath" + "time" + + "github.com/cockroachdb/errors" + "github.com/sirupsen/logrus" + + "github.com/vmware-tanzu/velero/pkg/cbtservice" + "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" + "github.com/vmware-tanzu/velero/pkg/uploader" + "github.com/vmware-tanzu/velero/pkg/uploader/cbt" +) + +var openBlockDeviceFunc = openBlockDevice + +type parentBackupInfo struct { + parentObject udmrepo.ID + changeID string + volumeID string +} + +// Backup backup specific sourcePath and update progress +func Backup(ctx context.Context, blkUp Uploader, repoWriter udmrepo.BackupRepo, sourcePath string, realSource string, cbtSource cbtservice.SourceInfo, + forceFull bool, parentSnapshot string, cbtService cbtservice.Service, uploaderCfg map[string]string, tags map[string]string, log logrus.FieldLogger) (uploader.SnapshotInfo, bool, error) { + if blkUp == nil { + return uploader.SnapshotInfo{}, false, errors.New("get empty block uploader") + } + + source, err := filepath.Abs(sourcePath) + if err != nil { + return uploader.SnapshotInfo{}, false, errors.Wrapf(err, "invalid source path %s", sourcePath) + } + + source = filepath.Clean(source) + + sourceInfo := sourceInfo{ + realSource: filepath.Clean(realSource), + } + + if realSource == "" { + sourceInfo.realSource = source + } + + sourceInfo.dev, err = openBlockDeviceFunc(source, true) + if err != nil { + return uploader.SnapshotInfo{}, false, errors.Wrapf(err, "error opening block device %s", source) + } + + defer sourceInfo.dev.Close() + + sourceInfo.size, err = sourceInfo.dev.Seek(0, io.SeekEnd) + if err != nil { + return uploader.SnapshotInfo{}, false, errors.Wrapf(err, "error getting length of block device %s", source) + } + + _, err = sourceInfo.dev.Seek(0, io.SeekStart) + if err != nil { + return uploader.SnapshotInfo{}, false, errors.Wrapf(err, "error reset pos of block device %s", source) + } + + snapID, backupSize, err := snapshotSource(ctx, repoWriter, blkUp, sourceInfo, forceFull, parentSnapshot, cbtSource, cbtService, tags, uploaderCfg, log, "Block Uploader") + snapshotInfo := uploader.SnapshotInfo{ + ID: snapID, + Size: sourceInfo.size, + IncrementalSize: backupSize, + } + + return snapshotInfo, false, err +} + +func snapshotSource( + ctx context.Context, + rep udmrepo.BackupRepo, + u Uploader, + source sourceInfo, + forceFull bool, + parentSnapshot string, + cbtSource cbtservice.SourceInfo, + cbtService cbtservice.Service, + snapshotTags map[string]string, + uploaderCfg map[string]string, + log logrus.FieldLogger, + description string, +) (string, int64, error) { + log.Info("Start to snapshot...") + snapshotStartTime := time.Now() + + parentBackup := getParentBackupInfo(ctx, rep, forceFull, parentSnapshot, cbtSource.VolumeID, source.realSource, snapshotTags, log) + + bitmap := cbt.NewBitmap(blockSize, uint64(source.size), cbtSource.Snapshot, parentBackup.changeID, parentBackup.volumeID) + + err := cbt.SetBitmapOrFull(ctx, cbtService, bitmap) + if err != nil { + parentBackup.parentObject = "" + log.WithError(err).Warnf("Failed to create CBT with source %v, fallback to real full backup", cbtSource) + } + + snap, backupSize, err := u.Backup(source, parentBackup.parentObject, bitmap.Iterator(), uploaderCfg) + if err != nil { + return "", 0, errors.Wrapf(err, "Failed to run uploader backup for si %v", source) + } + + snap.Tags = make(map[string]string) + snap.Tags[uploader.CBTChangeIDTag] = cbtSource.ChangeID + snap.Tags[uploader.CBTVolumeIDTag] = cbtSource.VolumeID + if snapshotTags != nil { + maps.Copy(snap.Tags, snapshotTags) + } + + snap.Description = description + + snapID, err := rep.SaveSnapshot(ctx, snap) + if err != nil { + return "", 0, errors.Wrapf(err, "Failed to save snapshot %v", snap) + } + + if err = rep.Flush(ctx); err != nil { + return "", 0, errors.Wrapf(err, "Failed to flush repository") + } + + log.Infof("Created snapshot with root %v and ID %v in %v", snap.RootObject, snapID, time.Since(snapshotStartTime).Truncate(time.Second)) + + return string(snapID), backupSize, nil +} + +func getParentBackupInfo(ctx context.Context, rep udmrepo.BackupRepo, forceFull bool, parentSnapshot string, volumeID string, realSource string, snapshotTags map[string]string, log logrus.FieldLogger) parentBackupInfo { + var previous *udmrepo.Snapshot + if !forceFull { + if parentSnapshot != "" { + snap, err := rep.GetSnapshot(ctx, udmrepo.ID(parentSnapshot)) + if err != nil { + log.WithError(err).Warn("Failed to load previous snapshot, fallback to full backup") + } else { + previous = &snap + log.Infof("Using provided parent snapshot %s", parentSnapshot) + } + } else { + log.Infof("Searching for parent snapshot") + + snap, err := findPreviousSnapshot(ctx, rep, realSource, snapshotTags, nil, log) + if err != nil { + log.WithError(err).Warn("Failed to search previous snapshot, fallback to full backup") + } else { + previous = &snap + log.Infof("Using previous snapshot %s", snap.RootObject.ID) + } + } + } else { + log.Info("Forcing full snapshot") + } + + parentInfo := parentBackupInfo{} + if previous != nil { + if previous.Tags == nil { + log.Warnf("No tag from parent snapshot %s, fallback to full backup", parentSnapshot) + } else if previous.Tags[uploader.CBTChangeIDTag] == "" { + log.Warnf("No ChangeID tag from parent snapshot %s, fallback to full backup", parentSnapshot) + } else if previous.Tags[uploader.CBTVolumeIDTag] == "" { + log.Warnf("No VolumeID tag from parent snapshot %s, fallback to full backup", parentSnapshot) + } else if previous.Tags[uploader.CBTVolumeIDTag] != volumeID { + log.Warnf("VolumeID %s from parent snapshot %s is not expected as %s, fallback to full backup", previous.Tags[uploader.CBTVolumeIDTag], parentSnapshot, volumeID) + } else if obj, err := loadObjectFromSnapshot(ctx, rep, previous); err != nil { + log.WithError(err).Warnf("Failed to load object from parent snapshot %s, fallback to full backup", parentSnapshot) + } else { + parentInfo.parentObject = obj + parentInfo.changeID = previous.Tags[uploader.CBTChangeIDTag] + parentInfo.volumeID = previous.Tags[uploader.CBTVolumeIDTag] + + log.Infof("Using parent snapshot %s, start time %v, end time %v, description %s", parentSnapshot, previous.StartTime, previous.EndTime, previous.Description) + } + } + + return parentInfo +} + +// Restore restore specific sourcePath with given snapshotID and update progress +func Restore(ctx context.Context, blkUp Uploader, rep udmrepo.BackupRepo, snapshotID, dest string, uploaderCfg map[string]string, log logrus.FieldLogger) (int64, error) { + log.Info("Start to restore...") + + snapshot, err := rep.GetSnapshot(ctx, udmrepo.ID(snapshotID)) + if err != nil { + return 0, errors.Wrapf(err, "Unable to load snapshot %v", snapshotID) + } + + log.Infof("Restore from snapshot %s, description %s, created time %v, tags %v", snapshotID, snapshot.Description, snapshot.EndTime, snapshot.Tags) + + bitmap := cbt.NewBitmap(blockSize, uint64(snapshot.TotalSize), "", "", "") + bitmap.SetFull() + + destPath, err := filepath.Abs(dest) + if err != nil { + return 0, errors.Wrapf(err, "invalid dest path '%s'", dest) + } + + destPath = filepath.Clean(destPath) + + destDev, err := openBlockDeviceFunc(destPath, false) + if err != nil { + return 0, errors.Wrapf(err, "error opening block device '%s'", destPath) + } + + defer destDev.Close() + + size, err := blkUp.Restore(snapshot, destInfo{dev: destDev, path: destPath}, bitmap.Iterator(), uploaderCfg) + if err != nil { + return 0, errors.Wrapf(err, "error restoring to block dev %s", destPath) + } + + return size, nil +} + +func findPreviousSnapshot(ctx context.Context, rep udmrepo.BackupRepo, path string, snapshotTags map[string]string, noLaterThan *time.Time, log logrus.FieldLogger) (udmrepo.Snapshot, error) { + snaps, err := rep.ListSnapshot(ctx, path) + if err != nil { + return udmrepo.Snapshot{}, errors.Wrapf(err, "error list snapshots for %s", path) + } + + var previous *udmrepo.Snapshot + + for _, snap := range snaps { + log.Debugf("Found one snapshot %s, start time %v, tags %v", snap.RootObject.ID, snap.StartTime, snap.Tags) + + requester, found := snap.Tags[uploader.SnapshotRequesterTag] + if !found { + continue + } + + if requester != snapshotTags[uploader.SnapshotRequesterTag] { + continue + } + + uploaderName, found := snap.Tags[uploader.SnapshotUploaderTag] + if !found { + continue + } + + if uploaderName != uploader.BlockType { + continue + } + + if noLaterThan != nil && snap.StartTime.After(*noLaterThan) { + continue + } + + if previous == nil || snap.StartTime.After(previous.StartTime) { + previous = &snap + } + } + + if previous == nil { + return udmrepo.Snapshot{}, errors.Errorf("no matching snapshot found for source %s", path) + } + + return *previous, nil +} diff --git a/pkg/uploader/block/snapshot_test.go b/pkg/uploader/block/snapshot_test.go new file mode 100644 index 000000000..8f6338311 --- /dev/null +++ b/pkg/uploader/block/snapshot_test.go @@ -0,0 +1,634 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Tests live in package block (not block_test) so they can access unexported +// types sourceInfo and destInfo, which appear in the Uploader interface. +package block + +import ( + "context" + "os" + "testing" + "time" + + "github.com/cockroachdb/errors" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/vmware-tanzu/velero/pkg/cbtservice" + "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" + udmrepomocks "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/mocks" + "github.com/vmware-tanzu/velero/pkg/uploader" + cbttypes "github.com/vmware-tanzu/velero/pkg/uploader/cbt/types" +) + +type mockUploader struct { + mock.Mock +} + +func (m *mockUploader) Backup(src sourceInfo, parent udmrepo.ID, iter cbttypes.Iterator, cfg map[string]string) (udmrepo.Snapshot, int64, error) { + args := m.Called(src, parent, iter, cfg) + return args.Get(0).(udmrepo.Snapshot), args.Get(1).(int64), args.Error(2) +} + +func (m *mockUploader) Restore(snap udmrepo.Snapshot, dest destInfo, iter cbttypes.Iterator, cfg map[string]string) (int64, error) { + args := m.Called(snap, dest, iter, cfg) + return args.Get(0).(int64), args.Error(1) +} + +func testLog() logrus.FieldLogger { + l := logrus.New() + l.SetLevel(logrus.DebugLevel) + return l +} + +func tempFile(t *testing.T, content string) *os.File { + t.Helper() + f, err := os.CreateTemp(t.TempDir(), "blktest-*") + require.NoError(t, err) + if content != "" { + _, err = f.WriteString(content) + require.NoError(t, err) + } + t.Cleanup(func() { + f.Close() + os.Remove(f.Name()) + }) + return f +} + +func TestBackup(t *testing.T) { + testCases := []struct { + name string + useNilBlkup bool + setupOpenDev func(t *testing.T) *os.File + setupMocks func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) + expectedErrStr string + checkInfo func(*testing.T, uploader.SnapshotInfo) + }{ + { + name: "nil uploader returns error", + useNilBlkup: true, + expectedErrStr: "get empty block uploader", + }, + { + name: "openBlockDevice error", + expectedErrStr: "error opening block device", + }, + { + name: "SnapshotSource error propagates", + setupOpenDev: func(t *testing.T) *os.File { + t.Helper() + return tempFile(t, "") + }, + setupMocks: func(blkup *mockUploader, _ *udmrepomocks.BackupRepo) { + blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(udmrepo.Snapshot{}, int64(0), errors.New("I/O error")) + }, + expectedErrStr: "Failed to run uploader backup", + }, + { + name: "success returns correct SnapshotInfo", + setupOpenDev: func(t *testing.T) *os.File { + t.Helper() + return tempFile(t, "test-block-data") + }, + setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { + blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(8), nil) + repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-001"), nil) + repo.On("Flush", mock.Anything).Return(nil) + }, + checkInfo: func(t *testing.T, info uploader.SnapshotInfo) { + t.Helper() + assert.Equal(t, "snap-001", info.ID) + assert.Equal(t, int64(8), info.IncrementalSize) + assert.Positive(t, info.Size) + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + mockBlkup := &mockUploader{} + mockRepo := udmrepomocks.NewBackupRepo(t) + + var blkup Uploader + if !tc.useNilBlkup { + blkup = mockBlkup + } + + if tc.setupOpenDev != nil { + f := tc.setupOpenDev(t) + openBlockDeviceFunc = func(_ string, _ bool) (*os.File, error) { + return f, nil + } + } else { + openBlockDeviceFunc = func(_ string, _ bool) (*os.File, error) { + return nil, errors.New("device not available") + } + } + + if tc.setupMocks != nil { + tc.setupMocks(mockBlkup, mockRepo) + } + + info, isEmpty, err := Backup( + ctx, blkup, mockRepo, + "/dev/sda", "", + cbtservice.SourceInfo{}, + true, "", nil, + map[string]string{}, map[string]string{}, + testLog(), + ) + + if tc.expectedErrStr != "" { + require.Error(t, err) + require.ErrorContains(t, err, tc.expectedErrStr) + } else { + require.NoError(t, err) + assert.False(t, isEmpty) + } + + if tc.checkInfo != nil { + tc.checkInfo(t, info) + } + + mockBlkup.AssertExpectations(t) + }) + } +} + +func TestSnapshotSource(t *testing.T) { + baseSource := sourceInfo{realSource: "/test/vol", size: 1024} + + testCases := []struct { + name string + setupMocks func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) + expectedErrStr string + expectedSnapID string + expectedSize int64 + }{ + { + name: "uploader Backup error", + setupMocks: func(blkup *mockUploader, _ *udmrepomocks.BackupRepo) { + blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(udmrepo.Snapshot{}, int64(0), errors.New("uploader error")) + }, + expectedErrStr: "Failed to run uploader backup", + }, + { + name: "SaveSnapshot error", + setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { + blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(udmrepo.Snapshot{}, int64(0), nil) + repo.On("SaveSnapshot", mock.Anything, mock.Anything). + Return(udmrepo.ID(""), errors.New("save failed")) + }, + expectedErrStr: "Failed to save snapshot", + }, + { + name: "Flush error", + setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { + blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(udmrepo.Snapshot{}, int64(0), nil) + repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-001"), nil) + repo.On("Flush", mock.Anything).Return(errors.New("flush failed")) + }, + expectedErrStr: "Failed to flush repository", + }, + { + name: "success with nil cbtService falls back to full bitmap", + setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { + blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root"}}, int64(512), nil) + repo.On("SaveSnapshot", mock.Anything, mock.Anything).Return(udmrepo.ID("snap-success"), nil) + repo.On("Flush", mock.Anything).Return(nil) + }, + expectedSnapID: "snap-success", + expectedSize: 512, + }, + { + name: "tags from cbtSource and snapshotTags are merged onto snapshot", + setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { + blkup.On("Backup", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(udmrepo.Snapshot{}, int64(0), nil) + repo.On("SaveSnapshot", mock.Anything, mock.MatchedBy(func(snap udmrepo.Snapshot) bool { + return snap.Tags[uploader.CBTChangeIDTag] == "cid-1" && + snap.Tags[uploader.CBTVolumeIDTag] == "vid-1" && + snap.Tags["custom"] == "val" && + snap.Description == "Block Uploader" + })).Return(udmrepo.ID("snap-tags"), nil) + repo.On("Flush", mock.Anything).Return(nil) + }, + expectedSnapID: "snap-tags", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + mockBlkup := &mockUploader{} + mockRepo := udmrepomocks.NewBackupRepo(t) + + tc.setupMocks(mockBlkup, mockRepo) + + cbtSrc := cbtservice.SourceInfo{ChangeID: "cid-1", VolumeID: "vid-1"} + snapshotTags := map[string]string{"custom": "val"} + + snapID, size, err := snapshotSource( + ctx, mockRepo, mockBlkup, + baseSource, + true, "", + cbtSrc, nil, + snapshotTags, map[string]string{}, + testLog(), "Block Uploader", + ) + + if tc.expectedErrStr != "" { + require.Error(t, err) + require.ErrorContains(t, err, tc.expectedErrStr) + } else { + require.NoError(t, err) + assert.Equal(t, tc.expectedSnapID, snapID) + assert.Equal(t, tc.expectedSize, size) + } + + mockBlkup.AssertExpectations(t) + }) + } +} + +func TestGetParentBackupInfo(t *testing.T) { + const volumeID = "vol-123" + const realSource = "/test/source" + + snapshotTags := map[string]string{ + uploader.SnapshotRequesterTag: "test-requester", + uploader.SnapshotUploaderTag: uploader.BlockType, + } + + validSnap := udmrepo.Snapshot{ + RootObject: udmrepo.ObjectMetadata{ID: "root-obj"}, + Tags: map[string]string{ + uploader.CBTChangeIDTag: "cid-abc", + uploader.CBTVolumeIDTag: volumeID, + uploader.SnapshotRequesterTag: "test-requester", + uploader.SnapshotUploaderTag: uploader.BlockType, + }, + } + + testCases := []struct { + name string + forceFull bool + parentSnapshot string + setupMocks func(repo *udmrepomocks.BackupRepo) + expectEmpty bool + expectedParent udmrepo.ID + expectedCID string + expectedVID string + }{ + { + name: "forceFull skips all parent lookup", + forceFull: true, + expectEmpty: true, + }, + { + name: "GetSnapshot fails — falls back to full", + parentSnapshot: "snap-parent", + setupMocks: func(repo *udmrepomocks.BackupRepo) { + repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-parent")). + Return(udmrepo.Snapshot{}, errors.New("not found")) + }, + expectEmpty: true, + }, + { + name: "parent snapshot has nil tags — falls back to full", + parentSnapshot: "snap-notags", + setupMocks: func(repo *udmrepomocks.BackupRepo) { + repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-notags")). + Return(udmrepo.Snapshot{Tags: nil}, nil) + }, + expectEmpty: true, + }, + { + name: "parent snapshot missing ChangeID tag — falls back to full", + parentSnapshot: "snap-nocid", + setupMocks: func(repo *udmrepomocks.BackupRepo) { + repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-nocid")). + Return(udmrepo.Snapshot{Tags: map[string]string{uploader.CBTVolumeIDTag: volumeID}}, nil) + }, + expectEmpty: true, + }, + { + name: "parent snapshot missing VolumeID tag — falls back to full", + parentSnapshot: "snap-novid", + setupMocks: func(repo *udmrepomocks.BackupRepo) { + repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-novid")). + Return(udmrepo.Snapshot{Tags: map[string]string{uploader.CBTChangeIDTag: "cid"}}, nil) + }, + expectEmpty: true, + }, + { + name: "parent snapshot VolumeID mismatch — falls back to full", + parentSnapshot: "snap-vidmismatch", + setupMocks: func(repo *udmrepomocks.BackupRepo) { + repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-vidmismatch")). + Return(udmrepo.Snapshot{Tags: map[string]string{ + uploader.CBTChangeIDTag: "cid", + uploader.CBTVolumeIDTag: "different-vol", + }}, nil) + }, + expectEmpty: true, + }, + { + name: "valid parent snapshot — returns parent info", + parentSnapshot: "snap-valid", + setupMocks: func(repo *udmrepomocks.BackupRepo) { + repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-valid")). + Return(validSnap, nil) + repo.On("ReadMetadata", mock.Anything, udmrepo.ID("root-obj")). + Return(&udmrepo.Metadata{SubObjects: []udmrepo.ObjectMetadata{{ID: "root-obj"}}}, nil) + }, + expectedParent: "root-obj", + expectedCID: "cid-abc", + expectedVID: volumeID, + }, + { + name: "no parentSnapshot — ListSnapshot fails — falls back to full", + setupMocks: func(repo *udmrepomocks.BackupRepo) { + repo.On("ListSnapshot", mock.Anything, realSource). + Return(nil, errors.New("list error")) + }, + expectEmpty: true, + }, + { + name: "no parentSnapshot — no matching snapshot — falls back to full", + setupMocks: func(repo *udmrepomocks.BackupRepo) { + repo.On("ListSnapshot", mock.Anything, realSource). + Return([]udmrepo.Snapshot{{Tags: map[string]string{"other": "tag"}}}, nil) + }, + expectEmpty: true, + }, + { + name: "no parentSnapshot — matching snapshot found — returns parent info", + setupMocks: func(repo *udmrepomocks.BackupRepo) { + repo.On("ListSnapshot", mock.Anything, realSource). + Return([]udmrepo.Snapshot{validSnap}, nil) + repo.On("ReadMetadata", mock.Anything, udmrepo.ID("root-obj")). + Return(&udmrepo.Metadata{SubObjects: []udmrepo.ObjectMetadata{{ID: "root-obj"}}}, nil) + }, + expectedParent: "root-obj", + expectedCID: "cid-abc", + expectedVID: volumeID, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + mockRepo := udmrepomocks.NewBackupRepo(t) + + if tc.setupMocks != nil { + tc.setupMocks(mockRepo) + } + + info := getParentBackupInfo(ctx, mockRepo, tc.forceFull, tc.parentSnapshot, volumeID, realSource, snapshotTags, testLog()) + + if tc.expectEmpty { + assert.Empty(t, info.parentObject) + assert.Empty(t, info.changeID) + assert.Empty(t, info.volumeID) + } else { + assert.Equal(t, tc.expectedParent, info.parentObject) + assert.Equal(t, tc.expectedCID, info.changeID) + assert.Equal(t, tc.expectedVID, info.volumeID) + } + }) + } +} + +func TestFindPreviousSnapshot(t *testing.T) { + snapshotTags := map[string]string{ + uploader.SnapshotRequesterTag: "test-requester", + uploader.SnapshotUploaderTag: uploader.BlockType, + } + + matchingSnap := func(id string, start time.Time) udmrepo.Snapshot { + return udmrepo.Snapshot{ + RootObject: udmrepo.ObjectMetadata{ID: udmrepo.ID(id)}, + StartTime: start, + Tags: map[string]string{ + uploader.SnapshotRequesterTag: "test-requester", + uploader.SnapshotUploaderTag: uploader.BlockType, + }, + } + } + + testCases := []struct { + name string + setupMocks func(repo *udmrepomocks.BackupRepo) + expectedErrStr string + expectedID string + }{ + { + name: "ListSnapshot error", + setupMocks: func(repo *udmrepomocks.BackupRepo) { + repo.On("ListSnapshot", mock.Anything, "source"). + Return(nil, errors.New("list error")) + }, + expectedErrStr: "error list snapshots", + }, + { + name: "empty snapshot list — no match", + setupMocks: func(repo *udmrepomocks.BackupRepo) { + repo.On("ListSnapshot", mock.Anything, "source"). + Return([]udmrepo.Snapshot{}, nil) + }, + expectedErrStr: "no matching snapshot found", + }, + { + name: "snapshots without matching tags are filtered", + setupMocks: func(repo *udmrepomocks.BackupRepo) { + repo.On("ListSnapshot", mock.Anything, "source"). + Return([]udmrepo.Snapshot{ + {Tags: map[string]string{"unrelated": "tag"}}, + {Tags: nil}, + }, nil) + }, + expectedErrStr: "no matching snapshot found", + }, + { + name: "snapshot with wrong requester tag is filtered", + setupMocks: func(repo *udmrepomocks.BackupRepo) { + repo.On("ListSnapshot", mock.Anything, "source"). + Return([]udmrepo.Snapshot{{ + Tags: map[string]string{ + uploader.SnapshotRequesterTag: "other-requester", + uploader.SnapshotUploaderTag: uploader.BlockType, + }, + }}, nil) + }, + expectedErrStr: "no matching snapshot found", + }, + { + name: "snapshot with wrong uploader tag is filtered", + setupMocks: func(repo *udmrepomocks.BackupRepo) { + repo.On("ListSnapshot", mock.Anything, "source"). + Return([]udmrepo.Snapshot{{ + Tags: map[string]string{ + uploader.SnapshotRequesterTag: "test-requester", + uploader.SnapshotUploaderTag: "kopia", + }, + }}, nil) + }, + expectedErrStr: "no matching snapshot found", + }, + { + name: "single matching snapshot is returned", + setupMocks: func(repo *udmrepomocks.BackupRepo) { + repo.On("ListSnapshot", mock.Anything, "source"). + Return([]udmrepo.Snapshot{matchingSnap("snap-a", time.Now())}, nil) + }, + expectedID: "snap-a", + }, + { + name: "most recent of multiple matching snapshots is returned", + setupMocks: func(repo *udmrepomocks.BackupRepo) { + now := time.Now() + repo.On("ListSnapshot", mock.Anything, "source"). + Return([]udmrepo.Snapshot{ + matchingSnap("snap-old", now.Add(-2*time.Hour)), + matchingSnap("snap-new", now.Add(-time.Minute)), + matchingSnap("snap-mid", now.Add(-time.Hour)), + }, nil) + }, + expectedID: "snap-new", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + mockRepo := udmrepomocks.NewBackupRepo(t) + tc.setupMocks(mockRepo) + + snap, err := findPreviousSnapshot(ctx, mockRepo, "source", snapshotTags, nil, testLog()) + + if tc.expectedErrStr != "" { + require.Error(t, err) + require.ErrorContains(t, err, tc.expectedErrStr) + } else { + require.NoError(t, err) + assert.Equal(t, udmrepo.ID(tc.expectedID), snap.RootObject.ID) + } + }) + } +} + +func TestRestore(t *testing.T) { + storedSnap := udmrepo.Snapshot{Description: "test snapshot"} + + testCases := []struct { + name string + setupMocks func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) + setupOpenDev func(t *testing.T) *os.File + expectedErrStr string + expectedSize int64 + }{ + { + name: "GetSnapshot error", + setupMocks: func(_ *mockUploader, repo *udmrepomocks.BackupRepo) { + repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-001")). + Return(udmrepo.Snapshot{}, errors.New("not found")) + }, + expectedErrStr: "Unable to load snapshot", + }, + { + name: "openBlockDevice error", + setupMocks: func(_ *mockUploader, repo *udmrepomocks.BackupRepo) { + repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-001")). + Return(storedSnap, nil) + }, + expectedErrStr: "error opening block device", + }, + { + name: "Restore error", + setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { + repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-001")). + Return(storedSnap, nil) + blkup.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(int64(0), errors.New("restore I/O error")) + }, + setupOpenDev: func(t *testing.T) *os.File { + t.Helper() + return tempFile(t, "") + }, + expectedErrStr: "error restoring to block dev", + }, + { + name: "success returns size", + setupMocks: func(blkup *mockUploader, repo *udmrepomocks.BackupRepo) { + repo.On("GetSnapshot", mock.Anything, udmrepo.ID("snap-001")). + Return(storedSnap, nil) + blkup.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(int64(4096), nil) + }, + setupOpenDev: func(t *testing.T) *os.File { + t.Helper() + return tempFile(t, "") + }, + expectedSize: 4096, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + mockBlkup := &mockUploader{} + mockRepo := udmrepomocks.NewBackupRepo(t) + + tc.setupMocks(mockBlkup, mockRepo) + + if tc.setupOpenDev != nil { + f := tc.setupOpenDev(t) + openBlockDeviceFunc = func(_ string, _ bool) (*os.File, error) { + return f, nil + } + } else { + openBlockDeviceFunc = func(_ string, _ bool) (*os.File, error) { + return nil, errors.New("device not available") + } + } + + size, err := Restore(ctx, mockBlkup, mockRepo, "snap-001", "/dev/sdb", map[string]string{}, testLog()) + + if tc.expectedErrStr != "" { + require.Error(t, err) + require.ErrorContains(t, err, tc.expectedErrStr) + assert.Equal(t, int64(0), size) + } else { + require.NoError(t, err) + assert.Equal(t, tc.expectedSize, size) + } + + mockBlkup.AssertExpectations(t) + }) + } +} diff --git a/pkg/uploader/block/uploader.go b/pkg/uploader/block/uploader.go new file mode 100644 index 000000000..75e913cb7 --- /dev/null +++ b/pkg/uploader/block/uploader.go @@ -0,0 +1,336 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package block + +import ( + "context" + "io" + "os" + "runtime" + "strings" + + "github.com/cockroachdb/errors" + "github.com/sirupsen/logrus" + + "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" + "github.com/vmware-tanzu/velero/pkg/uploader" + cbt "github.com/vmware-tanzu/velero/pkg/uploader/cbt/types" + "github.com/vmware-tanzu/velero/pkg/util/freelist" +) + +var ErrCanceled = errors.New("uploader is canceled") + +const ( + blockSize = (1 << 20) + bufferSize = 100 << 20 +) + +type sourceInfo struct { + dev *os.File + realSource string + size int64 +} + +type destInfo struct { + dev *os.File + path string +} + +type Uploader interface { + Backup(sourceInfo, udmrepo.ID, cbt.Iterator, map[string]string) (udmrepo.Snapshot, int64, error) + Restore(udmrepo.Snapshot, destInfo, cbt.Iterator, map[string]string) (int64, error) +} + +type blockUploader struct { + ctx context.Context + repoWriter udmrepo.BackupRepo + progress uploader.ProgressUpdater + log logrus.FieldLogger +} + +func NewUploader(ctx context.Context, repoWriter udmrepo.BackupRepo, progress uploader.ProgressUpdater, log logrus.FieldLogger) Uploader { + return &blockUploader{ + ctx: ctx, + repoWriter: repoWriter, + progress: progress, + log: log, + } +} + +func (blkup *blockUploader) Backup(source sourceInfo, parentObject udmrepo.ID, bitmap cbt.Iterator, configs map[string]string) (udmrepo.Snapshot, int64, error) { + snapStart := blkup.repoWriter.Time() + + if bitmap == nil { + return udmrepo.Snapshot{}, 0, errors.New("bitmap is not available") + } + + backupMode := udmrepo.ObjectDataBackupModeInc + if parentObject == "" { + backupMode = udmrepo.ObjectDataBackupModeFull + } + + destObj, err := blkup.repoWriter.NewObjectWriter(blkup.ctx, udmrepo.ObjectWriteOptions{ + Description: "BDEV:" + getObjectName(source.realSource), + DataType: udmrepo.ObjectDataTypeData, + AccessMode: udmrepo.ObjectDataAccessModeBlock, + ParentObject: parentObject, + BackupMode: backupMode, + AsyncWrites: runtime.NumCPU(), + }) + if err != nil { + return udmrepo.Snapshot{}, 0, errors.Wrap(err, "error creating object writer") + } + + defer destObj.Close() + + id, backupSize, objectSize, err := blkup.backupObject(source.dev, destObj, bitmap, source.size) + if err != nil { + return udmrepo.Snapshot{}, 0, errors.Wrapf(err, "error backing up bdev %s", source.realSource) + } + + entryID, err := blkup.repoWriter.WriteMetadata(blkup.ctx, &udmrepo.Metadata{ + SubObjects: []udmrepo.ObjectMetadata{ + { + ID: id, + Name: getObjectName(source.realSource), + Type: udmrepo.ObjectDataTypeData, + Size: objectSize, + Permissions: 0o777, + }, + }, + }, + udmrepo.ObjectWriteOptions{ + Description: "bdev-root", + }) + if err != nil { + return udmrepo.Snapshot{}, 0, errors.Wrap(err, "error writing metadata") + } + + snapEnd := blkup.repoWriter.Time() + + return udmrepo.Snapshot{ + Source: source.realSource, + StartTime: snapStart, + EndTime: snapEnd, + Description: source.realSource, + TotalSize: objectSize, + RootObject: udmrepo.ObjectMetadata{ + ID: entryID, + Name: "bdev-root", + Type: udmrepo.ObjectDataTypeMetadata, + Permissions: 0o777, + }, + }, backupSize, nil +} + +// TODO implement in following PRs +func (blkup *blockUploader) Restore(snapshot udmrepo.Snapshot, dest destInfo, bitmap cbt.Iterator, configs map[string]string) (int64, error) { + return 0, errors.New("not implemented") +} + +func (blkup *blockUploader) backupObject(dev *os.File, dest udmrepo.ObjectWriter, bitmap cbt.Iterator, totalLength int64) (udmrepo.ID, int64, int64, error) { + backupSize, objectSize, err := blkup.backupData(dev, dest, bitmap, totalLength) + if err != nil { + return "", backupSize, objectSize, err + } + + id, err := dest.Result() + return id, backupSize, objectSize, err +} + +type readResult struct { + buffer []byte + offset int64 + err error +} + +func (r *readResult) resetBuffer(list *freelist.FreeList) { + if r.buffer != nil { + list.Return(r.buffer) + r.buffer = nil + } +} + +func (blkup *blockUploader) backupData(reader io.ReaderAt, writer udmrepo.ObjectWriter, bitmap cbt.Iterator, totalLength int64) (int64, int64, error) { + blockSize := bitmap.BlockSize() + list := freelist.New(bufferSize, int(blockSize)) + resultChan := make(chan readResult, list.Capacity()) + totalCount := bitmap.Count() + aligned := (totalLength + int64(blockSize) - 1) / int64(blockSize) * int64(blockSize) + + quit := make(chan struct{}) + defer close(quit) + + go func() { + defer close(resultChan) + + offset, valid := bitmap.Next() + var buffer []byte + for valid { + select { + case <-blkup.ctx.Done(): + return + case <-quit: + return + case buffer = <-list.Chunks(): + } + + length := blockSize + if offset+uint64(length) > uint64(totalLength) { + length = uint(uint64(totalLength) - offset) + clear(buffer) + } + + readBytes, err := reader.ReadAt(buffer[:length], int64(offset)) + if err == nil && readBytes <= 0 { + err = io.ErrUnexpectedEOF + } + + r := readResult{ + buffer: buffer, + offset: int64(offset), + err: err, + } + + if r.err != nil { + r.resetBuffer(list) + } + + resultChan <- r + + if r.err != nil { + return + } + + offset, valid = bitmap.Next() + } + }() + + var lastPos int64 + var result readResult + var written int64 + var curCount int64 + var writeErr error + var readerRunning bool + + for curCount < int64(totalCount) { + select { + case <-blkup.ctx.Done(): + writeErr = ErrCanceled + case result, readerRunning = <-resultChan: + if !readerRunning { + if blkup.ctx.Err() != nil { + writeErr = ErrCanceled + } else { + writeErr = io.ErrUnexpectedEOF + } + } + } + + if writeErr != nil { + break + } + + if result.err != nil { + writeErr = result.err + break + } + + n, err := writer.WriteAt(result.buffer, result.offset) + if err != nil { + writeErr = err + break + } + + if blockSize != uint(n) { + writeErr = io.ErrShortWrite + break + } + + written += int64(blockSize) + lastPos = result.offset + int64(blockSize) + result.resetBuffer(list) + curCount++ + + blkup.progress.UpdateProgress(&uploader.Progress{BytesDone: lastPos, TotalBytes: aligned}) + } + + result.resetBuffer(list) + + if writeErr != nil { + return written, aligned, writeErr + } + + if lastPos < aligned { + s, err := copyTailData(reader, writer, totalLength, int64(blockSize)) + if err != nil { + return written, aligned, errors.Wrapf(err, "unable to write tail data at %v", lastPos) + } + + written += s + + blkup.progress.UpdateProgress(&uploader.Progress{BytesDone: aligned, TotalBytes: aligned}) + } + + return written, aligned, nil +} + +func copyTailData(source io.ReaderAt, writer udmrepo.ObjectWriter, totalLength int64, blockSize int64) (int64, error) { + roundUp := (totalLength + blockSize - 1) / blockSize * blockSize + roundDown := totalLength / blockSize * blockSize + length := totalLength - roundDown + + if length == 0 { + if _, err := writer.WriteAt(nil, roundUp); err != nil { + return -1, errors.Wrapf(err, "error writing sparse to %v", roundUp) + } + } else { + buffer := make([]byte, blockSize) + if _, err := source.ReadAt(buffer[:length], roundDown); err != nil { + return -1, errors.Wrapf(err, "error reading tail data with length %v", length) + } + + if _, err := writer.WriteAt(buffer, roundDown); err != nil { + return -1, errors.Wrapf(err, "error writing tail data at %v", roundDown) + } + } + + return length, nil +} + +func getObjectName(source string) string { + s := strings.ReplaceAll(source, "/", "-") + s = strings.ReplaceAll(s, "\\", "-") + return strings.Trim(s, "-") +} + +func loadObjectFromSnapshot(ctx context.Context, rep udmrepo.BackupRepo, snapshot *udmrepo.Snapshot) (udmrepo.ID, error) { + if snapshot == nil { + return "", errors.New("snapshot is empty") + } + + meta, err := rep.ReadMetadata(ctx, snapshot.RootObject.ID) + if err != nil { + return "", errors.Wrapf(err, "error reading snapshot metadata for %s", snapshot.Description) + } + + if len(meta.SubObjects) != 1 { + return "", errors.Errorf("unexpected number of bdev object (%d) for snapshot %s", len(meta.SubObjects), snapshot.Description) + } + + return meta.SubObjects[0].ID, nil +} diff --git a/pkg/uploader/block/uploader_test.go b/pkg/uploader/block/uploader_test.go new file mode 100644 index 000000000..88fd4771e --- /dev/null +++ b/pkg/uploader/block/uploader_test.go @@ -0,0 +1,461 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package block + +import ( + "bytes" + "context" + "io" + "os" + "testing" + "time" + + "github.com/cockroachdb/errors" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" + udmrepomocks "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/mocks" + "github.com/vmware-tanzu/velero/pkg/uploader" + cbt "github.com/vmware-tanzu/velero/pkg/uploader/cbt/types" + cbtmocks "github.com/vmware-tanzu/velero/pkg/uploader/cbt/types/mocks" +) + +type mockProgressUpdater struct { + mock.Mock +} + +func (m *mockProgressUpdater) UpdateProgress(p *uploader.Progress) { + m.Called(p) +} + +func TestNewUploader(t *testing.T) { + ctx := context.Background() + repoWriter := udmrepomocks.NewBackupRepo(t) + progress := &mockProgressUpdater{} + log := logrus.New() + + uploader := NewUploader(ctx, repoWriter, progress, log) + + blkup, ok := uploader.(*blockUploader) + assert.True(t, ok) + assert.Equal(t, ctx, blkup.ctx) + assert.Equal(t, repoWriter, blkup.repoWriter) + assert.Equal(t, progress, blkup.progress) + assert.Equal(t, log, blkup.log) +} + +func TestGetObjectName(t *testing.T) { + testCases := []struct { + name string + source string + expected string + }{ + { + name: "no slashes", + source: "test", + expected: "test", + }, + { + name: "unix path", + source: "/var/lib/kubelet/pods/uuid/volumes/test", + expected: "var-lib-kubelet-pods-uuid-volumes-test", + }, + { + name: "windows path", + source: `c:\var\lib\kubelet\pods\uuid\volumes\test`, + expected: `c:-var-lib-kubelet-pods-uuid-volumes-test`, + }, + { + name: "mixed slashes", + source: `c:\var/lib\kubelet/pods`, + expected: `c:-var-lib-kubelet-pods`, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := getObjectName(tc.source) + assert.Equal(t, tc.expected, result) + }) + } +} + +func TestCopyTailData(t *testing.T) { + testCases := []struct { + name string + totalLength int64 + blockSize int64 + sourceData []byte + writeErr error + readErr error + expected int64 + expectErr bool + }{ + { + name: "tail length 0", + totalLength: 2048, + blockSize: 1024, + expected: 0, + }, + { + name: "tail length 512 with 1024 block size", + totalLength: 1536, + blockSize: 1024, + sourceData: make([]byte, 1536), + expected: 512, + }, + { + name: "tail length with write error", + totalLength: 1536, + blockSize: 1024, + sourceData: make([]byte, 1536), + writeErr: errors.New("write error"), + expectErr: true, + }, + { + name: "tail length 0 with sparse write error", + totalLength: 2048, + blockSize: 1024, + writeErr: errors.New("write error"), + expectErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + writer := udmrepomocks.NewObjectWriter(t) + var source io.ReaderAt + + if tc.totalLength%tc.blockSize == 0 { + writer.On("WriteAt", []byte(nil), tc.totalLength).Return(0, tc.writeErr) + } else { + length := tc.totalLength - (tc.totalLength/tc.blockSize)*tc.blockSize + paddedData := make([]byte, tc.blockSize) + copy(paddedData[:length], tc.sourceData) + + source = bytes.NewReader(tc.sourceData) + writer.On("WriteAt", paddedData, (tc.totalLength/tc.blockSize)*tc.blockSize).Return(int(tc.blockSize), tc.writeErr) + } + + n, err := copyTailData(source, writer, tc.totalLength, tc.blockSize) + if tc.expectErr { + assert.Error(t, err) + } else { + require.NoError(t, err) + assert.Equal(t, tc.expected, n) + } + }) + } +} + +func TestBlockUploaderBackup(t *testing.T) { + testCases := []struct { + name string + nilBitmap bool + createObjErr error + writeMetaErr error + writeObjErr error + parentObj udmrepo.ID + cancelCtx bool + cancelInProgress bool + readDataErr bool + shortWrite bool + fewerBlocks bool + expectErr bool + expectErrStr string + }{ + { + name: "nil bitmap", + nilBitmap: true, + expectErr: true, + }, + { + name: "canceled context", + cancelCtx: true, + expectErr: true, + expectErrStr: "uploader is canceled", + }, + { + name: "canceled in progress", + cancelInProgress: true, + expectErr: true, + expectErrStr: "error backing up bdev /data/volume1: uploader is canceled", + }, + { + name: "create object writer err", + createObjErr: errors.New("create obj err"), + expectErr: true, + }, + { + name: "read data err", + readDataErr: true, + expectErr: true, + expectErrStr: "EOF", + }, + { + name: "short write err", + shortWrite: true, + expectErr: true, + expectErrStr: "short write", + }, + { + name: "unexpected EOF fewer blocks", + fewerBlocks: true, + expectErr: true, + expectErrStr: "unexpected EOF", + }, + { + name: "write meta err", + writeMetaErr: errors.New("write meta err"), + expectErr: true, + }, + { + name: "success full backup", + parentObj: "", + expectErr: false, + }, + { + name: "success inc backup", + parentObj: "parent-01", + expectErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + + if tc.cancelCtx { + cancel() + } else if tc.cancelInProgress { + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() + } else { + defer cancel() + } + + repoWriter := udmrepomocks.NewBackupRepo(t) + progress := &mockProgressUpdater{} + progress.On("UpdateProgress", mock.Anything).Return() + log := logrus.New() + log.Out = io.Discard + + blkup := NewUploader(ctx, repoWriter, progress, log) + + f, err := os.CreateTemp(t.TempDir(), "blktest-*") + require.NoError(t, err) + defer os.Remove(f.Name()) + defer f.Close() + + if tc.cancelInProgress { + require.NoError(t, f.Truncate(2*1048576)) + } else if tc.readDataErr { + // Don't truncate so that reading hits EOF immediately + } else { + require.NoError(t, f.Truncate(1048576)) + } + + fi, err := f.Stat() + require.NoError(t, err) + + srcInfo := sourceInfo{ + dev: f, + realSource: "/data/volume1", + size: fi.Size(), + } + + if tc.readDataErr { + srcInfo.size = 1048576 + } + + repoWriter.On("Time").Return(time.Now()) + + var iterator cbt.Iterator + if !tc.nilBitmap { + iterMock := cbtmocks.NewIterator(t) + iterator = iterMock + + backupMode := udmrepo.ObjectDataBackupModeInc + if tc.parentObj == "" { + backupMode = udmrepo.ObjectDataBackupModeFull + } + + objWriter := udmrepomocks.NewObjectWriter(t) + if tc.createObjErr == nil { + objWriter.On("Close").Return(nil) + + if tc.cancelInProgress { + iterMock.On("BlockSize").Return(uint(1048576)) + iterMock.On("Count").Return(uint64(1000)) + iterMock.On("Next").Return(uint64(0), true) + + objWriter.On("WriteAt", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + <-ctx.Done() + }).Return(1048576, nil) + objWriter.On("Result").Return(udmrepo.ID(""), errors.New("write failed")).Maybe() + } else if tc.cancelCtx { + iterMock.On("BlockSize").Return(uint(1048576)) + iterMock.On("Count").Return(uint64(1)) + iterMock.On("Next").Return(uint64(0), true).Maybe() + + objWriter.On("Result").Return(udmrepo.ID(""), errors.New("write failed")).Maybe() + } else if tc.shortWrite { + iterMock.On("BlockSize").Return(uint(1048576)) + iterMock.On("Count").Return(uint64(1)) + iterMock.On("Next").Return(uint64(0), true) + + objWriter.On("WriteAt", mock.Anything, mock.Anything).Return(512, nil) + objWriter.On("Result").Return(udmrepo.ID(""), errors.New("write failed")).Maybe() + } else if tc.fewerBlocks { + iterMock.On("BlockSize").Return(uint(1048576)) + iterMock.On("Count").Return(uint64(5)) + iterMock.On("Next").Return(uint64(0), false) + + objWriter.On("Result").Return(udmrepo.ID(""), errors.New("write failed")).Maybe() + } else if tc.readDataErr { + iterMock.On("BlockSize").Return(uint(1048576)) + iterMock.On("Count").Return(uint64(1)) + iterMock.On("Next").Return(uint64(0), true) + + objWriter.On("Result").Return(udmrepo.ID(""), errors.New("write failed")).Maybe() + } else { + // Setup backupData sequence: next returns false immediately + iterMock.On("BlockSize").Return(uint(1048576)) + iterMock.On("Count").Return(uint64(0)) + iterMock.On("Next").Return(uint64(0), false) + + if tc.writeObjErr != nil { + objWriter.On("WriteAt", mock.Anything, mock.Anything).Return(0, tc.writeObjErr) + objWriter.On("Result").Return(udmrepo.ID(""), errors.New("write failed")) + } else { + objWriter.On("WriteAt", mock.Anything, mock.Anything).Return(1048576, nil) + objWriter.On("Result").Return(udmrepo.ID("obj-01"), nil) + repoWriter.On("WriteMetadata", mock.Anything, mock.Anything, mock.Anything).Return(udmrepo.ID("meta-01"), tc.writeMetaErr) + } + } + } + + repoWriter.On("NewObjectWriter", mock.Anything, mock.MatchedBy(func(opt udmrepo.ObjectWriteOptions) bool { + return opt.Description == "BDEV:data-volume1" && opt.BackupMode == backupMode + })).Return(objWriter, tc.createObjErr) + } + + snap, size, err := blkup.Backup(srcInfo, tc.parentObj, iterator, nil) + + if tc.expectErr { + require.Error(t, err) + if tc.expectErrStr != "" { + assert.Contains(t, err.Error(), tc.expectErrStr) + } + } else { + require.NoError(t, err) + assert.Equal(t, "/data/volume1", snap.Source) + assert.Equal(t, udmrepo.ID("meta-01"), snap.RootObject.ID) + assert.Equal(t, int64(0), size) + } + }) + } +} + +func TestLoadObjectFromSnapshot(t *testing.T) { + testCases := []struct { + name string + snapshot *udmrepo.Snapshot + setupMocks func(repo *udmrepomocks.BackupRepo) + expectedErrStr string + expectedID udmrepo.ID + }{ + { + name: "nil snapshot", + snapshot: nil, + expectedErrStr: "snapshot is empty", + }, + { + name: "ReadMetadata error", + snapshot: &udmrepo.Snapshot{ + RootObject: udmrepo.ObjectMetadata{ID: "root-obj"}, + }, + setupMocks: func(repo *udmrepomocks.BackupRepo) { + repo.On("ReadMetadata", mock.Anything, udmrepo.ID("root-obj")). + Return(nil, errors.New("read error")) + }, + expectedErrStr: "error reading snapshot metadata", + }, + { + name: "unexpected number of subobjects (0)", + snapshot: &udmrepo.Snapshot{ + RootObject: udmrepo.ObjectMetadata{ID: "root-obj"}, + }, + setupMocks: func(repo *udmrepomocks.BackupRepo) { + repo.On("ReadMetadata", mock.Anything, udmrepo.ID("root-obj")). + Return(&udmrepo.Metadata{SubObjects: []udmrepo.ObjectMetadata{}}, nil) + }, + expectedErrStr: "unexpected number of bdev object", + }, + { + name: "unexpected number of subobjects (2)", + snapshot: &udmrepo.Snapshot{ + RootObject: udmrepo.ObjectMetadata{ID: "root-obj"}, + }, + setupMocks: func(repo *udmrepomocks.BackupRepo) { + repo.On("ReadMetadata", mock.Anything, udmrepo.ID("root-obj")). + Return(&udmrepo.Metadata{SubObjects: []udmrepo.ObjectMetadata{{ID: "obj-1"}, {ID: "obj-2"}}}, nil) + }, + expectedErrStr: "unexpected number of bdev object", + }, + { + name: "success", + snapshot: &udmrepo.Snapshot{ + RootObject: udmrepo.ObjectMetadata{ID: "root-obj"}, + }, + setupMocks: func(repo *udmrepomocks.BackupRepo) { + repo.On("ReadMetadata", mock.Anything, udmrepo.ID("root-obj")). + Return(&udmrepo.Metadata{SubObjects: []udmrepo.ObjectMetadata{{ID: "bdev-obj"}}}, nil) + }, + expectedID: "bdev-obj", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + mockRepo := udmrepomocks.NewBackupRepo(t) + + if tc.setupMocks != nil { + tc.setupMocks(mockRepo) + } + + id, err := loadObjectFromSnapshot(ctx, mockRepo, tc.snapshot) + + if tc.expectedErrStr != "" { + require.Error(t, err) + require.ErrorContains(t, err, tc.expectedErrStr) + assert.Empty(t, id) + } else { + require.NoError(t, err) + assert.Equal(t, tc.expectedID, id) + } + }) + } +} diff --git a/pkg/uploader/cbt/set.go b/pkg/uploader/cbt/set.go index 5919419e6..11361cf77 100644 --- a/pkg/uploader/cbt/set.go +++ b/pkg/uploader/cbt/set.go @@ -19,7 +19,7 @@ package cbt import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/pkg/cbtservice" "github.com/vmware-tanzu/velero/pkg/uploader/cbt/types" diff --git a/pkg/uploader/kopia/block_backup.go b/pkg/uploader/kopia/block_backup.go index ad90b723f..eb3435856 100644 --- a/pkg/uploader/kopia/block_backup.go +++ b/pkg/uploader/kopia/block_backup.go @@ -23,9 +23,9 @@ import ( "os" "syscall" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/fs/virtualfs" - "github.com/pkg/errors" ) const ErrNotPermitted = "operation not permitted" diff --git a/pkg/uploader/kopia/block_restore.go b/pkg/uploader/kopia/block_restore.go index 4f28a59de..33f1b72e0 100644 --- a/pkg/uploader/kopia/block_restore.go +++ b/pkg/uploader/kopia/block_restore.go @@ -26,9 +26,9 @@ import ( "path/filepath" "syscall" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/snapshot/restore" - "github.com/pkg/errors" ) type BlockOutput struct { diff --git a/pkg/uploader/kopia/flush_volume_linux.go b/pkg/uploader/kopia/flush_volume_linux.go index 98234e1b9..d73091a24 100644 --- a/pkg/uploader/kopia/flush_volume_linux.go +++ b/pkg/uploader/kopia/flush_volume_linux.go @@ -22,7 +22,7 @@ package kopia import ( "os" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "golang.org/x/sys/unix" ) diff --git a/pkg/uploader/kopia/progress_test.go b/pkg/uploader/kopia/progress_test.go index 8c18bb85b..065d6e395 100644 --- a/pkg/uploader/kopia/progress_test.go +++ b/pkg/uploader/kopia/progress_test.go @@ -20,7 +20,7 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/uploader" diff --git a/pkg/uploader/kopia/restore_output.go b/pkg/uploader/kopia/restore_output.go index 74311d38a..986530d52 100644 --- a/pkg/uploader/kopia/restore_output.go +++ b/pkg/uploader/kopia/restore_output.go @@ -17,8 +17,8 @@ limitations under the License. package kopia import ( + "github.com/cockroachdb/errors" "github.com/kopia/kopia/snapshot/restore" - "github.com/pkg/errors" ) var errFlushUnsupported = errors.New("flush is not supported") diff --git a/pkg/uploader/kopia/shim.go b/pkg/uploader/kopia/shim.go index f146348fa..4a3908185 100644 --- a/pkg/uploader/kopia/shim.go +++ b/pkg/uploader/kopia/shim.go @@ -21,7 +21,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" diff --git a/pkg/uploader/kopia/snapshot.go b/pkg/uploader/kopia/snapshot.go index 1924ed35b..217ff531f 100644 --- a/pkg/uploader/kopia/snapshot.go +++ b/pkg/uploader/kopia/snapshot.go @@ -28,6 +28,7 @@ import ( "github.com/sirupsen/logrus" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/fs/localfs" "github.com/kopia/kopia/repo" @@ -36,7 +37,6 @@ import ( "github.com/kopia/kopia/snapshot/policy" "github.com/kopia/kopia/snapshot/restore" "github.com/kopia/kopia/snapshot/snapshotfs" - "github.com/pkg/errors" "github.com/vmware-tanzu/velero/pkg/kopia" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" diff --git a/pkg/uploader/kopia/snapshot_test.go b/pkg/uploader/kopia/snapshot_test.go index 984b92af5..36f30d82c 100644 --- a/pkg/uploader/kopia/snapshot_test.go +++ b/pkg/uploader/kopia/snapshot_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/fs/virtualfs" "github.com/kopia/kopia/repo" @@ -30,7 +31,6 @@ import ( "github.com/kopia/kopia/snapshot/policy" "github.com/kopia/kopia/snapshot/restore" "github.com/kopia/kopia/snapshot/snapshotfs" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/uploader/provider/block.go b/pkg/uploader/provider/block.go index b1eed428a..9135bb67b 100644 --- a/pkg/uploader/provider/block.go +++ b/pkg/uploader/provider/block.go @@ -18,9 +18,10 @@ package provider import ( "context" + "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/internal/credentials" @@ -28,8 +29,12 @@ import ( repokeys "github.com/vmware-tanzu/velero/pkg/repository/keys" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" "github.com/vmware-tanzu/velero/pkg/uploader" + "github.com/vmware-tanzu/velero/pkg/uploader/block" ) +var blockBackupFunc = block.Backup +var blockRestoreFunc = block.Restore + type blockProvider struct { requestorType string bkRepo udmrepo.BackupRepo @@ -88,7 +93,6 @@ func (bp *blockProvider) GetPassword(param any) (string, error) { return strings.TrimSpace(rawPass), nil } -// TODO: implement in the following PRs func (bp *blockProvider) RunBackup( ctx context.Context, path string, @@ -100,10 +104,57 @@ func (bp *blockProvider) RunBackup( volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (string, bool, int64, int64, error) { - return "", false, 0, 0, errors.New("block backup not implemented") + if updater == nil { + return "", false, 0, 0, errors.New("backup progress updater is invalid") + } + + if path == "" { + return "", false, 0, 0, errors.New("path is empty") + } + + log := bp.log.WithFields(logrus.Fields{ + "path": path, + "realSource": realSource, + "parentSnapshot": parentSnapshot, + }) + + log.Infof("Run block backup, CBT source info: %v", cbtParam.Source) + + blkUploader := block.NewUploader(ctx, bp.bkRepo, updater, log) + + if tags == nil { + tags = make(map[string]string) + } + tags[uploader.SnapshotRequesterTag] = bp.requestorType + tags[uploader.SnapshotUploaderTag] = uploader.BlockType + + if realSource != "" { + realSource = fmt.Sprintf("%s/%s/%s", bp.requestorType, uploader.BlockType, realSource) + } + + snapshotInfo, _, err := blockBackupFunc(ctx, blkUploader, bp.bkRepo, path, realSource, cbtParam.Source, forceFull, parentSnapshot, cbtParam.Service, uploaderCfg, tags, log) + + if err == block.ErrCanceled { + log.Warn("Block backup is canceled") + return snapshotInfo.ID, false, snapshotInfo.Size, snapshotInfo.IncrementalSize, ErrorCanceled + } + + if err != nil { + return snapshotInfo.ID, false, snapshotInfo.Size, snapshotInfo.IncrementalSize, errors.Wrapf(err, "Failed to run block backup") + } + + updater.UpdateProgress( + &uploader.Progress{ + TotalBytes: snapshotInfo.Size, + BytesDone: snapshotInfo.Size, + }, + ) + + log.Infof("Block backup finished, snapshot ID %s, backup size %d", snapshotInfo.ID, snapshotInfo.Size) + + return snapshotInfo.ID, false, snapshotInfo.Size, snapshotInfo.IncrementalSize, nil } -// TODO: implement in the following PRs func (bp *blockProvider) RunRestore( ctx context.Context, snapshotID string, @@ -111,5 +162,35 @@ func (bp *blockProvider) RunRestore( volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, updater uploader.ProgressUpdater) (int64, error) { - return 0, errors.New("block restore not implemented") + if updater == nil { + return 0, errors.New("restore progress updater is invalid") + } + + log := bp.log.WithFields(logrus.Fields{ + "snapshotID": snapshotID, + "volumePath": volumePath, + }) + log.Info("Starting restore") + + blkUploader := block.NewUploader(ctx, bp.bkRepo, updater, log) + + size, err := blockRestoreFunc(ctx, blkUploader, bp.bkRepo, snapshotID, volumePath, uploaderCfg, log) + + if err == block.ErrCanceled { + log.Warn("Block restore is canceled") + return 0, ErrorCanceled + } + + if err != nil { + return 0, errors.Wrapf(err, "Failed to run block restore") + } + + updater.UpdateProgress(&uploader.Progress{ + TotalBytes: size, + BytesDone: size, + }) + + log.Infof("Block restore finished, restore size %v", size) + + return size, nil } diff --git a/pkg/uploader/provider/block_test.go b/pkg/uploader/provider/block_test.go index a48b0a862..ad8f68b52 100644 --- a/pkg/uploader/provider/block_test.go +++ b/pkg/uploader/provider/block_test.go @@ -17,9 +17,10 @@ limitations under the License. package provider import ( + "context" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -29,9 +30,12 @@ import ( "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/internal/credentials/mocks" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/cbtservice" "github.com/vmware-tanzu/velero/pkg/repository" "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" udmrepomocks "github.com/vmware-tanzu/velero/pkg/repository/udmrepo/mocks" + "github.com/vmware-tanzu/velero/pkg/uploader" + "github.com/vmware-tanzu/velero/pkg/uploader/block" ) func TestNewBlockUploaderProvider(t *testing.T) { @@ -125,6 +129,16 @@ func TestBlockProviderClose(t *testing.T) { mockBRepo.AssertExpectations(t) } +type blockMockProgressUpdater struct { + lastProgress *uploader.Progress + callCount int +} + +func (u *blockMockProgressUpdater) UpdateProgress(p *uploader.Progress) { + u.lastProgress = p + u.callCount++ +} + func TestBlockProviderGetPassword(t *testing.T) { testCases := []struct { name string @@ -185,3 +199,286 @@ func TestBlockProviderGetPassword(t *testing.T) { }) } } + +func TestBlockProviderRunBackup(t *testing.T) { + const requestorType = "test-requestor" + + testCases := []struct { + name string + path string + realSource string + tags map[string]string + updater uploader.ProgressUpdater + mockBackupResult uploader.SnapshotInfo + mockBackupErr error + expectedID string + expectedSize int64 + expectedIncrSize int64 + expectError bool + expectedErrStr string + skipMock bool + checkCaptures func(*testing.T, string, map[string]string) + }{ + { + name: "nil updater returns error", + path: "/dev/sda", + updater: nil, + expectError: true, + expectedErrStr: "backup progress updater is invalid", + skipMock: true, + }, + { + name: "empty path returns error", + path: "", + updater: &FakeBackupProgressUpdater{}, + expectError: true, + expectedErrStr: "path is empty", + skipMock: true, + }, + { + name: "success returns correct snapshot info and updates progress", + path: "/dev/sda", + updater: &blockMockProgressUpdater{}, + mockBackupResult: uploader.SnapshotInfo{ + ID: "snap-001", + Size: 1024, + IncrementalSize: 512, + }, + expectedID: "snap-001", + expectedSize: 1024, + expectedIncrSize: 512, + }, + { + name: "canceled backup returns ErrorCanceled with partial snapshot info", + path: "/dev/sda", + updater: &FakeBackupProgressUpdater{}, + mockBackupResult: uploader.SnapshotInfo{ + ID: "snap-canceled", + Size: 2048, + IncrementalSize: 1024, + }, + mockBackupErr: block.ErrCanceled, + expectedID: "snap-canceled", + expectedSize: 2048, + expectedIncrSize: 1024, + expectError: true, + expectedErrStr: "uploader is canceled", + }, + { + name: "generic backup error is wrapped", + path: "/dev/sda", + updater: &FakeBackupProgressUpdater{}, + mockBackupErr: errors.New("disk I/O error"), + expectError: true, + expectedErrStr: "Failed to run block backup", + }, + { + name: "nil tags are initialized with required tags", + path: "/dev/sda", + tags: nil, + updater: &FakeBackupProgressUpdater{}, + mockBackupResult: uploader.SnapshotInfo{ID: "snap-tags"}, + expectedID: "snap-tags", + checkCaptures: func(t *testing.T, _ string, tags map[string]string) { + t.Helper() + assert.Equal(t, requestorType, tags[uploader.SnapshotRequesterTag]) + assert.Equal(t, uploader.BlockType, tags[uploader.SnapshotUploaderTag]) + }, + }, + { + name: "non-empty realSource is prefixed with requestorType and BlockType", + path: "/dev/sda", + realSource: "my-volume", + updater: &FakeBackupProgressUpdater{}, + mockBackupResult: uploader.SnapshotInfo{ID: "snap-source"}, + expectedID: "snap-source", + checkCaptures: func(t *testing.T, realSource string, _ map[string]string) { + t.Helper() + assert.Equal(t, requestorType+"/"+uploader.BlockType+"/my-volume", realSource) + }, + }, + { + name: "empty realSource is passed through unchanged", + path: "/dev/sda", + realSource: "", + updater: &FakeBackupProgressUpdater{}, + mockBackupResult: uploader.SnapshotInfo{ID: "snap-nosource"}, + expectedID: "snap-nosource", + checkCaptures: func(t *testing.T, realSource string, _ map[string]string) { + t.Helper() + assert.Empty(t, realSource) + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + mockBRepo := udmrepomocks.NewBackupRepo(t) + + var capturedRealSrc string + var capturedTags map[string]string + + if !tc.skipMock { + blockBackupFunc = func(_ context.Context, _ block.Uploader, _ udmrepo.BackupRepo, _ string, realSource string, _ cbtservice.SourceInfo, _ bool, _ string, _ cbtservice.Service, _ map[string]string, tags map[string]string, _ logrus.FieldLogger) (uploader.SnapshotInfo, bool, error) { + capturedRealSrc = realSource + capturedTags = tags + return tc.mockBackupResult, false, tc.mockBackupErr + } + } + + bp := &blockProvider{ + requestorType: requestorType, + bkRepo: mockBRepo, + log: logrus.New(), + } + + snapshotID, isEmpty, size, incrSize, err := bp.RunBackup( + t.Context(), + tc.path, + tc.realSource, + tc.tags, + false, + "", + CBTParam{}, + uploader.PersistentVolumeBlock, + map[string]string{}, + tc.updater, + ) + + assert.Equal(t, tc.expectedID, snapshotID) + assert.Equal(t, tc.expectedSize, size) + assert.Equal(t, tc.expectedIncrSize, incrSize) + + if tc.expectError { + require.Error(t, err) + if tc.expectedErrStr != "" { + require.ErrorContains(t, err, tc.expectedErrStr) + } + } else { + require.NoError(t, err) + assert.False(t, isEmpty) + if mu, ok := tc.updater.(*blockMockProgressUpdater); ok { + assert.Equal(t, 1, mu.callCount) + require.NotNil(t, mu.lastProgress) + assert.Equal(t, tc.expectedSize, mu.lastProgress.TotalBytes) + assert.Equal(t, tc.expectedSize, mu.lastProgress.BytesDone) + } + } + + if tc.checkCaptures != nil { + tc.checkCaptures(t, capturedRealSrc, capturedTags) + } + }) + } +} + +func TestBlockProviderRunRestore(t *testing.T) { + testCases := []struct { + name string + snapshotID string + volumePath string + updater uploader.ProgressUpdater + mockRestoreSize int64 + mockRestoreErr error + expectedSize int64 + expectError bool + expectedErrStr string + checkCaptures func(*testing.T, string, string) + }{ + { + name: "nil updater returns error", + updater: nil, + expectError: true, + expectedErrStr: "restore progress updater is invalid", + }, + { + name: "success returns size and updates progress", + snapshotID: "snap-001", + volumePath: "/dev/sdb", + updater: &blockMockProgressUpdater{}, + mockRestoreSize: 4096, + expectedSize: 4096, + }, + { + name: "canceled restore returns ErrorCanceled", + snapshotID: "snap-canceled", + volumePath: "/dev/sdb", + updater: &FakeRestoreProgressUpdater{}, + mockRestoreErr: block.ErrCanceled, + expectError: true, + expectedErrStr: "uploader is canceled", + }, + { + name: "generic restore error is wrapped", + snapshotID: "snap-error", + volumePath: "/dev/sdb", + updater: &FakeRestoreProgressUpdater{}, + mockRestoreErr: errors.New("disk read error"), + expectError: true, + expectedErrStr: "Failed to run block restore", + }, + { + name: "snapshotID and volumePath are forwarded to restore func", + snapshotID: "snap-fwd", + volumePath: "/dev/sdc", + updater: &FakeRestoreProgressUpdater{}, + mockRestoreSize: 512, + expectedSize: 512, + checkCaptures: func(t *testing.T, snapshotID, volumePath string) { + t.Helper() + assert.Equal(t, "snap-fwd", snapshotID) + assert.Equal(t, "/dev/sdc", volumePath) + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + mockBRepo := udmrepomocks.NewBackupRepo(t) + + var capturedSnapshotID string + var capturedVolumePath string + + blockRestoreFunc = func(_ context.Context, _ block.Uploader, _ udmrepo.BackupRepo, snapshotID string, volumePath string, _ map[string]string, _ logrus.FieldLogger) (int64, error) { + capturedSnapshotID = snapshotID + capturedVolumePath = volumePath + return tc.mockRestoreSize, tc.mockRestoreErr + } + + bp := &blockProvider{ + bkRepo: mockBRepo, + log: logrus.New(), + } + + size, err := bp.RunRestore( + t.Context(), + tc.snapshotID, + tc.volumePath, + uploader.PersistentVolumeBlock, + map[string]string{}, + tc.updater, + ) + + if tc.expectError { + require.Error(t, err) + if tc.expectedErrStr != "" { + require.ErrorContains(t, err, tc.expectedErrStr) + } + assert.Equal(t, int64(0), size) + } else { + require.NoError(t, err) + assert.Equal(t, tc.expectedSize, size) + if mu, ok := tc.updater.(*blockMockProgressUpdater); ok { + assert.Equal(t, 1, mu.callCount) + require.NotNil(t, mu.lastProgress) + assert.Equal(t, tc.expectedSize, mu.lastProgress.TotalBytes) + assert.Equal(t, tc.expectedSize, mu.lastProgress.BytesDone) + } + } + + if tc.checkCaptures != nil { + tc.checkCaptures(t, capturedSnapshotID, capturedVolumePath) + } + }) + } +} diff --git a/pkg/uploader/provider/kopia.go b/pkg/uploader/provider/kopia.go index 620b0776d..682b2053e 100644 --- a/pkg/uploader/provider/kopia.go +++ b/pkg/uploader/provider/kopia.go @@ -22,8 +22,8 @@ import ( "strings" "sync/atomic" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/snapshot/upload" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/uploader" @@ -120,7 +120,8 @@ func (kp *kopiaProvider) RunBackup( _ CBTParam, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, - updater uploader.ProgressUpdater) (string, bool, int64, int64, error) { + updater uploader.ProgressUpdater, +) (string, bool, int64, int64, error) { if updater == nil { return "", false, 0, 0, errors.New("Need to initial backup progress updater first") } diff --git a/pkg/uploader/provider/kopia_test.go b/pkg/uploader/provider/kopia_test.go index 092333a24..bfb544c26 100644 --- a/pkg/uploader/provider/kopia_test.go +++ b/pkg/uploader/provider/kopia_test.go @@ -22,9 +22,9 @@ import ( "testing" "time" + "github.com/cockroachdb/errors" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot/upload" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/uploader/provider/provider.go b/pkg/uploader/provider/provider.go index 0c1caaffe..26b7b84f2 100644 --- a/pkg/uploader/provider/provider.go +++ b/pkg/uploader/provider/provider.go @@ -20,7 +20,7 @@ import ( "context" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" diff --git a/pkg/uploader/types.go b/pkg/uploader/types.go index 12ff1dc52..9c700193f 100644 --- a/pkg/uploader/types.go +++ b/pkg/uploader/types.go @@ -26,6 +26,8 @@ const ( BlockType = "velero-block" SnapshotRequesterTag = "snapshot-requester" SnapshotUploaderTag = "snapshot-uploader" + CBTChangeIDTag = "cbt-change-id" + CBTVolumeIDTag = "cbt-volume-id" ) type PersistentVolumeMode string @@ -49,8 +51,9 @@ func ValidateUploaderType(t string) (string, error) { } type SnapshotInfo struct { - ID string `json:"id"` - Size int64 `json:"Size"` + ID string + Size int64 + IncrementalSize int64 } // Progress which defined two variables to record progress diff --git a/pkg/uploader/util/uploader_config.go b/pkg/uploader/util/uploader_config.go index 5584ffbce..c221741bf 100644 --- a/pkg/uploader/util/uploader_config.go +++ b/pkg/uploader/util/uploader_config.go @@ -19,7 +19,7 @@ package util import ( "strconv" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) diff --git a/pkg/uploader/util/uploader_config_test.go b/pkg/uploader/util/uploader_config_test.go index 593bce4f0..46df8b714 100644 --- a/pkg/uploader/util/uploader_config_test.go +++ b/pkg/uploader/util/uploader_config_test.go @@ -20,7 +20,7 @@ import ( "reflect" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) diff --git a/pkg/util/actionhelpers/rbac.go b/pkg/util/actionhelpers/rbac.go index 1ecd97da2..521a8042a 100644 --- a/pkg/util/actionhelpers/rbac.go +++ b/pkg/util/actionhelpers/rbac.go @@ -19,7 +19,7 @@ package actionhelpers import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" rbacv1 "k8s.io/api/rbac/v1" rbacbeta "k8s.io/api/rbac/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/util/azure/credential.go b/pkg/util/azure/credential.go index b67b34f6c..f36eb43a6 100644 --- a/pkg/util/azure/credential.go +++ b/pkg/util/azure/credential.go @@ -23,7 +23,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" ) // NewCredential constructs a Credential that tries the config credential, workload identity credential diff --git a/pkg/util/azure/storage.go b/pkg/util/azure/storage.go index 49943a3f9..9f701e80b 100644 --- a/pkg/util/azure/storage.go +++ b/pkg/util/azure/storage.go @@ -27,7 +27,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" ) diff --git a/pkg/util/azure/util.go b/pkg/util/azure/util.go index e708d6ce3..5e3051336 100644 --- a/pkg/util/azure/util.go +++ b/pkg/util/azure/util.go @@ -29,8 +29,9 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" - "github.com/joho/godotenv" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" + + "github.com/vmware-tanzu/velero/pkg/util/dotenv" ) const ( @@ -68,7 +69,7 @@ func LoadCredentials(config map[string]string) (map[string]string, error) { } // put the credential file content into a map - creds, err := godotenv.Read(credFile) + creds, err := dotenv.Read(credFile) if err != nil { return nil, errors.Wrapf(err, "failed to read credentials from file %s", credFile) } diff --git a/pkg/util/collections/includes_excludes.go b/pkg/util/collections/includes_excludes.go index f326a4124..9405c0338 100644 --- a/pkg/util/collections/includes_excludes.go +++ b/pkg/util/collections/includes_excludes.go @@ -21,8 +21,8 @@ import ( "github.com/vmware-tanzu/velero/internal/resourcepolicies" + "github.com/cockroachdb/errors" "github.com/gobwas/glob" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/validation" "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/util/collections/includes_excludes_test.go b/pkg/util/collections/includes_excludes_test.go index 1d700a729..5f6bb970e 100644 --- a/pkg/util/collections/includes_excludes_test.go +++ b/pkg/util/collections/includes_excludes_test.go @@ -21,7 +21,7 @@ import ( "github.com/vmware-tanzu/velero/internal/resourcepolicies" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/pkg/util/csi/volume_snapshot.go b/pkg/util/csi/volume_snapshot.go index ed6371f7b..b78455bc8 100644 --- a/pkg/util/csi/volume_snapshot.go +++ b/pkg/util/csi/volume_snapshot.go @@ -20,13 +20,15 @@ import ( "context" "encoding/json" "fmt" + "os" + "strconv" "strings" "time" + "github.com/cockroachdb/errors" jsonpatch "github.com/evanphx/json-patch/v5" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -598,72 +600,102 @@ func WaitUntilVSCHandleIsReady( log logrus.FieldLogger, csiSnapshotTimeout time.Duration, ) (*snapshotv1api.VolumeSnapshotContent, error) { - // We'll wait 10m for the VSC to be reconciled polling - // every 5s unless backup's csiSnapshotTimeout is set - interval := 5 * time.Second + // We'll wait for the VSC to be reconciled, trying a fast poll interval first + // before falling back to a slower poll interval for the full csiSnapshotTimeout. vsc := new(snapshotv1api.VolumeSnapshotContent) + var interval time.Duration - err := wait.PollUntilContextTimeout( + pollFunc := func(ctx context.Context) (bool, error) { + vs := new(snapshotv1api.VolumeSnapshot) + if err := crClient.Get( + ctx, + crclient.ObjectKeyFromObject(volSnap), + vs, + ); err != nil { + return false, + errors.Wrapf( + err, + "failed to get volumesnapshot %s/%s", + volSnap.Namespace, volSnap.Name, + ) + } + + if vs.Status == nil || vs.Status.BoundVolumeSnapshotContentName == nil { + log.Infof("Waiting for CSI driver to reconcile volumesnapshot %s/%s. Retrying in %ds", + volSnap.Namespace, volSnap.Name, interval/time.Second) + return false, nil + } + + if err := crClient.Get( + ctx, + crclient.ObjectKey{ + Name: *vs.Status.BoundVolumeSnapshotContentName, + }, + vsc, + ); err != nil { + return false, + errors.Wrapf( + err, + "failed to get VolumeSnapshotContent %s for VolumeSnapshot %s/%s", + *vs.Status.BoundVolumeSnapshotContentName, vs.Namespace, vs.Name, + ) + } + + // we need to wait for the VolumeSnapshotContent + // to have a snapshot handle because during restore, + // we'll use that snapshot handle as the source for + // the VolumeSnapshotContent so it's statically + // bound to the existing snapshot. + if vsc.Status == nil || + vsc.Status.SnapshotHandle == nil { + log.Infof( + "Waiting for VolumeSnapshotContents %s to have snapshot handle. Retrying in %ds", + vsc.Name, interval/time.Second) + if vsc.Status != nil && + vsc.Status.Error != nil { + log.Warnf("VolumeSnapshotContent %s has error: %v", + vsc.Name, *vsc.Status.Error.Message) + } + return false, nil + } + + return true, nil + } + + var err error + frequentPolling, err := strconv.ParseBool(os.Getenv("CSI_SNAPSHOT_EARLY_FREQUENT_POLLING")) + + if err == nil && frequentPolling { + // The short interval for the first ten seconds is due to the fact that + // Microsoft VSS backups have a hard-coded unfreeze call after 10 seconds, + // so we need to minimize waiting time during the first 10 seconds. + // First poll with a short interval and timeout. + interval = 1 * time.Second + timeout := 10 * time.Second + err = wait.PollUntilContextTimeout( + context.Background(), + interval, + timeout, + true, + pollFunc, + ) + + if err == nil { + return vsc, nil + } + if !wait.Interrupted(err) { + return nil, err + } + } + + // If the first poll timed out, poll with a longer interval and the full timeout. + interval = 5 * time.Second + err = wait.PollUntilContextTimeout( context.Background(), interval, csiSnapshotTimeout, true, - func(ctx context.Context) (bool, error) { - vs := new(snapshotv1api.VolumeSnapshot) - if err := crClient.Get( - ctx, - crclient.ObjectKeyFromObject(volSnap), - vs, - ); err != nil { - return false, - errors.Wrapf( - err, - "failed to get volumesnapshot %s/%s", - volSnap.Namespace, volSnap.Name, - ) - } - - if vs.Status == nil || vs.Status.BoundVolumeSnapshotContentName == nil { - log.Infof("Waiting for CSI driver to reconcile volumesnapshot %s/%s. Retrying in %ds", - volSnap.Namespace, volSnap.Name, interval/time.Second) - return false, nil - } - - if err := crClient.Get( - ctx, - crclient.ObjectKey{ - Name: *vs.Status.BoundVolumeSnapshotContentName, - }, - vsc, - ); err != nil { - return false, - errors.Wrapf( - err, - "failed to get VolumeSnapshotContent %s for VolumeSnapshot %s/%s", - *vs.Status.BoundVolumeSnapshotContentName, vs.Namespace, vs.Name, - ) - } - - // we need to wait for the VolumeSnapshotContent - // to have a snapshot handle because during restore, - // we'll use that snapshot handle as the source for - // the VolumeSnapshotContent so it's statically - // bound to the existing snapshot. - if vsc.Status == nil || - vsc.Status.SnapshotHandle == nil { - log.Infof( - "Waiting for VolumeSnapshotContents %s to have snapshot handle. Retrying in %ds", - vsc.Name, interval/time.Second) - if vsc.Status != nil && - vsc.Status.Error != nil { - log.Warnf("VolumeSnapshotContent %s has error: %v", - vsc.Name, *vsc.Status.Error.Message) - } - return false, nil - } - - return true, nil - }, + pollFunc, ) if err != nil { diff --git a/pkg/util/datamover/datamover.go b/pkg/util/datamover/datamover.go new file mode 100644 index 000000000..59dd1499b --- /dev/null +++ b/pkg/util/datamover/datamover.go @@ -0,0 +1,43 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package datamover holds the shared data mover type identifiers and helpers. +// It must remain a leaf package (stdlib-only imports) so it can be referenced +// from anywhere in the codebase without introducing import cycles. +package datamover + +const ( + // DataMoverTypeVelero refers to the default built-in data mover. The default + // data mover may change among releases; see GetDefaultBuiltInDataMover. + DataMoverTypeVelero = "velero" + // DataMoverTypeVeleroFs refers to the Velero file system data mover. + DataMoverTypeVeleroFs = "velero-fs" + // DataMoverTypeVeleroBlock refers to the Velero block data mover. + DataMoverTypeVeleroBlock = "velero-block" +) + +// IsBuiltInDataMover reports whether the given data mover value refers to a +// Velero built-in data mover (an empty value or the default "velero" alias). +func IsBuiltInDataMover(dataMover string) bool { + return dataMover == "" || dataMover == DataMoverTypeVelero +} + +// GetDefaultBuiltInDataMover returns the data mover used when the default +// built-in data mover ("velero"/empty) is selected. The default may change +// between releases; currently it is the file system data mover. +func GetDefaultBuiltInDataMover() string { + return DataMoverTypeVeleroFs +} diff --git a/pkg/util/datamover/datamover_test.go b/pkg/util/datamover/datamover_test.go new file mode 100644 index 000000000..8576aed0e --- /dev/null +++ b/pkg/util/datamover/datamover_test.go @@ -0,0 +1,56 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package datamover + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsBuiltInDataMover(t *testing.T) { + testcases := []struct { + name string + dataMover string + want bool + }{ + { + name: "empty dataMover is builtin", + dataMover: "", + want: true, + }, + { + name: "velero dataMover is builtin", + dataMover: "velero", + want: true, + }, + { + name: "kopia dataMover is not builtin", + dataMover: "kopia", + want: false, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(tt *testing.T) { + assert.Equal(tt, tc.want, IsBuiltInDataMover(tc.dataMover)) + }) + } +} + +func TestGetDefaultBuiltInDataMover(t *testing.T) { + assert.Equal(t, DataMoverTypeVeleroFs, GetDefaultBuiltInDataMover()) +} diff --git a/pkg/util/dotenv/dotenv.go b/pkg/util/dotenv/dotenv.go new file mode 100644 index 000000000..23c0cda01 --- /dev/null +++ b/pkg/util/dotenv/dotenv.go @@ -0,0 +1,165 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dotenv + +import ( + "bufio" + "fmt" + "os" + "strings" +) + +// Read parses dotenv-style files and returns merged key/value pairs. +func Read(filenames ...string) (map[string]string, error) { + filenames = filenamesOrDefault(filenames) + envMap := make(map[string]string) + + for _, filename := range filenames { + fileMap, err := readFile(filename) + if err != nil { + return nil, err + } + + for key, value := range fileMap { + envMap[key] = value + } + } + + return envMap, nil +} + +// Overload loads dotenv-style files into process env vars, overriding existing values. +func Overload(filenames ...string) error { + filenames = filenamesOrDefault(filenames) + + for _, filename := range filenames { + envMap, err := readFile(filename) + if err != nil { + return err + } + + for key, value := range envMap { + if err := os.Setenv(key, value); err != nil { + return err + } + } + } + + return nil +} + +func filenamesOrDefault(filenames []string) []string { + if len(filenames) == 0 { + return []string{".env"} + } + return filenames +} + +func readFile(filename string) (map[string]string, error) { + file, err := os.Open(filename) + if err != nil { + return nil, err + } + defer file.Close() + + envMap := make(map[string]string) + scanner := bufio.NewScanner(file) + + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + line = stripInlineComment(line) + key, value, err := parseLine(line) + if err != nil { + return nil, err + } + envMap[key] = value + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + return envMap, nil +} + +func stripInlineComment(line string) string { + inSingle := false + inDouble := false + for i, r := range line { + switch r { + case '\'': + if !inDouble { + inSingle = !inSingle + } + case '"': + if !inSingle { + inDouble = !inDouble + } + case '#': + if !inSingle && !inDouble { + return strings.TrimSpace(line[:i]) + } + } + } + return line +} + +func parseLine(line string) (string, string, error) { + if strings.HasPrefix(line, "export ") { + line = strings.TrimSpace(strings.TrimPrefix(line, "export ")) + } + + sep := strings.Index(line, "=") + colon := strings.Index(line, ":") + if sep == -1 || (colon != -1 && colon < sep) { + sep = colon + } + if sep == -1 { + return "", "", fmt.Errorf("invalid dotenv line: %q", line) + } + + key := strings.TrimSpace(line[:sep]) + rawValue := strings.TrimSpace(line[sep+1:]) + if key == "" { + return "", "", fmt.Errorf("invalid dotenv line: %q", line) + } + + value := parseValue(rawValue) + return key, value, nil +} + +func parseValue(value string) string { + if len(value) >= 2 { + if strings.HasPrefix(value, `"`) && strings.HasSuffix(value, `"`) { + unquoted := strings.TrimSuffix(strings.TrimPrefix(value, `"`), `"`) + unquoted = strings.ReplaceAll(unquoted, `\n`, "\n") + unquoted = strings.ReplaceAll(unquoted, `\r`, "\r") + unquoted = strings.ReplaceAll(unquoted, `\\`, `\`) + unquoted = strings.ReplaceAll(unquoted, `\"`, `"`) + return unquoted + } + if strings.HasPrefix(value, "'") && strings.HasSuffix(value, "'") { + return strings.TrimSuffix(strings.TrimPrefix(value, "'"), "'") + } + } + + return value +} diff --git a/pkg/util/encode/encode.go b/pkg/util/encode/encode.go index b7cbdc1c7..a213e458b 100644 --- a/pkg/util/encode/encode.go +++ b/pkg/util/encode/encode.go @@ -23,7 +23,7 @@ import ( "fmt" "io" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" diff --git a/pkg/util/exec/exec.go b/pkg/util/exec/exec.go index 109118d58..bdcfef08f 100644 --- a/pkg/util/exec/exec.go +++ b/pkg/util/exec/exec.go @@ -21,7 +21,7 @@ import ( "io" "os/exec" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" ) diff --git a/pkg/repository/udmrepo/kopialib/freelist/freelist.go b/pkg/util/freelist/freelist.go similarity index 100% rename from pkg/repository/udmrepo/kopialib/freelist/freelist.go rename to pkg/util/freelist/freelist.go diff --git a/pkg/repository/udmrepo/kopialib/freelist/freelist_test.go b/pkg/util/freelist/freelist_test.go similarity index 100% rename from pkg/repository/udmrepo/kopialib/freelist/freelist_test.go rename to pkg/util/freelist/freelist_test.go diff --git a/pkg/util/kube/node.go b/pkg/util/kube/node.go index ba6853624..3426e508f 100644 --- a/pkg/util/kube/node.go +++ b/pkg/util/kube/node.go @@ -18,7 +18,7 @@ package kube import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/util/kube/node_test.go b/pkg/util/kube/node_test.go index 9f14c380b..612b8f977 100644 --- a/pkg/util/kube/node_test.go +++ b/pkg/util/kube/node_test.go @@ -19,7 +19,7 @@ package kube import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/util/kube/pod.go b/pkg/util/kube/pod.go index 4dc423272..3ced95feb 100644 --- a/pkg/util/kube/pod.go +++ b/pkg/util/kube/pod.go @@ -23,7 +23,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/util/kube/pod_test.go b/pkg/util/kube/pod_test.go index aa8d4db99..1d54071c3 100644 --- a/pkg/util/kube/pod_test.go +++ b/pkg/util/kube/pod_test.go @@ -27,8 +27,8 @@ import ( "testing" "time" + "github.com/cockroachdb/errors" "github.com/google/uuid" - "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" diff --git a/pkg/util/kube/pvc_pv.go b/pkg/util/kube/pvc_pv.go index fa886bf60..7db9df3e4 100644 --- a/pkg/util/kube/pvc_pv.go +++ b/pkg/util/kube/pvc_pv.go @@ -20,11 +20,12 @@ import ( "context" "encoding/json" "fmt" + "maps" "strings" "time" + "github.com/cockroachdb/errors" jsonpatch "github.com/evanphx/json-patch/v5" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -161,6 +162,42 @@ func EnsureDeletePVC(ctx context.Context, pvcGetter corev1client.CoreV1Interface return nil } +func EnsureDeletePV(ctx context.Context, pvGetter corev1client.CoreV1Interface, pvName string, timeout time.Duration) error { + err := pvGetter.PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{}) + if err != nil { + return errors.Wrapf(err, "error to delete pv %s", pvName) + } + + if timeout == 0 { + return nil + } + + var updated *corev1api.PersistentVolume + err = wait.PollUntilContextTimeout(ctx, waitInternal, timeout, true, func(ctx context.Context) (bool, error) { + pv, err := pvGetter.PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + + return false, errors.Wrapf(err, "error to get pv %s", pvName) + } + + updated = pv + return false, nil + }) + + if err != nil { + if errors.Is(err, context.DeadlineExceeded) { + return errors.Errorf("timeout to assure pv %s is deleted, finalizers in pv %v", pvName, updated.Finalizers) + } else { + return errors.Wrapf(err, "error to ensure pv deleted for %s", pvName) + } + } + + return nil +} + // EnsurePVDeleted ensures a PV has been deleted. This function is supposed to be called after EnsureDeletePVC // If timeout is 0, it doesn't wait and return nil func EnsurePVDeleted(ctx context.Context, pvGetter corev1client.CoreV1Interface, pvName string, timeout time.Duration) error { @@ -269,6 +306,86 @@ func ResetPVBinding(ctx context.Context, pvGetter corev1client.CoreV1Interface, return updated, nil } +func RebindPV(ctx context.Context, pvGetter corev1client.CoreV1Interface, pvName string, source *corev1api.PersistentVolume, + pvc *corev1api.PersistentVolumeClaim, policy corev1api.PersistentVolumeReclaimPolicy, fsType string) (*corev1api.PersistentVolume, error) { + if source == nil { + return nil, errors.New("source PV is required to rebind PV") + } + + if pvc == nil { + return nil, errors.New("target PVC is required to rebind PV") + } + + pvLabel := make(map[string]string) + + maps.Copy(pvLabel, source.Labels) + + if pvc.Spec.Selector != nil { + maps.Copy(pvLabel, pvc.Spec.Selector.MatchLabels) + } + + pv := &corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvName, + Labels: pvLabel, + }, + Spec: corev1api.PersistentVolumeSpec{ + Capacity: source.Spec.Capacity, + PersistentVolumeSource: clonePVSource(&source.Spec.PersistentVolumeSource, fsType), + AccessModes: source.Spec.AccessModes, + PersistentVolumeReclaimPolicy: policy, + StorageClassName: source.Spec.StorageClassName, + VolumeMode: pvc.Spec.VolumeMode, + NodeAffinity: source.Spec.NodeAffinity, + VolumeAttributesClassName: source.Spec.VolumeAttributesClassName, + MountOptions: source.Spec.MountOptions, + ClaimRef: &corev1api.ObjectReference{ + Kind: pvc.Kind, + Namespace: pvc.Namespace, + Name: pvc.Name, + }, + }, + } + + return pvGetter.PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}) +} + +func clonePVSource(source *corev1api.PersistentVolumeSource, newFSType string) corev1api.PersistentVolumeSource { + newSource := source.DeepCopy() + + if newSource.CSI != nil && newSource.CSI.VolumeAttributes != nil { + delete(newSource.CSI.VolumeAttributes, "storage.kubernetes.io/csiProvisionerIdentity") + } + + if newFSType != "" { + if newSource.CSI != nil { + newSource.CSI.FSType = newFSType + } else if newSource.AWSElasticBlockStore != nil { + newSource.AWSElasticBlockStore.FSType = newFSType + } else if newSource.AzureDisk != nil { + newSource.AzureDisk.FSType = &newFSType + } else if newSource.VsphereVolume != nil { + newSource.VsphereVolume.FSType = newFSType + } else if newSource.GCEPersistentDisk != nil { + newSource.GCEPersistentDisk.FSType = newFSType + } else if newSource.Cinder != nil { + newSource.Cinder.FSType = newFSType + } else if newSource.ISCSI != nil { + newSource.ISCSI.FSType = newFSType + } else if newSource.RBD != nil { + newSource.RBD.FSType = newFSType + } else if newSource.FC != nil { + newSource.FC.FSType = newFSType + } else if newSource.Local != nil { + newSource.Local.FSType = &newFSType + } else if newSource.FlexVolume != nil { + newSource.FlexVolume.FSType = newFSType + } + } + + return *newSource +} + // SetPVReclaimPolicy sets the specified reclaim policy to a PV func SetPVReclaimPolicy(ctx context.Context, pvGetter corev1client.CoreV1Interface, pv *corev1api.PersistentVolume, policy corev1api.PersistentVolumeReclaimPolicy) (*corev1api.PersistentVolume, error) { @@ -566,22 +683,71 @@ func GetPVAttachedNode(ctx context.Context, pv string, storageClient storagev1.S return "", nil } -func GetPVAttachedNodes(ctx context.Context, pv string, storageClient storagev1.StorageV1Interface) ([]string, error) { +func getPVAttachment(ctx context.Context, pv string, storageClient storagev1.StorageV1Interface) ([]*storagev1api.VolumeAttachment, error) { vaList, err := storageClient.VolumeAttachments().List(ctx, metav1.ListOptions{}) if err != nil { return nil, errors.Wrapf(err, "error listing volumeattachment") } - nodes := []string{} + attachments := []*storagev1api.VolumeAttachment{} for _, va := range vaList.Items { if va.Spec.Source.PersistentVolumeName != nil && *va.Spec.Source.PersistentVolumeName == pv { - nodes = append(nodes, va.Spec.NodeName) + attachments = append(attachments, &va) } } + return attachments, nil +} + +func GetPVAttachedNodes(ctx context.Context, pv string, storageClient storagev1.StorageV1Interface) ([]string, error) { + attachments, err := getPVAttachment(ctx, pv, storageClient) + if err != nil { + return nil, errors.Wrap(err, "error listing volumeattachment") + } + + nodes := []string{} + for _, attach := range attachments { + nodes = append(nodes, attach.Spec.NodeName) + } + return nodes, nil } +func WaitVolumeDetached(ctx context.Context, storageClient storagev1.StorageV1Interface, pv string, timeout time.Duration) error { + attachments, err := getPVAttachment(ctx, pv, storageClient) + if err != nil { + return errors.Wrap(err, "error listing volumeattachment") + } + + err = wait.PollUntilContextTimeout(ctx, waitInternal, timeout, true, func(ctx context.Context) (bool, error) { + left := []*storagev1api.VolumeAttachment{} + for _, attach := range attachments { + if _, err := storageClient.VolumeAttachments().Get(ctx, attach.Name, metav1.GetOptions{}); err == nil { + left = append(left, attach) + } else if !apierrors.IsNotFound(err) { + return false, err // Return the error if it's not a NotFound error + } + } + + if len(left) == 0 { + return true, nil + } + + attachments = left + + return false, nil + }) + + if err != nil { + if errors.Is(err, context.DeadlineExceeded) { + return errors.Errorf("timeout waiting for volume %s to be detached", pv) + } + return errors.Wrapf(err, "error waiting for volume %s to be detached", pv) + } + + return nil +} + func GetVolumeTopology(ctx context.Context, volumeClient corev1client.CoreV1Interface, storageClient storagev1.StorageV1Interface, pvName string, scName string) (*corev1api.NodeSelector, error) { if pvName == "" || scName == "" { return nil, errors.Errorf("invalid parameter, pv %s, sc %s", pvName, scName) @@ -607,3 +773,19 @@ func GetVolumeTopology(ctx context.Context, volumeClient corev1client.CoreV1Inte return pv.Spec.NodeAffinity.Required, nil } + +func GetVolumeModeByPVC(pvc *corev1api.PersistentVolumeClaim) corev1api.PersistentVolumeMode { + if pvc.Spec.VolumeMode != nil { + return *pvc.Spec.VolumeMode + } + + return corev1api.PersistentVolumeFilesystem +} + +func GetVolumeModeByPV(pv *corev1api.PersistentVolume) corev1api.PersistentVolumeMode { + if pv.Spec.VolumeMode != nil { + return *pv.Spec.VolumeMode + } + + return corev1api.PersistentVolumeFilesystem +} diff --git a/pkg/util/kube/pvc_pv_test.go b/pkg/util/kube/pvc_pv_test.go index 63b8e1edd..9b93f2971 100644 --- a/pkg/util/kube/pvc_pv_test.go +++ b/pkg/util/kube/pvc_pv_test.go @@ -20,11 +20,13 @@ import ( "testing" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" @@ -707,7 +709,7 @@ func TestEnsureDeletePVC(t *testing.T) { } } -func TestEnsureDeletePV(t *testing.T) { +func TestEnsurePVDeleted(t *testing.T) { pvObject := &corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", @@ -2049,3 +2051,405 @@ func TestGetVolumeTopology(t *testing.T) { }) } } + +func TestEnsureDeletePV(t *testing.T) { + pvObj := &corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-pv", + }, + } + + tests := []struct { + name string + pvName string + timeout time.Duration + kubeClientObj []runtime.Object + kubeReactors []reactor + expectedErr string + }{ + { + name: "delete error", + pvName: "fake-pv", + kubeReactors: []reactor{ + { + verb: "delete", + resource: "persistentvolumes", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, errors.New("delete error") + }, + }, + }, + expectedErr: "error to delete pv fake-pv: delete error", + }, + { + name: "success without wait", + pvName: "fake-pv", + timeout: 0, + kubeClientObj: []runtime.Object{pvObj}, + }, + { + name: "success with wait", + pvName: "fake-pv", + timeout: time.Second, + kubeReactors: []reactor{ + { + verb: "get", + resource: "persistentvolumes", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, apierrors.NewNotFound(corev1api.Resource("persistentvolumes"), "fake-pv") + }, + }, + }, + kubeClientObj: []runtime.Object{pvObj}, + }, + { + name: "get error during wait", + pvName: "fake-pv", + timeout: time.Millisecond, + kubeClientObj: []runtime.Object{pvObj}, + kubeReactors: []reactor{ + { + verb: "get", + resource: "persistentvolumes", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, errors.New("get error") + }, + }, + }, + expectedErr: "error to ensure pv deleted for fake-pv: error to get pv fake-pv: get error", + }, + { + name: "wait timeout", + pvName: "fake-pv", + timeout: time.Millisecond, + kubeClientObj: []runtime.Object{pvObj}, + kubeReactors: []reactor{ + { + verb: "delete", + resource: "persistentvolumes", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, nil // fake delete, pv will still be in tracker + }, + }, + }, + expectedErr: "timeout to assure pv fake-pv is deleted, finalizers in pv []", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) + for _, reactor := range test.kubeReactors { + fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) + } + + err := EnsureDeletePV(t.Context(), fakeKubeClient.CoreV1(), test.pvName, test.timeout) + if test.expectedErr != "" { + assert.EqualError(t, err, test.expectedErr) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestRebindPV(t *testing.T) { + sourcePV := &corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-pv", + Labels: map[string]string{ + "key1": "val1", + }, + Annotations: map[string]string{ + "anno1": "val1", + KubeAnnBoundByController: "true", + }, + }, + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeSource: corev1api.PersistentVolumeSource{ + CSI: &corev1api.CSIPersistentVolumeSource{ + Driver: "fake-driver", + VolumeHandle: "fake-handle", + }, + }, + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadWriteOnce}, + PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimRetain, + StorageClassName: "fake-sc", + }, + } + + targetPVC := &corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "fake-ns", + Name: "target-pvc", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "key1": "val3", + "key2": "val2", + }, + }, + }, + } + + tests := []struct { + name string + pvName string + sourcePV *corev1api.PersistentVolume + targetPVC *corev1api.PersistentVolumeClaim + kubeClientObj []runtime.Object + kubeReactors []reactor + expectedErr string + expected *corev1api.PersistentVolume + }{ + { + name: "source is nil", + expectedErr: "source PV is required to rebind PV", + }, + { + name: "target pvc is nil", + sourcePV: sourcePV, + expectedErr: "target PVC is required to rebind PV", + }, + { + name: "create error", + pvName: "rebind-pv", + sourcePV: sourcePV, + targetPVC: targetPVC, + kubeReactors: []reactor{ + { + verb: "create", + resource: "persistentvolumes", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, errors.New("create error") + }, + }, + }, + expectedErr: "create error", + }, + { + name: "success", + pvName: "rebind-pv", + sourcePV: sourcePV, + targetPVC: targetPVC, + expected: &corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rebind-pv", + Labels: map[string]string{ + "key1": "val3", + "key2": "val2", + }, + Annotations: nil, + }, + Spec: corev1api.PersistentVolumeSpec{ + Capacity: sourcePV.Spec.Capacity, + PersistentVolumeSource: corev1api.PersistentVolumeSource{ + CSI: &corev1api.CSIPersistentVolumeSource{ + Driver: "fake-driver", + VolumeHandle: "fake-handle", + FSType: "ext4", + }, + }, + AccessModes: sourcePV.Spec.AccessModes, + PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, + StorageClassName: sourcePV.Spec.StorageClassName, + VolumeMode: targetPVC.Spec.VolumeMode, + NodeAffinity: sourcePV.Spec.NodeAffinity, + ClaimRef: &corev1api.ObjectReference{ + Kind: targetPVC.Kind, + Namespace: "fake-ns", + Name: "target-pvc", + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) + for _, reactor := range test.kubeReactors { + fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) + } + + pv, err := RebindPV(t.Context(), fakeKubeClient.CoreV1(), test.pvName, test.sourcePV, test.targetPVC, corev1api.PersistentVolumeReclaimDelete, "ext4") + + if test.expectedErr != "" { + assert.EqualError(t, err, test.expectedErr) + } else { + require.NoError(t, err) + assert.Equal(t, test.expected, pv) + } + }) + } +} + +func TestClonePVSource(t *testing.T) { + fsTypeExt4 := "ext4" + + tests := []struct { + name string + source *corev1api.PersistentVolumeSource + newFSType string + expected corev1api.PersistentVolumeSource + }{ + { + name: "no new fsType", + source: &corev1api.PersistentVolumeSource{ + CSI: &corev1api.CSIPersistentVolumeSource{ + Driver: "fake-driver", + }, + }, + newFSType: "", + expected: corev1api.PersistentVolumeSource{ + CSI: &corev1api.CSIPersistentVolumeSource{ + Driver: "fake-driver", + }, + }, + }, + { + name: "csi source with new fsType", + source: &corev1api.PersistentVolumeSource{ + CSI: &corev1api.CSIPersistentVolumeSource{ + Driver: "fake-driver", + FSType: "ext3", + }, + }, + newFSType: "ext4", + expected: corev1api.PersistentVolumeSource{ + CSI: &corev1api.CSIPersistentVolumeSource{ + Driver: "fake-driver", + FSType: "ext4", + }, + }, + }, + { + name: "awsEBS source with new fsType", + source: &corev1api.PersistentVolumeSource{ + AWSElasticBlockStore: &corev1api.AWSElasticBlockStoreVolumeSource{ + VolumeID: "fake-id", + }, + }, + newFSType: "ext4", + expected: corev1api.PersistentVolumeSource{ + AWSElasticBlockStore: &corev1api.AWSElasticBlockStoreVolumeSource{ + VolumeID: "fake-id", + FSType: "ext4", + }, + }, + }, + { + name: "azureDisk source with new fsType", + source: &corev1api.PersistentVolumeSource{ + AzureDisk: &corev1api.AzureDiskVolumeSource{ + DiskName: "fake-disk", + }, + }, + newFSType: "ext4", + expected: corev1api.PersistentVolumeSource{ + AzureDisk: &corev1api.AzureDiskVolumeSource{ + DiskName: "fake-disk", + FSType: &fsTypeExt4, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + actual := clonePVSource(test.source, test.newFSType) + assert.Equal(t, test.expected, actual) + }) + } +} + +func TestWaitVolumeDetached(t *testing.T) { + pvName := "test-pv" + otherPVName := "other-pv" + + volAttach1 := &storagev1api.VolumeAttachment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "va-1", + }, + Spec: storagev1api.VolumeAttachmentSpec{ + Source: storagev1api.VolumeAttachmentSource{ + PersistentVolumeName: &pvName, + }, + }, + } + + volAttach2 := &storagev1api.VolumeAttachment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "va-2", + }, + Spec: storagev1api.VolumeAttachmentSpec{ + Source: storagev1api.VolumeAttachmentSource{ + PersistentVolumeName: &otherPVName, + }, + }, + } + + tests := []struct { + name string + kubeReactors []reactor + timeout time.Duration + expectedErr string + storageObjs []runtime.Object + }{ + { + name: "no volume attachments", + timeout: time.Second, + }, + { + name: "volume attachments exist and deleted", + timeout: time.Second, + storageObjs: []runtime.Object{volAttach1, volAttach2}, + kubeReactors: []reactor{ + { + verb: "get", + resource: "volumeattachments", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, apierrors.NewNotFound(schema.GroupResource{Group: "storage.k8s.io", Resource: "volumeattachments"}, "va-1") + }, + }, + }, + }, + { + name: "volume attachments exist and not deleted", + timeout: 2 * time.Millisecond, + storageObjs: []runtime.Object{volAttach1, volAttach2}, + expectedErr: "timeout waiting for volume test-pv to be detached", + }, + { + name: "get returns error", + timeout: time.Second, + storageObjs: []runtime.Object{volAttach1, volAttach2}, + kubeReactors: []reactor{ + { + verb: "get", + resource: "volumeattachments", + reactorFunc: func(action clientTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, errors.New("fake-error") + }, + }, + }, + expectedErr: "error waiting for volume test-pv to be detached: fake-error", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeKubeClient := fake.NewSimpleClientset(test.storageObjs...) + for _, reactor := range test.kubeReactors { + fakeKubeClient.Fake.PrependReactor(reactor.verb, reactor.resource, reactor.reactorFunc) + } + err := WaitVolumeDetached(t.Context(), fakeKubeClient.StorageV1(), pvName, test.timeout) + if test.expectedErr != "" { + assert.EqualError(t, err, test.expectedErr) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/pkg/util/kube/resource_requirements.go b/pkg/util/kube/resource_requirements.go index 12cf7a79c..5c2abecfa 100644 --- a/pkg/util/kube/resource_requirements.go +++ b/pkg/util/kube/resource_requirements.go @@ -17,7 +17,7 @@ limitations under the License. package kube import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" diff --git a/pkg/util/kube/secrets.go b/pkg/util/kube/secrets.go index e9cb4c04c..f1d19b84e 100644 --- a/pkg/util/kube/secrets.go +++ b/pkg/util/kube/secrets.go @@ -19,7 +19,7 @@ package kube import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" kbclient "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/pkg/util/kube/security_context.go b/pkg/util/kube/security_context.go index 1fd911649..9b62b32fb 100644 --- a/pkg/util/kube/security_context.go +++ b/pkg/util/kube/security_context.go @@ -19,7 +19,7 @@ package kube import ( "strconv" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" "sigs.k8s.io/yaml" ) diff --git a/pkg/util/kube/utils.go b/pkg/util/kube/utils.go index d93effd7f..d76dad4a3 100644 --- a/pkg/util/kube/utils.go +++ b/pkg/util/kube/utils.go @@ -23,7 +23,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" diff --git a/pkg/util/kube/utils_test.go b/pkg/util/kube/utils_test.go index df23903a0..23db12a41 100644 --- a/pkg/util/kube/utils_test.go +++ b/pkg/util/kube/utils_test.go @@ -730,3 +730,95 @@ func TestVerifyJsonConfigs(t *testing.T) { }) } } + +func TestGetVolumeModeByPVC(t *testing.T) { + modeFilesystem := corev1api.PersistentVolumeFilesystem + modeBlock := corev1api.PersistentVolumeBlock + + tests := []struct { + name string + pvc *corev1api.PersistentVolumeClaim + expected corev1api.PersistentVolumeMode + }{ + { + name: "nil VolumeMode returns Filesystem", + pvc: &corev1api.PersistentVolumeClaim{ + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeMode: nil, + }, + }, + expected: corev1api.PersistentVolumeFilesystem, + }, + { + name: "Filesystem VolumeMode returns Filesystem", + pvc: &corev1api.PersistentVolumeClaim{ + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeMode: &modeFilesystem, + }, + }, + expected: corev1api.PersistentVolumeFilesystem, + }, + { + name: "Block VolumeMode returns Block", + pvc: &corev1api.PersistentVolumeClaim{ + Spec: corev1api.PersistentVolumeClaimSpec{ + VolumeMode: &modeBlock, + }, + }, + expected: corev1api.PersistentVolumeBlock, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + actual := GetVolumeModeByPVC(test.pvc) + assert.Equal(t, test.expected, actual) + }) + } +} + +func TestGetVolumeModeByPV(t *testing.T) { + modeFilesystem := corev1api.PersistentVolumeFilesystem + modeBlock := corev1api.PersistentVolumeBlock + + tests := []struct { + name string + pv *corev1api.PersistentVolume + expected corev1api.PersistentVolumeMode + }{ + { + name: "nil VolumeMode returns Filesystem", + pv: &corev1api.PersistentVolume{ + Spec: corev1api.PersistentVolumeSpec{ + VolumeMode: nil, + }, + }, + expected: corev1api.PersistentVolumeFilesystem, + }, + { + name: "Filesystem VolumeMode returns Filesystem", + pv: &corev1api.PersistentVolume{ + Spec: corev1api.PersistentVolumeSpec{ + VolumeMode: &modeFilesystem, + }, + }, + expected: corev1api.PersistentVolumeFilesystem, + }, + { + name: "Block VolumeMode returns Block", + pv: &corev1api.PersistentVolume{ + Spec: corev1api.PersistentVolumeSpec{ + VolumeMode: &modeBlock, + }, + }, + expected: corev1api.PersistentVolumeBlock, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + actual := GetVolumeModeByPV(test.pv) + assert.Equal(t, test.expected, actual) + }) + } +} diff --git a/pkg/util/logging/dual_mode_logger.go b/pkg/util/logging/dual_mode_logger.go index f5533c8eb..efcfceb3a 100644 --- a/pkg/util/logging/dual_mode_logger.go +++ b/pkg/util/logging/dual_mode_logger.go @@ -21,7 +21,7 @@ import ( "io" "os" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" ) diff --git a/pkg/util/logging/error_location_hook.go b/pkg/util/logging/error_location_hook.go index 5246a8318..b89236ecb 100644 --- a/pkg/util/logging/error_location_hook.go +++ b/pkg/util/logging/error_location_hook.go @@ -21,7 +21,7 @@ import ( "strconv" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors/errbase" "github.com/sirupsen/logrus" ) @@ -32,7 +32,7 @@ const ( // ErrorLocationHook is a logrus hook that attaches error location information // to log entries if an error is being logged and it has stack-trace information -// (i.e. if it originates from or is wrapped by github.com/pkg/errors, or if it +// (for example, if it carries a stack trace from wrapped errors), or if it // implements the errorLocationer interface, like errors returned from plugins // typically do). type ErrorLocationHook struct{} @@ -90,8 +90,8 @@ type LocationInfo struct { } // GetFrameLocationInfo returns the location of a frame. -func GetFrameLocationInfo(frame errors.Frame) LocationInfo { - // see https://godoc.org/github.com/pkg/errors#Frame.Format for +func GetFrameLocationInfo(frame errbase.StackFrame) LocationInfo { + // see https://pkg.go.dev/github.com/cockroachdb/errors#Frame.Format for // details on formatting verbs functionNameAndFileAndLine := fmt.Sprintf("%+v", frame) @@ -121,7 +121,7 @@ type errorLocationer interface { type stackTracer interface { error - StackTrace() errors.StackTrace + StackTrace() errbase.StackTrace } type causer interface { diff --git a/pkg/util/logging/error_location_hook_test.go b/pkg/util/logging/error_location_hook_test.go index f6f230c02..a76937e42 100644 --- a/pkg/util/logging/error_location_hook_test.go +++ b/pkg/util/logging/error_location_hook_test.go @@ -20,7 +20,7 @@ import ( "errors" "testing" - pkgerrs "github.com/pkg/errors" + pkgerrs "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/pkg/util/logging/log_merge_hook.go b/pkg/util/logging/log_merge_hook.go index b993cb38a..385453925 100644 --- a/pkg/util/logging/log_merge_hook.go +++ b/pkg/util/logging/log_merge_hook.go @@ -21,7 +21,7 @@ import ( "io" "os" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" ) diff --git a/pkg/util/logging/log_merge_hook_test.go b/pkg/util/logging/log_merge_hook_test.go index 43b5ae1cb..d4e7870c4 100644 --- a/pkg/util/logging/log_merge_hook_test.go +++ b/pkg/util/logging/log_merge_hook_test.go @@ -21,7 +21,7 @@ import ( "os" "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/pkg/util/podvolume/pod_volume.go b/pkg/util/podvolume/pod_volume.go index 7c7e0f9c4..3c9ad2127 100644 --- a/pkg/util/podvolume/pod_volume.go +++ b/pkg/util/podvolume/pod_volume.go @@ -21,7 +21,7 @@ import ( "strings" "sync" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" crclient "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/pkg/util/results/result_test.go b/pkg/util/results/result_test.go index 26017c35f..85f94364c 100644 --- a/pkg/util/results/result_test.go +++ b/pkg/util/results/result_test.go @@ -19,7 +19,7 @@ package results import ( "testing" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" ) diff --git a/pkg/util/third_party.go b/pkg/util/third_party.go index 400c7a898..81b964454 100644 --- a/pkg/util/third_party.go +++ b/pkg/util/third_party.go @@ -31,4 +31,6 @@ var ThirdPartyTolerations = []string{ const ( VSphereCNSFastCloneAnno = "csi.vsphere.volume/fast-provisioning" + VSphereCNSSnapshotAnno = "csi.vsphere.volume/snapshot" + VSphereCNSChangeIDAnno = "csi.vsphere.volume/change-id" ) diff --git a/site/content/docs/main/api-types/backup.md b/site/content/docs/main/api-types/backup.md index 3bad516e3..30aeb1180 100644 --- a/site/content/docs/main/api-types/backup.md +++ b/site/content/docs/main/api-types/backup.md @@ -178,6 +178,13 @@ spec: # processed. Only "exec" hooks are supported. post: # Same content as pre above. + # BackupType specifies how volume data is backed up, with possible values including Full and Incremental. + # BackupType is optional. If it's not set, it will default to Full. + # BackupType is only meaningful for data mover backup, including CSI snapshot fs backup, CSI snapshot block backup, and fs backup. + # For CSI only backup and Velero native backup, backupType doesn't take effect. + # Full means data mover will forcefully upload all data in volumes. + # Incremental means data mover will only upload data change since last snapshot. + backupType: Full # Status about the Backup. Users should not set any data here. status: # The version of this Backup. The only version supported is 1. diff --git a/site/content/docs/main/resource-filtering.md b/site/content/docs/main/resource-filtering.md index cbfdb2816..88584b362 100644 --- a/site/content/docs/main/resource-filtering.md +++ b/site/content/docs/main/resource-filtering.md @@ -287,6 +287,11 @@ The policies YAML config file would look like this: # pvc matches specific phase(s) pvcPhase: - Pending + # pvc matches specific volume mode + pvcVolumeMode: Block + # pvc matches specific access mode(s) + pvcAccessModes: + - ReadWriteOnce action: type: skip - conditions: @@ -380,6 +385,8 @@ Currently, Velero supports the volume attributes listed below: - storageClass: matching volumes those with specified `storageClass`, such as `gp2`, `ebs-sc` in eks - volume sources: matching volumes that used specified volume sources. Currently we support nfs or csi backend volume source - pvcPhase: matching volumes based on the phase of their associated PVCs (Pending, Bound, Lost) +- pvcVolumeMode: matching volumes based on the volume mode of their associated PVCs (Filesystem, Block) +- pvcAccessModes: matching volumes based on the access modes of their associated PVCs (ReadWriteOnce, ReadOnlyMany, ReadWriteMany, ReadWriteOncePod). All configured access modes must be present on the PVC. Velero supported conditions and format listed below: - capacity @@ -521,6 +528,72 @@ Velero supported conditions and format listed below: type: skip ``` +- pvc VolumeMode + + This condition filters PVC-backed volumes based on the volume mode of their associated PVCs. The condition is specified as a single volume mode to match. The volume matches this condition if the PVC's volume mode exactly matches the configured value. Matching is case-sensitive, so `block` does not match `Block`. Supported volume modes are: `Filesystem` and `Block`. If `pvcVolumeMode` is omitted from a policy, volume mode is not restricted. Non-PVC volumes, such as `emptyDir`, `configMap`, or inline volumes without an associated PVC, do not match policies that require this condition. + ```yaml + pvcVolumeMode: Block + ``` + + Some examples: + - Skip Block PVCs: Skip backup of volumes whose associated PVC uses `Block` volume mode. + ```yaml + volumePolicies: + - conditions: + pvcVolumeMode: Block + action: + type: skip + ``` + - Combine with other conditions: You can combine PVC volume mode conditions with other conditions like PVC phase, storage class, or labels. + ```yaml + volumePolicies: + - conditions: + pvcVolumeMode: Block + pvcPhase: + - Bound + action: + type: snapshot + ``` + +- pvc AccessModes + + This condition filters PVC-backed volumes based on the access modes of their associated PVCs. The condition is specified as a list of access modes to match. The volume matches this condition only if the PVC has all of the access modes in the list. Matching is case-sensitive, so `readwriteonce` does not match `ReadWriteOnce`. Supported access modes are: `ReadWriteOnce`, `ReadOnlyMany`, `ReadWriteMany`, and `ReadWriteOncePod`. Non-PVC volumes, such as `emptyDir`, `configMap`, or inline volumes without an associated PVC, do not match policies that require this condition. + ```yaml + pvcAccessModes: + - ReadWriteOnce + ``` + + Some examples: + - Skip ReadWriteOnce PVCs: Skip backup of volumes whose associated PVC includes the `ReadWriteOnce` access mode. + ```yaml + volumePolicies: + - conditions: + pvcAccessModes: + - ReadWriteOnce + action: + type: skip + ``` + - Match multiple access modes: Apply an action to volumes whose associated PVC includes both `ReadOnlyMany` and `ReadWriteMany`. + ```yaml + volumePolicies: + - conditions: + pvcAccessModes: + - ReadOnlyMany + - ReadWriteMany + action: + type: snapshot + ``` + - Combine with other conditions: You can combine PVC access mode conditions with other conditions like PVC volume mode, PVC phase, storage class, or labels. + ```yaml + volumePolicies: + - conditions: + pvcAccessModes: + - ReadWriteOnce + pvcVolumeMode: Block + action: + type: snapshot + ``` + ### Resource policies rules @@ -631,3 +704,85 @@ volumePolicies: 3. The outcome would be that velero would perform `fs-backup` operation on both the volumes - `fs-backup` on `Volume 1` because `Volume 1` satisfies the criteria for `fs-backup` action. - Also, for Volume 2 as no matching action was found so legacy approach will be used as a fallback option for this volume (`fs-backup` operation will be done as `defaultVolumesToFSBackup: true` is specified by the user). + +### Global backup volume policies + +Resource policies (volume policies) are normally opt-in per backup via `--resource-policies-configmap`. An administrator can instead configure a cluster-wide baseline that applies to **every** backup by starting the Velero server with the `--global-backup-volume-policies-configmap` flag, pointing at a ConfigMap in the Velero install namespace: + +```bash +velero server --global-backup-volume-policies-configmap global-volume-policy +``` + +The ConfigMap uses the exact same format as a per-backup resource policies ConfigMap (a single data key holding a `ResourcePolicies` YAML document): + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: global-volume-policy + namespace: velero +data: + policies.yaml: | + version: v1 + volumePolicies: + - conditions: + storageClass: + - gp2 + action: + type: skip +``` + +#### Behavior + +- **Only `volumePolicies` apply globally.** If the global ConfigMap contains `includeExcludePolicy`, `clusterScopedFilterPolicy`, or `namespacedFilterPolicies`, those sections are ignored and a warning is logged. Those filters are tied to a specific backup use case, so they remain per-backup only. +- **Merge semantics.** When a backup runs, the effective `volumePolicies` list is the backup-level policies followed by the global policies: + + ``` + merged.volumePolicies = backup.volumePolicies ++ global.volumePolicies + ``` + + Because the first matching policy wins, a backup can override the global baseline for a specific volume while still inheriting every global rule it does not override. If a backup references no resource policy, the global policy applies on its own. +- **Validation.** The global ConfigMap is validated at server startup (the server fails to start if it is missing or invalid) and again on each backup (a backup whose global policy has become missing or invalid is moved to the `FailedValidation` phase). + +#### Example + +Global policy (`--global-backup-volume-policies-configmap=global-volume-policy`): skip `gp2` volumes. + +```yaml +version: v1 +volumePolicies: + - conditions: + storageClass: + - gp2 + action: + type: skip +``` + +Backup-level policy (`--resource-policies-configmap backup01`): `fs-backup` NFS volumes. + +```yaml +version: v1 +volumePolicies: + - conditions: + nfs: {} + action: + type: fs-backup +``` + +Effective (merged) policy used for the backup — backup rules first, then global: + +```yaml +version: v1 +volumePolicies: + - conditions: + nfs: {} + action: + type: fs-backup + - conditions: + storageClass: + - gp2 + action: + type: skip +``` + +When a global policy contributes to a backup, `velero backup describe` surfaces the contributing ConfigMap under a `Global volume policies` section. diff --git a/site/content/docs/main/restore-reference.md b/site/content/docs/main/restore-reference.md index b14a1dac9..82bb9d505 100644 --- a/site/content/docs/main/restore-reference.md +++ b/site/content/docs/main/restore-reference.md @@ -27,7 +27,7 @@ The following is an overview of Velero's restore process that starts after you r 1. The Velero client makes a call to the Kubernetes API server to create a [`Restore`](api-types/restore.md) object. -1. The `RestoreController` notices the new Restore object and performs validation. +1. The `RestoreController` notices the new `Restore` object and performs validation. This includes verifying that the referenced backup is in a usable phase. Only backups in `Completed` or `PartiallyFailed` phase are accepted as restore sources. 1. The `RestoreController` fetches basic information about the backup being restored, like the [BackupStorageLocation](locations.md) (BSL). It also fetches a tarball of the cluster resources in the backup, any volumes that will be restored using File System Backup, and any volume snapshots to be restored. @@ -63,7 +63,7 @@ The following is an overview of Velero's restore process that starts after you r 1. Once the resource is created on the target cluster, Velero may take some additional steps or wait for additional processes to complete before moving onto the next resource to restore. * If the resource is a Pod, the `RestoreController` will execute any [Restore Hooks](restore-hooks.md) and wait for the hook to finish. - * If the resource is a PV restored by File System Backup, the `RestoreController` waits for File System Backup’s restore to complete. The `RestoreController` sets a timeout for any resources restored with File System Backup during a restore. The default timeout is 4 hours, but you can configure this be setting using `--fs-backup-timeout` restore option. + * If the resource is a PV restored by File System Backup, the `RestoreController` starts a File System Backup’s restore. Velero continues to restore more resources while the file system restore is running. The `RestoreController` sets a timeout for any resources restored with File System Backup during a restore. The default timeout is 4 hours, but you can configure this be setting using `--fs-backup-timeout` restore option. The restore will not finish until either the file system restore is completed or times out. * If the resource is a Custom Resource Definition, the `RestoreController` waits for its availability in the cluster. The timeout is 1 minute. If any failures happen finishing these steps, the `RestoreController` will log an error in the restore result and will continue restoring. @@ -78,26 +78,41 @@ By default, Velero will restore resources in the following order: * VolumeSnapshotClass * VolumeSnapshotContents * VolumeSnapshots +* DataUploads * PersistentVolumes * PersistentVolumeClaims +* ClusterRoles +* Roles +* ServiceAccounts +* ClusterRoleBindings +* RoleBindings * Secrets * ConfigMaps -* ServiceAccounts * LimitRanges +* PriorityClasses * Pods * ReplicaSets +* ClusterClasses +* Endpoints +* Services +* ClusterBootstraps * Clusters * ClusterResourceSets +* Apps (apps.kappctrl.k14s.io) +* PackageInstalls -It's recommended that you use the default order for your restores. You are able to customize this order if you need to by setting the `--restore-resource-priorities` flag on the Velero server and specifying a different resource order. This customized order will apply to all future restores. You don't have to specify all resources in the `--restore-resource-priorities` flag. Velero will append resources not listed to the end of your customized list in alphabetical order. +It's recommended that you use the default order for your restores. You are able to customize this order if you need to by setting the `--restore-resource-priorities` flag on the Velero server and specifying a different resource order. This customized order will apply to all future restores. You don't have to specify all resources in the `--restore-resource-priorities` flag. The priority list contains two parts which are split by the `-` element: resources before the `-` element are restored first as high priorities, resources after the `-` element are restored last as low priorities, and any resource not in the list will be restored alphabetically between the high and low priorities. ```shell velero server \ --restore-resource-priorities=customresourcedefinitions,namespaces,storageclasses,\ volumesnapshotclass.snapshot.storage.k8s.io,volumesnapshotcontents.snapshot.storage.k8s.io,\ -volumesnapshots.snapshot.storage.k8s.io,persistentvolumes,persistentvolumeclaims,secrets,\ -configmaps,serviceaccounts,limitranges,pods,replicasets.apps,clusters.cluster.x-k8s.io,\ -clusterresourcesets.addons.cluster.x-k8s.io +volumesnapshots.snapshot.storage.k8s.io,datauploads.velero.io,persistentvolumes,\ +persistentvolumeclaims,clusterroles,roles,serviceaccounts,clusterrolebindings,rolebindings,\ +secrets,configmaps,limitranges,priorityclasses,pods,replicasets.apps,\ +clusterclasses.cluster.x-k8s.io,endpoints,services,-,clusterbootstraps.run.tanzu.vmware.com,\ +clusters.cluster.x-k8s.io,clusterresourcesets.addons.cluster.x-k8s.io,apps.kappctrl.k14s.io,\ +packageinstalls.packaging.carvel.dev ``` diff --git a/site/content/docs/main/volume-group-snapshots.md b/site/content/docs/main/volume-group-snapshots.md index 95cf33ff9..e62a75624 100644 --- a/site/content/docs/main/volume-group-snapshots.md +++ b/site/content/docs/main/volume-group-snapshots.md @@ -122,7 +122,15 @@ Before using Volume Group Snapshots with Velero, ensure your environment meets t - Kubernetes 1.20+ (when VolumeGroupSnapshot API was introduced) - Check your version: `kubectl version --short` -### 2. VolumeGroupSnapshot CRDs +### 2. External-Snapshotter Version +Velero 1.18.1+ uses the VolumeGroupSnapshot v1beta2 API. This requires external-snapshotter v8.2.0 or later, which introduced v1beta2 support. Older versions of external-snapshotter only ship v1beta1 CRDs and are not compatible with VGS in Velero 1.18.1+. + +```bash +# Check your external-snapshotter CRD version +kubectl get crd volumegroupsnapshotcontents.groupsnapshot.storage.k8s.io -o jsonpath='{.spec.versions[*].name}' +``` + +### 3. VolumeGroupSnapshot CRDs Check the Volume Group Snapshot CRDs on your cluster: ```bash @@ -130,7 +138,7 @@ Check the Volume Group Snapshot CRDs on your cluster: kubectl get crd | grep volumegroup ``` -### 3. CSI Driver Support +### 4. CSI Driver Support Verify your CSI driver supports Volume Group Snapshots: ```bash @@ -141,7 +149,7 @@ kubectl get volumegroupsnapshotclass kubectl describe csidriver ebs.csi.aws.com ``` -### 4. VolumeGroupSnapshotClass Configuration +### 5. VolumeGroupSnapshotClass Configuration Ensure a VolumeGroupSnapshotClass exists for your storage and is properly labeled for Velero discovery: ```bash diff --git a/site/content/docs/v1.17/restore-reference.md b/site/content/docs/v1.17/restore-reference.md index b062175cc..92a5f538b 100644 --- a/site/content/docs/v1.17/restore-reference.md +++ b/site/content/docs/v1.17/restore-reference.md @@ -78,26 +78,41 @@ By default, Velero will restore resources in the following order: * VolumeSnapshotClass * VolumeSnapshotContents * VolumeSnapshots +* DataUploads * PersistentVolumes * PersistentVolumeClaims +* ClusterRoles +* Roles +* ServiceAccounts +* ClusterRoleBindings +* RoleBindings * Secrets * ConfigMaps -* ServiceAccounts * LimitRanges +* PriorityClasses * Pods * ReplicaSets +* ClusterClasses +* Endpoints +* Services +* ClusterBootstraps * Clusters * ClusterResourceSets +* Apps (apps.kappctrl.k14s.io) +* PackageInstalls -It's recommended that you use the default order for your restores. You are able to customize this order if you need to by setting the `--restore-resource-priorities` flag on the Velero server and specifying a different resource order. This customized order will apply to all future restores. You don't have to specify all resources in the `--restore-resource-priorities` flag. Velero will append resources not listed to the end of your customized list in alphabetical order. +It's recommended that you use the default order for your restores. You are able to customize this order if you need to by setting the `--restore-resource-priorities` flag on the Velero server and specifying a different resource order. This customized order will apply to all future restores. You don't have to specify all resources in the `--restore-resource-priorities` flag. The priority list contains two parts which are split by the `-` element: resources before the `-` element are restored first as high priorities, resources after the `-` element are restored last as low priorities, and any resource not in the list will be restored alphabetically between the high and low priorities. ```shell velero server \ --restore-resource-priorities=customresourcedefinitions,namespaces,storageclasses,\ volumesnapshotclass.snapshot.storage.k8s.io,volumesnapshotcontents.snapshot.storage.k8s.io,\ -volumesnapshots.snapshot.storage.k8s.io,persistentvolumes,persistentvolumeclaims,secrets,\ -configmaps,serviceaccounts,limitranges,pods,replicasets.apps,clusters.cluster.x-k8s.io,\ -clusterresourcesets.addons.cluster.x-k8s.io +volumesnapshots.snapshot.storage.k8s.io,datauploads.velero.io,persistentvolumes,\ +persistentvolumeclaims,clusterroles,roles,serviceaccounts,clusterrolebindings,rolebindings,\ +secrets,configmaps,limitranges,priorityclasses,pods,replicasets.apps,\ +clusterclasses.cluster.x-k8s.io,endpoints,services,-,clusterbootstraps.run.tanzu.vmware.com,\ +clusters.cluster.x-k8s.io,clusterresourcesets.addons.cluster.x-k8s.io,apps.kappctrl.k14s.io,\ +packageinstalls.packaging.carvel.dev ``` diff --git a/site/content/docs/v1.18/restore-reference.md b/site/content/docs/v1.18/restore-reference.md index b14a1dac9..2726f8a9f 100644 --- a/site/content/docs/v1.18/restore-reference.md +++ b/site/content/docs/v1.18/restore-reference.md @@ -78,26 +78,41 @@ By default, Velero will restore resources in the following order: * VolumeSnapshotClass * VolumeSnapshotContents * VolumeSnapshots +* DataUploads * PersistentVolumes * PersistentVolumeClaims +* ClusterRoles +* Roles +* ServiceAccounts +* ClusterRoleBindings +* RoleBindings * Secrets * ConfigMaps -* ServiceAccounts * LimitRanges +* PriorityClasses * Pods * ReplicaSets +* ClusterClasses +* Endpoints +* Services +* ClusterBootstraps * Clusters * ClusterResourceSets +* Apps (apps.kappctrl.k14s.io) +* PackageInstalls -It's recommended that you use the default order for your restores. You are able to customize this order if you need to by setting the `--restore-resource-priorities` flag on the Velero server and specifying a different resource order. This customized order will apply to all future restores. You don't have to specify all resources in the `--restore-resource-priorities` flag. Velero will append resources not listed to the end of your customized list in alphabetical order. +It's recommended that you use the default order for your restores. You are able to customize this order if you need to by setting the `--restore-resource-priorities` flag on the Velero server and specifying a different resource order. This customized order will apply to all future restores. You don't have to specify all resources in the `--restore-resource-priorities` flag. The priority list contains two parts which are split by the `-` element: resources before the `-` element are restored first as high priorities, resources after the `-` element are restored last as low priorities, and any resource not in the list will be restored alphabetically between the high and low priorities. ```shell velero server \ --restore-resource-priorities=customresourcedefinitions,namespaces,storageclasses,\ volumesnapshotclass.snapshot.storage.k8s.io,volumesnapshotcontents.snapshot.storage.k8s.io,\ -volumesnapshots.snapshot.storage.k8s.io,persistentvolumes,persistentvolumeclaims,secrets,\ -configmaps,serviceaccounts,limitranges,pods,replicasets.apps,clusters.cluster.x-k8s.io,\ -clusterresourcesets.addons.cluster.x-k8s.io +volumesnapshots.snapshot.storage.k8s.io,datauploads.velero.io,persistentvolumes,\ +persistentvolumeclaims,clusterroles,roles,serviceaccounts,clusterrolebindings,rolebindings,\ +secrets,configmaps,limitranges,priorityclasses,pods,replicasets.apps,\ +clusterclasses.cluster.x-k8s.io,endpoints,services,-,clusterbootstraps.run.tanzu.vmware.com,\ +clusters.cluster.x-k8s.io,clusterresourcesets.addons.cluster.x-k8s.io,apps.kappctrl.k14s.io,\ +packageinstalls.packaging.carvel.dev ``` diff --git a/site/content/docs/v1.18/volume-group-snapshots.md b/site/content/docs/v1.18/volume-group-snapshots.md index 95cf33ff9..e62a75624 100644 --- a/site/content/docs/v1.18/volume-group-snapshots.md +++ b/site/content/docs/v1.18/volume-group-snapshots.md @@ -122,7 +122,15 @@ Before using Volume Group Snapshots with Velero, ensure your environment meets t - Kubernetes 1.20+ (when VolumeGroupSnapshot API was introduced) - Check your version: `kubectl version --short` -### 2. VolumeGroupSnapshot CRDs +### 2. External-Snapshotter Version +Velero 1.18.1+ uses the VolumeGroupSnapshot v1beta2 API. This requires external-snapshotter v8.2.0 or later, which introduced v1beta2 support. Older versions of external-snapshotter only ship v1beta1 CRDs and are not compatible with VGS in Velero 1.18.1+. + +```bash +# Check your external-snapshotter CRD version +kubectl get crd volumegroupsnapshotcontents.groupsnapshot.storage.k8s.io -o jsonpath='{.spec.versions[*].name}' +``` + +### 3. VolumeGroupSnapshot CRDs Check the Volume Group Snapshot CRDs on your cluster: ```bash @@ -130,7 +138,7 @@ Check the Volume Group Snapshot CRDs on your cluster: kubectl get crd | grep volumegroup ``` -### 3. CSI Driver Support +### 4. CSI Driver Support Verify your CSI driver supports Volume Group Snapshots: ```bash @@ -141,7 +149,7 @@ kubectl get volumegroupsnapshotclass kubectl describe csidriver ebs.csi.aws.com ``` -### 4. VolumeGroupSnapshotClass Configuration +### 5. VolumeGroupSnapshotClass Configuration Ensure a VolumeGroupSnapshotClass exists for your storage and is properly labeled for Velero discovery: ```bash diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index e35b94b53..a9ee3ce5f 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -22,10 +22,10 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" . "github.com/vmware-tanzu/velero/test" . "github.com/vmware-tanzu/velero/test/util/k8s" diff --git a/test/e2e/basic/api-group/enable_api_group_versions.go b/test/e2e/basic/api-group/enable_api_group_versions.go index 13ee3a39e..264e6f2ba 100644 --- a/test/e2e/basic/api-group/enable_api_group_versions.go +++ b/test/e2e/basic/api-group/enable_api_group_versions.go @@ -27,10 +27,10 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/test/e2e/basic/backup-volume-info/base.go b/test/e2e/basic/backup-volume-info/base.go index 2cd574b5c..7a1ff39e1 100644 --- a/test/e2e/basic/backup-volume-info/base.go +++ b/test/e2e/basic/backup-volume-info/base.go @@ -22,9 +22,9 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" . "github.com/vmware-tanzu/velero/test" diff --git a/test/e2e/basic/resources-check/namespaces.go b/test/e2e/basic/resources-check/namespaces.go index 922e4ed07..6b1a577f4 100644 --- a/test/e2e/basic/resources-check/namespaces.go +++ b/test/e2e/basic/resources-check/namespaces.go @@ -22,7 +22,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "github.com/vmware-tanzu/velero/test/e2e/test" diff --git a/test/e2e/basic/resources-check/namespaces_annotation.go b/test/e2e/basic/resources-check/namespaces_annotation.go index e698d4818..fd8012127 100644 --- a/test/e2e/basic/resources-check/namespaces_annotation.go +++ b/test/e2e/basic/resources-check/namespaces_annotation.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" . "github.com/vmware-tanzu/velero/test/e2e/test" . "github.com/vmware-tanzu/velero/test/util/k8s" diff --git a/test/e2e/basic/resources-check/rbac.go b/test/e2e/basic/resources-check/rbac.go index b79c3615f..0d40f00e6 100644 --- a/test/e2e/basic/resources-check/rbac.go +++ b/test/e2e/basic/resources-check/rbac.go @@ -36,8 +36,8 @@ import ( "fmt" "strings" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" - "github.com/pkg/errors" . "github.com/vmware-tanzu/velero/test/e2e/test" . "github.com/vmware-tanzu/velero/test/util/k8s" diff --git a/test/e2e/nodeagentconfig/cache_pvc.go b/test/e2e/nodeagentconfig/cache_pvc.go index 9104946c7..3c12b0e2c 100644 --- a/test/e2e/nodeagentconfig/cache_pvc.go +++ b/test/e2e/nodeagentconfig/cache_pvc.go @@ -22,8 +22,8 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" . "github.com/onsi/gomega" - "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" diff --git a/test/e2e/nodeagentconfig/node-agent-config.go b/test/e2e/nodeagentconfig/node-agent-config.go index 5266bffc5..3eb508234 100644 --- a/test/e2e/nodeagentconfig/node-agent-config.go +++ b/test/e2e/nodeagentconfig/node-agent-config.go @@ -23,8 +23,8 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" . "github.com/onsi/gomega" - "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/test/e2e/pv-backup/pv-backup-filter.go b/test/e2e/pv-backup/pv-backup-filter.go index 5a6730551..510c686db 100644 --- a/test/e2e/pv-backup/pv-backup-filter.go +++ b/test/e2e/pv-backup/pv-backup-filter.go @@ -6,9 +6,9 @@ import ( "strings" "unicode" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" . "github.com/vmware-tanzu/velero/test" . "github.com/vmware-tanzu/velero/test/e2e/test" diff --git a/test/e2e/repomaintenance/repo_maintenance_config.go b/test/e2e/repomaintenance/repo_maintenance_config.go index c0092c4bf..7c53655b5 100644 --- a/test/e2e/repomaintenance/repo_maintenance_config.go +++ b/test/e2e/repomaintenance/repo_maintenance_config.go @@ -22,8 +22,8 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" . "github.com/onsi/gomega" - "github.com/pkg/errors" batchv1api "k8s.io/api/batch/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/test/e2e/resource-filtering/base.go b/test/e2e/resource-filtering/base.go index f4070d9e7..e36de7145 100644 --- a/test/e2e/resource-filtering/base.go +++ b/test/e2e/resource-filtering/base.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "github.com/vmware-tanzu/velero/test/e2e/test" diff --git a/test/e2e/resource-filtering/exclude_label.go b/test/e2e/resource-filtering/exclude_label.go index 695d7d8ed..6cfd2d030 100644 --- a/test/e2e/resource-filtering/exclude_label.go +++ b/test/e2e/resource-filtering/exclude_label.go @@ -19,9 +19,9 @@ package filtering import ( "fmt" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" diff --git a/test/e2e/resource-filtering/exclude_namespaces.go b/test/e2e/resource-filtering/exclude_namespaces.go index 1b8e5da55..b90caa6e2 100644 --- a/test/e2e/resource-filtering/exclude_namespaces.go +++ b/test/e2e/resource-filtering/exclude_namespaces.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" . "github.com/vmware-tanzu/velero/test/e2e/test" diff --git a/test/e2e/resource-filtering/exclude_resources.go b/test/e2e/resource-filtering/exclude_resources.go index b8a7d2e73..a2346f0af 100644 --- a/test/e2e/resource-filtering/exclude_resources.go +++ b/test/e2e/resource-filtering/exclude_resources.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/test/e2e/resource-filtering/include_namespaces.go b/test/e2e/resource-filtering/include_namespaces.go index d511de212..538473db7 100644 --- a/test/e2e/resource-filtering/include_namespaces.go +++ b/test/e2e/resource-filtering/include_namespaces.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" . "github.com/vmware-tanzu/velero/test/e2e/test" diff --git a/test/e2e/resource-filtering/include_resources.go b/test/e2e/resource-filtering/include_resources.go index 22d7e968b..593efcc24 100644 --- a/test/e2e/resource-filtering/include_resources.go +++ b/test/e2e/resource-filtering/include_resources.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/test/e2e/resource-filtering/label_selector.go b/test/e2e/resource-filtering/label_selector.go index 9ecc66a0c..75d013b59 100644 --- a/test/e2e/resource-filtering/label_selector.go +++ b/test/e2e/resource-filtering/label_selector.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/test/e2e/resourcemodifiers/resource_modifiers.go b/test/e2e/resourcemodifiers/resource_modifiers.go index 06cbd91f1..e5efa8a06 100644 --- a/test/e2e/resourcemodifiers/resource_modifiers.go +++ b/test/e2e/resourcemodifiers/resource_modifiers.go @@ -20,9 +20,9 @@ import ( "fmt" "strings" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" . "github.com/vmware-tanzu/velero/test/e2e/test" "github.com/vmware-tanzu/velero/test/util/common" diff --git a/test/e2e/resourcepolicies/resource_policies.go b/test/e2e/resourcepolicies/resource_policies.go index f3254eb04..306fa6c71 100644 --- a/test/e2e/resourcepolicies/resource_policies.go +++ b/test/e2e/resourcepolicies/resource_policies.go @@ -21,9 +21,9 @@ import ( "strings" "unicode" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" diff --git a/test/e2e/schedule/ordered_resources.go b/test/e2e/schedule/ordered_resources.go index df0d8b972..8e5852f83 100644 --- a/test/e2e/schedule/ordered_resources.go +++ b/test/e2e/schedule/ordered_resources.go @@ -23,9 +23,9 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" "k8s.io/apimachinery/pkg/labels" waitutil "k8s.io/apimachinery/pkg/util/wait" kbclient "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index 7fd06aa74..d04f10af2 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -23,9 +23,9 @@ import ( "strings" "time" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" . "github.com/vmware-tanzu/velero/test" diff --git a/test/perf/basic/basic.go b/test/perf/basic/basic.go index 76bf605a6..6aa2bab0e 100644 --- a/test/perf/basic/basic.go +++ b/test/perf/basic/basic.go @@ -21,7 +21,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" . "github.com/vmware-tanzu/velero/test" . "github.com/vmware-tanzu/velero/test/perf/test" diff --git a/test/perf/e2e_suite_test.go b/test/perf/e2e_suite_test.go index 48a5ceec9..e0c4751ca 100644 --- a/test/perf/e2e_suite_test.go +++ b/test/perf/e2e_suite_test.go @@ -23,9 +23,9 @@ import ( "testing" "time" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" . "github.com/vmware-tanzu/velero/test" diff --git a/test/perf/metrics/minio.go b/test/perf/metrics/minio.go index 8ea7ae4c3..d4cf968b9 100644 --- a/test/perf/metrics/minio.go +++ b/test/perf/metrics/minio.go @@ -17,7 +17,7 @@ limitations under the License. package metrics import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/test/util/metrics" ) diff --git a/test/perf/metrics/nfs.go b/test/perf/metrics/nfs.go index 043a4f976..a0cfb689d 100644 --- a/test/perf/metrics/nfs.go +++ b/test/perf/metrics/nfs.go @@ -19,7 +19,7 @@ package metrics import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/test/util/metrics" ) diff --git a/test/perf/metrics/pod.go b/test/perf/metrics/pod.go index 56572f672..78908d346 100644 --- a/test/perf/metrics/pod.go +++ b/test/perf/metrics/pod.go @@ -22,7 +22,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" metricsclientset "k8s.io/metrics/pkg/client/clientset/versioned" diff --git a/test/perf/restore/restore.go b/test/perf/restore/restore.go index 6adbff5f4..8fa666044 100644 --- a/test/perf/restore/restore.go +++ b/test/perf/restore/restore.go @@ -20,7 +20,7 @@ import ( "context" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" . "github.com/vmware-tanzu/velero/test" . "github.com/vmware-tanzu/velero/test/perf/test" diff --git a/test/perf/test/test.go b/test/perf/test/test.go index 1bc8a3fc0..5716e5b11 100644 --- a/test/perf/test/test.go +++ b/test/perf/test/test.go @@ -22,9 +22,9 @@ import ( "math/rand" "time" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" . "github.com/vmware-tanzu/velero/test" diff --git a/test/pkg/client/client.go b/test/pkg/client/client.go index 331b786f2..142683970 100644 --- a/test/pkg/client/client.go +++ b/test/pkg/client/client.go @@ -20,7 +20,7 @@ import ( "fmt" "runtime" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" diff --git a/test/pkg/client/config.go b/test/pkg/client/config.go index 687c303e7..2a96e3467 100644 --- a/test/pkg/client/config.go +++ b/test/pkg/client/config.go @@ -23,7 +23,7 @@ import ( "strconv" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" ) const ( diff --git a/test/pkg/client/factory.go b/test/pkg/client/factory.go index 340cba587..9691c3492 100644 --- a/test/pkg/client/factory.go +++ b/test/pkg/client/factory.go @@ -24,7 +24,7 @@ import ( k8scheme "k8s.io/client-go/kubernetes/scheme" kbclient "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" diff --git a/test/util/csi/common.go b/test/util/csi/common.go index 373bc1502..b7c80732d 100644 --- a/test/util/csi/common.go +++ b/test/util/csi/common.go @@ -21,9 +21,9 @@ import ( "fmt" "strings" + "github.com/cockroachdb/errors" volumeSnapshotV1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned" - "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" diff --git a/test/util/k8s/common.go b/test/util/k8s/common.go index 8869caab3..40c37c12e 100644 --- a/test/util/k8s/common.go +++ b/test/util/k8s/common.go @@ -25,7 +25,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" diff --git a/test/util/k8s/configmap.go b/test/util/k8s/configmap.go index 39bcb0907..665f7b2a1 100644 --- a/test/util/k8s/configmap.go +++ b/test/util/k8s/configmap.go @@ -22,7 +22,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/test/util/k8s/crd.go b/test/util/k8s/crd.go index fe17fb0ae..a7a63f0f1 100644 --- a/test/util/k8s/crd.go +++ b/test/util/k8s/crd.go @@ -24,7 +24,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" ) diff --git a/test/util/k8s/namespace.go b/test/util/k8s/namespace.go index b46075fee..557782b99 100644 --- a/test/util/k8s/namespace.go +++ b/test/util/k8s/namespace.go @@ -24,7 +24,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/test/util/k8s/persistentvolumes.go b/test/util/k8s/persistentvolumes.go index aaa6b9ea2..7860e73a2 100644 --- a/test/util/k8s/persistentvolumes.go +++ b/test/util/k8s/persistentvolumes.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/test/util/k8s/pod.go b/test/util/k8s/pod.go index 9906e08b5..718beab98 100644 --- a/test/util/k8s/pod.go +++ b/test/util/k8s/pod.go @@ -22,7 +22,7 @@ import ( "fmt" "path" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" diff --git a/test/util/k8s/rbac.go b/test/util/k8s/rbac.go index b660a58d7..82b3ad200 100644 --- a/test/util/k8s/rbac.go +++ b/test/util/k8s/rbac.go @@ -21,7 +21,7 @@ import ( "fmt" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/test/util/k8s/sc.go b/test/util/k8s/sc.go index e6cd8e3b1..0d8e777ac 100644 --- a/test/util/k8s/sc.go +++ b/test/util/k8s/sc.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) diff --git a/test/util/k8s/secret.go b/test/util/k8s/secret.go index ea02f51d0..14b94feb8 100644 --- a/test/util/k8s/secret.go +++ b/test/util/k8s/secret.go @@ -22,7 +22,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/test/util/k8s/service.go b/test/util/k8s/service.go index e8cd098e1..a54df3eb6 100644 --- a/test/util/k8s/service.go +++ b/test/util/k8s/service.go @@ -21,7 +21,7 @@ import ( "fmt" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/test/util/k8s/serviceaccount.go b/test/util/k8s/serviceaccount.go index 31773d846..1658a8b5a 100644 --- a/test/util/k8s/serviceaccount.go +++ b/test/util/k8s/serviceaccount.go @@ -22,7 +22,7 @@ import ( "os" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/test/util/k8s/statefulset.go b/test/util/k8s/statefulset.go index f0ac3a651..027fbd9ae 100644 --- a/test/util/k8s/statefulset.go +++ b/test/util/k8s/statefulset.go @@ -22,7 +22,7 @@ import ( "context" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" ) diff --git a/test/util/kibishii/kibishii_utils.go b/test/util/kibishii/kibishii_utils.go index 5948a2c7b..b2d5c8ed9 100644 --- a/test/util/kibishii/kibishii_utils.go +++ b/test/util/kibishii/kibishii_utils.go @@ -28,8 +28,8 @@ import ( "context" + "github.com/cockroachdb/errors" . "github.com/onsi/ginkgo/v2" - "github.com/pkg/errors" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" diff --git a/test/util/metrics/minio.go b/test/util/metrics/minio.go index 163212713..289efe1cc 100644 --- a/test/util/metrics/minio.go +++ b/test/util/metrics/minio.go @@ -17,7 +17,7 @@ limitations under the License. package metrics import ( - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/test/util/providers" ) diff --git a/test/util/metrics/nfs.go b/test/util/metrics/nfs.go index 2ea6b2f1e..d2b0da87a 100644 --- a/test/util/metrics/nfs.go +++ b/test/util/metrics/nfs.go @@ -21,7 +21,7 @@ import ( "os/exec" "strings" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" ) func GetNFSPathDiskUsage(ctx context.Context, nfsServerPath string) (string, error) { diff --git a/test/util/providers/aws_utils.go b/test/util/providers/aws_utils.go index 7b8916cef..d12e3d71c 100644 --- a/test/util/providers/aws_utils.go +++ b/test/util/providers/aws_utils.go @@ -36,7 +36,7 @@ import ( ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/aws-sdk-go-v2/service/s3" s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" "github.com/vmware-tanzu/velero/test" diff --git a/test/util/providers/azure_utils.go b/test/util/providers/azure_utils.go index 468dbfe40..6c3d7cf1c 100644 --- a/test/util/providers/azure_utils.go +++ b/test/util/providers/azure_utils.go @@ -36,10 +36,10 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" - "github.com/joho/godotenv" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" + "github.com/vmware-tanzu/velero/pkg/util/dotenv" . "github.com/vmware-tanzu/velero/test" ) @@ -127,7 +127,7 @@ func loadCredentialsIntoEnv(credentialsFile string) error { return nil } - if err := godotenv.Overload(credentialsFile); err != nil { + if err := dotenv.Overload(credentialsFile); err != nil { return errors.Wrapf(err, "error loading environment from credentials file (%s)", credentialsFile) } return nil diff --git a/test/util/providers/common.go b/test/util/providers/common.go index a7c68dc37..2886e51eb 100644 --- a/test/util/providers/common.go +++ b/test/util/providers/common.go @@ -25,7 +25,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "github.com/vmware-tanzu/velero/internal/volume" velerotest "github.com/vmware-tanzu/velero/test" diff --git a/test/util/providers/gcloud_utils.go b/test/util/providers/gcloud_utils.go index 022d92ff2..a07a5b39a 100644 --- a/test/util/providers/gcloud_utils.go +++ b/test/util/providers/gcloud_utils.go @@ -26,7 +26,7 @@ import ( "context" "cloud.google.com/go/storage" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "golang.org/x/oauth2/google" "google.golang.org/api/compute/v1" "google.golang.org/api/iterator" diff --git a/test/util/report/report.go b/test/util/report/report.go index 6d5955392..8661bf949 100644 --- a/test/util/report/report.go +++ b/test/util/report/report.go @@ -19,8 +19,8 @@ package report import ( "os" - "github.com/pkg/errors" - "gopkg.in/yaml.v3" + "github.com/cockroachdb/errors" + "go.yaml.in/yaml/v3" "github.com/vmware-tanzu/velero/test" ) diff --git a/test/util/velero/install.go b/test/util/velero/install.go index 121f38760..44853c38a 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -28,7 +28,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "golang.org/x/mod/semver" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index df3aeba08..65f03e332 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -36,7 +36,7 @@ import ( "strings" "time" - "github.com/pkg/errors" + "github.com/cockroachdb/errors" "golang.org/x/mod/semver" schedulingv1api "k8s.io/api/scheduling/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"