diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 131b6ccb5..18cd57a9a 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -10,10 +10,13 @@ about: Tell us about a problem you are experiencing **What did you expect to happen:** +**The following information will help us better understand what's going on**: -**The output of the following commands will help us better understand what's going on**: -(Pasting long output into a [GitHub gist](https://gist.github.com) or other pastebin is fine.) +_If you are using velero v1.7.0+:_ +Please use `velero debug --backup --restore ` to generate the support bundle, and attach to this issue, more options please refer to `velero debug --help` +_If you are using earlier versions:_ +Please provide the output of the following commands (Pasting long output into a [GitHub gist](https://gist.github.com) or other pastebin is fine.) - `kubectl logs deployment/velero -n velero` - `velero backup describe ` or `kubectl get backup/ -n velero -o yaml` - `velero backup logs ` diff --git a/.github/auto-assignees.yml b/.github/auto-assignees.yml index 9fc3fe100..d798ce4d5 100644 --- a/.github/auto-assignees.yml +++ b/.github/auto-assignees.yml @@ -15,6 +15,8 @@ reviewers: - sseago - reasonerjt - ywk253100 + - blackpiglet + - qiuming-best tech-writer: - a-mccarthy diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 38aa77167..a908648a4 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -9,5 +9,5 @@ Fixes #(issue) # Please indicate you've done the following: - [ ] [Accepted the DCO](https://velero.io/docs/v1.5/code-standards/#dco-sign-off). Commits without the DCO will delay acceptance. -- [ ] [Created a changelog file](https://velero.io/docs/v1.5/code-standards/#adding-a-changelog) or added `/kind changelog-not-required`. +- [ ] [Created a changelog file](https://velero.io/docs/v1.5/code-standards/#adding-a-changelog) or added `/kind changelog-not-required` as a comment on this pull request. - [ ] Updated the corresponding documentation in `site/content/docs/main`. diff --git a/.github/stale.yml b/.github/stale.yml index 4c2adae9b..ce96631aa 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -14,9 +14,15 @@ exemptLabels: - Area/Design - Area/Documentation - Area/Plugins + - Bug - Enhancement/User + - kind/requirement + - kind/refactor - kind/tech-debt + - limitation - Needs investigation + - Needs triage + - Needs Product - P0 - Hair on fire - P1 - Important - P2 - Long-term important diff --git a/.github/workflows/crds-verify-kind.yaml b/.github/workflows/crds-verify-kind.yaml index 41ba94440..45b6d71a9 100644 --- a/.github/workflows/crds-verify-kind.yaml +++ b/.github/workflows/crds-verify-kind.yaml @@ -57,7 +57,6 @@ jobs: matrix: # Latest k8s versions. There's no series-based tag, nor is there a latest tag. k8s: - - 1.15.12 - 1.16.15 - 1.17.17 - 1.18.15 diff --git a/.github/workflows/e2e-test-kind.yaml b/.github/workflows/e2e-test-kind.yaml index 02f667f10..38fcd1d2a 100644 --- a/.github/workflows/e2e-test-kind.yaml +++ b/.github/workflows/e2e-test-kind.yaml @@ -14,7 +14,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.16 + go-version: 1.17 id: go # Look for a CLI that's made for this PR - name: Fetch built CLI @@ -69,8 +69,14 @@ jobs: - 1.20.2 - 1.21.1 - 1.22.0 + - 1.23.0 fail-fast: false steps: + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.17 + id: go - name: Check out the code uses: actions/checkout@v2 - name: Install MinIO @@ -116,4 +122,11 @@ jobs: ADDITIONAL_OBJECT_STORE_PROVIDER=aws ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \ ADDITIONAL_CREDS_FILE=/tmp/credential ADDITIONAL_BSL_BUCKET=additional-bucket \ GINKGO_FOCUS=Basic VELERO_IMAGE=velero:pr-test \ - make -C test/e2e run \ No newline at end of file + make -C test/e2e run + timeout-minutes: 30 + - name: Upload debug bundle + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: DebugBundle + path: /home/runner/work/velero/velero/test/e2e/debug-bundle* \ No newline at end of file diff --git a/.github/workflows/pr-ci-check.yml b/.github/workflows/pr-ci-check.yml index 9edf3c808..12bb06622 100644 --- a/.github/workflows/pr-ci-check.yml +++ b/.github/workflows/pr-ci-check.yml @@ -8,7 +8,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.16 + go-version: 1.17 id: go - name: Check out the code uses: actions/checkout@v2 diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 5c61ab7e6..4d232ef9d 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.16 + go-version: 1.17 id: go - name: Check out code into the Go module directory diff --git a/.goreleaser.yml b/.goreleaser.yml index fb6ee34b8..e7fa81f58 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -27,11 +27,9 @@ builds: - arm64 - ppc64le ignore: - # don't build arm/arm64 for darwin or windows + # don't build arm for darwin and arm/arm64 for windows - goos: darwin goarch: arm - - goos: darwin - goarch: arm64 - goos: darwin goarch: ppc64le - goos: windows diff --git a/ADOPTERS.md b/ADOPTERS.md index 636c96e41..44f25513c 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -14,7 +14,7 @@ If you're using Velero and want to add your organization to this list, sighup.io      mayadata.io      replicated.com - +cloudcasa.io ## Success Stories Below is a list of adopters of Velero in **production environments** that have @@ -56,8 +56,11 @@ MayaData is a large user of Velero as well as a contributor. MayaData offers a D Okteto integrates Velero in [Okteto Cloud][94] and [Okteto Enterprise][95] to periodically backup and restore our clusters for disaster recovery. Velero is also a core software building block to provide namespace cloning capabilities, a feature that allows our users cloning staging environments into their personal development namespace for providing production-like development environments. **[Replicated][100]**
-Replicated uses the Velero open source project to enable snapshots in [KOTS][101] to backup Kubernetes manifests & persistent volumes. In addition to the default functionality that Velero provides, [KOTS][101] provides a detailed interface in the [Admin Console][102] that can be used to manage the storage destination and schedule, and to perform and monitor the backup and restore process. -​ +Replicated uses the Velero open source project to enable snapshots in [KOTS][101] to backup Kubernetes manifests & persistent volumes. In addition to the default functionality that Velero provides, [KOTS][101] provides a detailed interface in the [Admin Console][102] that can be used to manage the storage destination and schedule, and to perform and monitor the backup and restore process.
+ +**[CloudCasa][103]**
+[Catalogic Software][104] integrates Velero with [CloudCasa][103] - A Smart Home in the Cloud for Backups. CloudCasa is a simple, scalable, cloud-native solution providing data protection and disaster recovery as a service. This solution is built using Kubernetes for protecting Kubernetes clusters.
+ ## Adding your organization to the list of Velero Adopters If you are using Velero and would like to be included in the list of `Velero Adopters`, add an SVG version of your logo to the `site/static/img/adopters` directory in this repo and submit a [pull request][3] with your change. Name the image file something that reflects your company (e.g., if your company is called Acme, name the image acme.png). See this for an example [PR][4]. @@ -110,3 +113,6 @@ If you would like to add your logo to a future `Adopters of Velero` section on [ [100]: https://www.replicated.com [101]: https://kots.io [102]: https://kots.io/kotsadm/snapshots/overview/ + +[103]: https://cloudcasa.io/ +[104]: https://www.catalogicsoftware.com/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c749bf2b..1dbfe3e1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,8 @@ ## Current release: - * [CHANGELOG-1.6.md][16] + * [CHANGELOG-1.7.md][17] ## Older releases: + * [CHANGELOG-1.6.md][16] * [CHANGELOG-1.5.md][15] * [CHANGELOG-1.4.md][14] * [CHANGELOG-1.3.md][13] @@ -19,6 +20,7 @@ * [CHANGELOG-0.3.md][1] +[17]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.7.md [16]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.6.md [15]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.5.md [14]: https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.4.md diff --git a/Dockerfile b/Dockerfile index bdead4f8f..4040e18dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM --platform=$BUILDPLATFORM golang:1.16 as builder-env +FROM --platform=$BUILDPLATFORM golang:1.17 as builder-env ARG GOPROXY ARG PKG diff --git a/MAINTAINERS.md b/MAINTAINERS.md index c2680edc7..2de9607ac 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -6,12 +6,13 @@ | Maintainer | GitHub ID | Affiliation | | --------------- | --------- | ----------- | -| Bridget McErlean | [zubron](https://github.com/zubron) | [VMware](https://www.github.com/vmware/) | -| Dave Smith-Uchida | [dsu-igeek](https://github.com/dsu-igeek) | [VMware](https://www.github.com/vmware/) | +| Dave Smith-Uchida | [dsu-igeek](https://github.com/dsu-igeek) | [Kasten](https://github.com/kastenhq/) | | JenTing Hsiao | [jenting](https://github.com/jenting) | [SUSE](https://github.com/SUSE/) | Scott Seago | [sseago](https://github.com/sseago) | [OpenShift](https://github.com/openshift) | Daniel Jiang | [reasonerjt](https://github.com/reasonerjt) | [VMware](https://www.github.com/vmware/) | Wenkai Yin | [ywk253100](https://github.com/ywk253100) | [VMware](https://www.github.com/vmware/) | +| Xun Jiang | [blackpiglet](https://github.com/blackpiglet) | [VMware](https://www.github.com/vmware/) | +| Ming Qiu | [qiuming-best](https://github.com/qiuming-best) | [VMware](https://www.github.com/vmware/) | ## Emeritus Maintainers * Adnan Abdulhussein ([prydonius](https://github.com/prydonius)) @@ -21,12 +22,14 @@ * Nolan Brubaker ([nrb](https://github.com/nrb)) * Ashish Amarnath ([ashish-amarnath](https://github.com/ashish-amarnath)) * Carlisia Thompson ([carlisia](https://github.com/carlisia)) +* Bridget McErlean ([zubron](https://github.com/zubron)) ## Velero Contributors & Stakeholders | Feature Area | Lead | | ----------------------------- | :---------------------: | -| Technical Lead | Dave Smith-Uchida (dsu-igeek) | +| Architect | Dave Smith-Uchida (dsu-igeek) | +| Technical Lead | Daniel Jiang (reasonerjt) | | Kubernetes CSI Liaison | | | Deployment | JenTing Hsiao (jenting) | | Community Management | Jonas Rosland (jonasrosland) | diff --git a/Makefile b/Makefile index 3884063c9..c17e24147 100644 --- a/Makefile +++ b/Makefile @@ -81,10 +81,10 @@ buildx not enabled, refusing to run this recipe see: https://velero.io/docs/main/build-from-source/#making-images-and-updating-velero for more info endef -# The version of restic binary to be downloaded for power architecture -RESTIC_VERSION ?= 0.12.0 +# The version of restic binary to be downloaded +RESTIC_VERSION ?= 0.12.1 -CLI_PLATFORMS ?= linux-amd64 linux-arm linux-arm64 darwin-amd64 windows-amd64 linux-ppc64le +CLI_PLATFORMS ?= linux-amd64 linux-arm linux-arm64 darwin-amd64 darwin-arm64 windows-amd64 linux-ppc64le BUILDX_PLATFORMS ?= $(subst -,/,$(ARCH)) BUILDX_OUTPUT_TYPE ?= docker @@ -338,9 +338,9 @@ changelog: # PUBLISH=false \ # make release # -# To run the release, which will publish a *DRAFT* GitHub release in github.com/vmware-tanzu/velero +# To run the release, which will publish a *DRAFT* GitHub release in github.com/vmware-tanzu/velero # (you still need to review/publish the GitHub release manually): -# GITHUB_TOKEN=your-github-token \ +# GITHUB_TOKEN=your-github-token \ # RELEASE_NOTES_FILE=changelogs/CHANGELOG-1.2.md \ # PUBLISH=true \ # make release @@ -359,11 +359,11 @@ serve-docs: build-image-hugo -it -p 1313:1313 \ $(HUGO_IMAGE) \ hugo server --bind=0.0.0.0 --enableGitInfo=false -# gen-docs generates a new versioned docs directory under site/content/docs. +# gen-docs generates a new versioned docs directory under site/content/docs. # Please read the documentation in the script for instructions on how to use it. gen-docs: @hack/release-tools/gen-docs.sh .PHONY: test-e2e test-e2e: local - $(MAKE) -C test/e2e run + $(MAKE) -e VERSION=$(VERSION) -C test/e2e run diff --git a/PROJECT b/PROJECT deleted file mode 100644 index 88609cdaa..000000000 --- a/PROJECT +++ /dev/null @@ -1,7 +0,0 @@ -domain: io -repo: github.com/vmware-tanzu/velero -resources: -- group: velero - kind: BackupStorageLocation - version: v1 -version: "2" diff --git a/README.md b/README.md index 4dcefa133..36cc3baff 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,18 @@ If you are ready to jump in and test, add code, or help with documentation, foll See [the list of releases][6] to find out about feature changes. +### Velero compatibility matrix + +The following is a list of the supported Kubernetes versions for each Velero version. + +| Velero version | Kubernetes versions| +|----------------|--------------------| +| 1.8 | 1.16-latest | +| 1.6.3-1.7.1 | 1.12-latest | +| 1.60-1.6.2 | 1.12-1.21 | +| 1.5 | 1.12-1.21 | +| 1.4 | 1.10-1.21 | + [1]: https://github.com/vmware-tanzu/velero/workflows/Main%20CI/badge.svg [2]: https://github.com/vmware-tanzu/velero/actions?query=workflow%3A"Main+CI" [4]: https://github.com/vmware-tanzu/velero/issues diff --git a/ROADMAP.md b/ROADMAP.md index 1528eec15..7bf9fc747 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -15,33 +15,28 @@ We work with and rely on community feedback to focus our efforts to improve Vele The following table includes the current roadmap for Velero. If you have any questions or would like to contribute to Velero, please attend a [community meeting](https://velero.io/community/) to discuss with our team. If you don't know where to start, we are always looking for contributors that will help us reduce technical, automation, and documentation debt. Please take the timelines & dates as proposals and goals. Priorities and requirements change based on community feedback, roadblocks encountered, community contributions, etc. If you depend on a specific item, we encourage you to attend community meetings to get updated status information, or help us deliver that feature by contributing to Velero. -`Last Updated: July 2021` +`Last Updated: October 2021` -#### 1.7.0 Roadmap (to be delivered early fall) -The release roadmap is split into Core items that are required for the release and desired items that may slip the release. +#### 1.8.0 Roadmap (to be delivered January/February 2021) -##### Core items -The top priority of 1.7 is to increase the technical health of Velero and be more efficient with Velero developer time by streamlining the release process and automating and expanding the E2E test suite. +|Issue|Description|Timeline|Notes| +|---|---|---|---| +|[4108](https://github.com/vmware-tanzu/velero/issues/4108), [4109](https://github.com/vmware-tanzu/velero/issues/4109)|Solution for CSI - Azure and AWS|2022 H1|Currently, Velero plugins for AWS and Azure cannot back up persistent volumes that were provisioned using the CSI driver. This will fix that.| +|[3229](https://github.com/vmware-tanzu/velero/issues/3229),[4112](https://github.com/vmware-tanzu/velero/issues/4112)|Moving data mover functionality from the Velero Plugin for vSphere into Velero proper|2022 H1|This work is a precursor to decoupling the Astrolabe snapshotting infrastructure.| +|[3533](https://github.com/vmware-tanzu/velero/issues/3533)|Upload Progress Monitoring|2022 H1|Finishing up the work done in the 1.7 timeframe. The data mover work depends on this.| +|[1975](https://github.com/vmware-tanzu/velero/issues/1975)|Test dual stack mode|2022 H1|We already tested IPv6, but we want to confirm that dual stack mode works as well.| +|[2082](https://github.com/vmware-tanzu/velero/issues/2082)|Delete Backup CRs on removing target location. |2022 H1|| +|[3516](https://github.com/vmware-tanzu/velero/issues/3516)|Restore issue with MutatingWebhookConfiguration v1beta1 API version|2022 H1|| +|[2308](https://github.com/vmware-tanzu/velero/issues/2308)|Restoring nodePort service that has nodePort preservation always fails if service already exists in the namespace|2022 H1|| +|[4115](https://github.com/vmware-tanzu/velero/issues/4115)|Support for multiple set of credentials for VolumeSnapshotLocations|2022 H1|| +|[1980](https://github.com/vmware-tanzu/velero/issues/1980)|Velero triggers backup immediately for scheduled backups|2022 H1|| +|[4067](https://github.com/vmware-tanzu/velero/issues/4067)|Pre and post backup and restore hooks|2022 H1|| +|[3742](https://github.com/vmware-tanzu/velero/issues/3742)|Carvel packaging for Velero for vSphere|2022 H1|AWS and Azure have been completed already.| +|[3285](https://github.com/vmware-tanzu/velero/issues/3285)|Design doc for Velero plugin versioning|2022 H1|| +|[4231](https://github.com/vmware-tanzu/velero/issues/4231)|Technical health (prioritizing giving developers confidence and saving developers time)|2022 H1|More automated tests (especially the pre-release manual tests) and more automation of the running of tests.| +|[4110](https://github.com/vmware-tanzu/velero/issues/4110)|Solution for CSI - GCP|2022 H1|Currently, the Velero plugin for GCP cannot back up persistent volumes that were provisioned using the CSI driver. This will fix that.| +|[3742](https://github.com/vmware-tanzu/velero/issues/3742)|Carvel packaging for Velero for restic|2022 H1|AWS and Azure have been completed already.| +|[3454](https://github.com/vmware-tanzu/velero/issues/3454),[4134](https://github.com/vmware-tanzu/velero/issues/4134),[4135](https://github.com/vmware-tanzu/velero/issues/4135)|Kubebuilder tech debt|2022 H1|| +|[4111](https://github.com/vmware-tanzu/velero/issues/4111)|Ignore items returned by ItemSnapshotter.AlsoHandles during backup|2022 H1|This will enable backup of complex objects, because we can then tell Velero to ignore things that were already backed up when Velero was previously called recursively.| -|Issue|Description| -|---|---| -||Streamline release process| -||Automate the running of the E2E tests| -||Convert pre-release manual tests to automated E2E tests| -|[3493](https://github.com/vmware-tanzu/velero/issues/3493)|[Carvel](https://github.com/vmware-tanzu/velero/issues/3493) based installation (in addition to the existing *velero install* CLI).| -|[675](https://github.com/vmware-tanzu/velero/issues/675)|Velero command to generate debugging information. Will integrate with [Crashd - Crash Diagnostics](https://github.com/vmware-tanzu/velero/issues/675)| -|[3285](https://github.com/vmware-tanzu/velero/issues/3285)|Design doc for Velero plugin versioning| -|[1975](https://github.com/vmware-tanzu/velero/issues/1975)|IPV6 support| -|[3533](https://github.com/vmware-tanzu/velero/issues/3533)|Upload Progress Monitoring| -|[3500](https://github.com/vmware-tanzu/velero/issues/3500)|Use distroless containers as a base| - - - -##### Items formerly in 1.7 that will slip due to staffing changes -|Issue|Description| -|---|---| -|[3536](https://github.com/vmware-tanzu/velero/issues/3536)|Manifest for backup/restore| -|[2066](https://github.com/vmware-tanzu/velero/issues/2066)|CSI Snapshots GA| -|[3535](https://github.com/vmware-tanzu/velero/issues/3535)|Design doc for multiple cluster support| -|[2922](https://github.com/vmware-tanzu/velero/issues/2922)|Plugin timeouts| -|[3531](https://github.com/vmware-tanzu/velero/issues/3531)|Test plan for Velero| \ No newline at end of file +Other work may make it into the 1.8 release, but this is the work that will be prioritized first. \ No newline at end of file diff --git a/Tiltfile b/Tiltfile index 7e72960c3..ba36e3346 100644 --- a/Tiltfile +++ b/Tiltfile @@ -16,7 +16,7 @@ k8s_yaml([ # default values settings = { - "default_registry": "", + "default_registry": "docker.io/velero", "enable_restic": False, "enable_debug": False, "debug_continue_on_start": True, # Continue the velero process by default when in debug mode @@ -50,7 +50,7 @@ git_sha = str(local("git rev-parse HEAD", quiet = True, echo_off = True)).strip( tilt_helper_dockerfile_header = """ # Tilt image -FROM golang:1.16.6 as tilt-helper +FROM golang:1.17 as tilt-helper # Support live reloading with Tilt RUN wget --output-document /restart.sh --quiet https://raw.githubusercontent.com/windmilleng/rerun-process-wrapper/master/restart.sh && \ @@ -90,20 +90,20 @@ def get_debug_flag(): # Set up a local_resource build of the Velero binary. The binary is written to _tiltbuild/velero. local_resource( "velero_server_binary", - cmd = 'cd ' + '.' + ';mkdir -p _tiltbuild;PKG=. BIN=velero GOOS=linux GOARCH=amd64 GIT_SHA=' + git_sha + ' VERSION=main GIT_TREE_STATE=dirty OUTPUT_DIR=_tiltbuild ' + get_debug_flag() + ' ./hack/build.sh', + cmd = 'cd ' + '.' + ';mkdir -p _tiltbuild;PKG=. BIN=velero GOOS=linux GOARCH=amd64 GIT_SHA=' + git_sha + ' VERSION=main GIT_TREE_STATE=dirty OUTPUT_DIR=_tiltbuild ' + get_debug_flag() + ' REGISTRY=' + settings.get("default_registry") + ' ./hack/build.sh', deps = ["cmd", "internal", "pkg"], ignore = ["pkg/cmd"], ) local_resource( "velero_local_binary", - cmd = 'cd ' + '.' + ';mkdir -p _tiltbuild/local;PKG=. BIN=velero GOOS=' + local_goos + ' GOARCH=amd64 GIT_SHA=' + git_sha + ' VERSION=main GIT_TREE_STATE=dirty OUTPUT_DIR=_tiltbuild/local ' + get_debug_flag() + ' ./hack/build.sh', + cmd = 'cd ' + '.' + ';mkdir -p _tiltbuild/local;PKG=. BIN=velero GOOS=' + local_goos + ' GOARCH=amd64 GIT_SHA=' + git_sha + ' VERSION=main GIT_TREE_STATE=dirty OUTPUT_DIR=_tiltbuild/local ' + get_debug_flag() + ' REGISTRY=' + settings.get("default_registry") + ' ./hack/build.sh', deps = ["internal", "pkg/cmd"], ) local_resource( "restic_binary", - cmd = 'cd ' + '.' + ';mkdir -p _tiltbuild/restic; BIN=velero GOOS=' + local_goos + ' GOARCH=amd64 RESTIC_VERSION=0.12.0 OUTPUT_DIR=_tiltbuild/restic ./hack/download-restic.sh', + cmd = 'cd ' + '.' + ';mkdir -p _tiltbuild/restic; BIN=velero GOOS=linux GOARCH=amd64 RESTIC_VERSION=0.12.0 OUTPUT_DIR=_tiltbuild/restic ./hack/download-restic.sh', ) # Note: we need a distro with a bash shell to exec into the Velero container diff --git a/changelogs/CHANGELOG-1.7.md b/changelogs/CHANGELOG-1.7.md new file mode 100644 index 000000000..fdff3c2b0 --- /dev/null +++ b/changelogs/CHANGELOG-1.7.md @@ -0,0 +1,80 @@ +## v1.7.0 +### 2021-09-07 + +### Download +https://github.com/vmware-tanzu/velero/releases/tag/v1.7.0 + +### Container Image +`velero/velero:v1.7.0` + +### Documentation +https://velero.io/docs/v1.7/ + +### Upgrading +https://velero.io/docs/v1.7/upgrade-to-1.7/ + +### Highlights + +#### Distroless images + +The Velero container images now use [distroless base images](https://github.com/GoogleContainerTools/distroless). +Using distroless images as the base ensures that only the packages and programs necessary for running Velero are included. +Unrelated libraries and OS packages, that often contain security vulnerabilities, are now excluded. +This change reduces the size of both the server and restic restore helper image by approximately 62MB. + +As the [distroless](https://github.com/GoogleContainerTools/distroless) images do not contain a shell, it will no longer be possible to exec into Velero containers using these images. + +#### New "debug" command + +This release introduces the new `velero debug` command. +This command collects information about a Velero installation, such as pod logs and resources managed by Velero, in a tarball which can be provided to the Velero maintainer team to help diagnose issues. + +### All changes + + * Distinguish between different unnamed node ports when preserving (#4026, @sseago) + * Validate namespace in Velero backup create command (#4057, @codegold79) + * Empty the "ClusterIPs" along with "ClusterIP" when "ClusterIP" isn't "None" (#4101, @ywk253100) + * Add a RestoreItemAction plugin (`velero.io/apiservice`) which skips the restore of any `APIService` which is managed by Kubernetes. These are identified using the `kube-aggregator.kubernetes.io/automanaged` label. (#4028, @zubron) + * Change the base image to distroless (#4055, @ywk253100) + * Updated the version of velero/velero-plugin-for-aws version from v1.2.0 to v1.2.1 (#4064, @kahirokunn) + * Skip the backup and restore of DownwardAPI volumes when using restic. (#4076, @zubron) + * Bump up Go to 1.16 (#3990, @reasonerjt) + * Fix restic error when volume is emptyDir and Pod not running (#3993, @mahaupt) + * Select the velero deployment with both label and container name (#3996, @ywk253100) + * Wait for the namespace to be deleted before removing the CRDs during uninstall. This deprecates the `--wait` flag of the `uninstall` command (#4007, @ywk253100) + * Use the cluster preferred CRD API version when polling for Velero CRD readiness. (#4015, @zubron) + * Implement velero debug (#4022, @reasonerjt) + * Skip the restore of volumes that originally came from a projected volume when using restic. (#3877, @zubron) + * Run the E2E test with kind(provision various versions of k8s cluster) and MinIO on Github Action (#3912, @ywk253100) + * Fix -install-velero flag for e2e tests (#3919, @jaidevmane) + * Upgrade Velero ClusterRoleBinding to use v1 API (#3926, @jenting) + * enable e2e tests to choose crd apiVersion (#3941, @sseago) + * Fixing multipleNamespaceTest bug - Missing expect statement in test (#3983, @jaidevmane) + * Add --client-page-size flag to server to allow chunking Kubernetes API LIST calls across multiple requests on large clusters (#3823, @dharmab) + * Fix CR restore regression introduced in 1.6 restore progress. (#3845, @sseago) + * Use region specified in the BackupStorageLocation spec when getting restic repo identifier. Originally fixed by @jala-dx in #3617. (#3857, @zubron) + * skip backuping projected volume when using restic (#3866, @alaypatel07) + * Install Kubernetes preferred CRDs API version (v1beta1/v1). (#3614, @jenting) + * Add Label to BackupSpec so that labels can explicitly be provided to Schedule.Spec.Template.Metadata.Labels which will be reflected on the backups created. (#3641, @arush-sal) + * Add PVC UID label to PodVolumeRestore (#3792, @sseago) + * Support pulling plugin images by digest (#3803, @2uasimojo) + * Added BackupPhaseUploading and BackupPhaseUploadingPartialFailure backup phases as part of Upload Progress Monitoring. (#3805, @dsmithuchida) + + Uploading (new) + The "Uploading" phase signifies that the main part of the backup, including + snapshotting has completed successfully and uploading is continuing. In + the event of an error during uploading, the phase will change to + UploadingPartialFailure. On success, the phase changes to Completed. The + backup cannot be restored from when it is in the Uploading state. + + UploadingPartialFailure (new) + The "UploadingPartialFailure" phase signifies that the main part of the backup, + including snapshotting has completed, but there were partial failures either + during the main part or during the uploading. The backup cannot be restored + from when it is in the UploadingPartialFailure state. + * 🐛 Fix plugin name derivation from image name (#3711, @ashish-amarnath) + * ✨ ⚠️ Remove CSI volumesnapshot artifact deletion + +This change requires https://github.com/vmware-tanzu/velero-plugin-for-csi/pull/86 for Velero to continue +deleting of CSI volumesnapshots when the corresponding backups are deleted. (#3734, @ashish-amarnath) + * use unstructured to marshal selective fields for service restore action (#3789, @alaypatel07) diff --git a/changelogs/CHANGELOG-1.8.md b/changelogs/CHANGELOG-1.8.md new file mode 100644 index 000000000..1e250f65d --- /dev/null +++ b/changelogs/CHANGELOG-1.8.md @@ -0,0 +1,110 @@ +## v1.8.0 +### 2022-01-14 + +### Download +https://github.com/vmware-tanzu/velero/releases/tag/v1.8.0 + +### Container Image +`velero/velero:v1.8.0` + +### Documentation +https://velero.io/docs/v1.8 + +### Upgrading +https://velero.io/docs/v1.8/upgrade-to-1.8/ + +### Highlights + +#### Velero plugins now support handling volumes created by the CSI drivers of cloud providers +Versions 1.4 of the Velero plugins for AWS, Azure and GCP now support snapshotting and restoring the persistent volumes provisioned by CSI driver via the APIs of the cloud providers. With this enhancement, users can backup and restore the persistent volumes on these cloud providers without using the Velero CSI plugin. The CSI plugin will remain beta and the feature flag `EnableCSI` will be disabled by default. + +For the version of the plugins and the CSI drivers they support respectively please see the table: + +| Plugin | Version | CSI Driver | +| --- | ----------- | ---------- | +| velero-plugin-for-aws | v1.4.0 | ebs.csi.aws.com | +| velero-plugin-for-microsoft-azure | v1.4.0 | disk.csi.azure.com | +| velero-plugin-for-gcp | v1.4.0 | pd.csi.storage.gke.io | + +#### IPv6 dual stack support +We've verified the functionality of Velero on IPv6 dual stack by successfully running the E2E test on IPv6 dual stack environment. +#### Refactor the controllers using Kubebuilder v3 +In this release we continued our code modernization work, rewriting some controllers using Kubebuilder v3. This work is ongoing and we will continue to make progress in future releases. +#### Enhancements to E2E test cases +More test cases have been added to the E2E test suite to improve the release health. +#### Respect the cron setting of scheduled backup +The creation time is now taken into account to calculate the next run for scheduled backup. + +#### Deleting BSLs also cleans up related resources + +When a Backup Storage Location (BSL) is deleted, backup and Restic repository resources will also be deleted. + +#### Breaking changes + +Starting in v1.8, Velero will only support Kubernetes v1 CRD meaning that Velero v1.8+ will only run on Kubernetes v1.16+. Before upgrading, make sure you are running a supported Kubernetes version. For more information, see our [compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatibility-matrix). + +#### Upload Progress Monitoring and Item Snapshotter +Item Snapshotter plugin API was merged. This will support both Upload Progress +monitoring and the planned Data Mover. Upload Progress monitoring PRs are +in progress for 1.9. + +### All changes + +* E2E test on ssr object with controller namespace mix-ups (#4521, @mqiu) +* Check whether the volume is provisioned by CSI driver or not by the annotation as well (#4513, @ywk253100) +* Initialize the labels field of `velero backup-location create` option to avoid #4484 (#4491, @ywk253100) +* Fix e2e 2500 namespaces scale test timeout problem (#4480, @mqiu) +* Add backup deletion e2e test (#4401, @danfengliu) +* Return the error when getting backup store in backup deletion controller (#4465, @reasonerjt) +* Ignore the provided port is already allocated error when restoring the LoadBalancer service (#4462, @ywk253100) +* Revert #4423 migrate backup sync controller to kubebuilder. (#4457, @jxun) +* Add rbac and annotation test cases (#4455, @mqiu) +* remove --crds-version in velero install command. (#4446, @jxun) +* Upgrade e2e test vsphere plugin (#4440, @mqiu) +* Fix e2e test failures for the inappropriate optimaze of velero install (#4438, @mqiu) +* Limit backup namespaces on test resource filtering cases (#4437, @mqiu) +* Bump up Go to 1.17 (#4431, @reasonerjt) +* Added ``-itemsnapshots.json.gz to the backup format. This file exists + when item snapshots are taken and contains an array of volume.Itemsnapshots + containing the information about the snapshots. This will not be used unless + upload progress monitoring and item snapshots are enabled and an ItemSnapshot + plugin is used to take snapshots. + +Also added DownloadTargetKindBackupItemSnapshots for retrieving the signed URL to download only the ``-itemsnapshots.json.gz part of a backup for use by +`velero backup describe`. (#4429, @dsmithuchida) +* Migrate backup sync controller from code-generator to kubebuilder. (#4423, @jxun) +* Added UploadProgressFeature flag to enable Upload Progress Monitoring and Item + Snapshotters. (#4416, @dsmithuchida) +* Added BackupWithResolvers and RestoreWithResolvers calls. Will eventually replace Backup and Restore methods. + Adds ItemSnapshotters to Backup and Restore workflows. (#4410, @dsu) +* Build for darwin-arm64 (#4409, @epk) +* Add resource filtering test cases (#4404, @mqiu) +* Fix the issue that the backup cannot be deleted after the application uninstalled (#4398, @ywk253100) +* Add restoreactionitem plugin to handle admission webhook configurations (#4397, @reasonerjt) +* Keep the annotation "pv.kubernetes.io/provisioned-by" when restoring PVs (#4391, @ywk253100) +* Adjust structure of e2e test codes (#4386, @mqiu) +* feat: migrate velero controller from kubebuilder v2 to v3 + From Velero v1.8, apiextesions.k8s.io/v1beta1 is no longer supported, + which means only CRD of apiextensions.k8s.io/v1 is supported, + and the supported Kubernetes version is updated to v1.16 and later. (#4382, @jxun) +* Delete backups and Restic repos associated with deleted BSL(s) (#4377, @codegold79) +* Add the key for GKE zone for AZ collection (#4376, @reasonerjt) +* Fix statefulsets volumeClaimTemplates storageClassName when use Changing PV/PVC Storage Classes (#4375, @Box-Cube) +* Fix snapshot e2e test issue of jsonpath (#4372, @danfengliu) +* Modify the timestamp in the name of a backup generated from schedule to use UTC. (#4353, @jxun) +* Read Availability zone from nodeAffinity requirements (#4350, @reasonerjt) +* Use factory.Namespace() to replace hardcoded velero namespace (#4346, @half-life666) +* Return the error if velero failed to detect S3 region for restic repo (#4343, @reasonerjt) +* Add init log option for velero controller-runtime manager. (#4341, @jxun) +* Ignore the `provided port is already allocated` error when restoring the `NodePort` service (#4336, @ywk253100) +* Fixed an issue with the `backup-location create` command where the BSL Credential field would be set to an invalid empty SecretKeySelector when no credential details were provided. (#4322, @zubron) +* fix buggy pager func (#4306, @alaypatel07) +* Don't create a backup immediately after creating a schedule (#4281, @ywk253100) +* Fix CVE-2020-29652 and CVE-2020-26160 (#4274, @ywk253100) +* Refine tag-release.sh to align with change in release process (#4185, @reasonerjt) +* Fix plugins incompatible issue in upgrade test (#4141, @danfengliu) +* Verify group before treating resource as cohabitating (#4126, @sseago) +* Added ItemSnapshotter plugin definition and plugin framework - addresses #3533. + Part of the Upload Progress enhancement (#3533) (#4077, @dsmithuchida) +* Add upgrade test in E2E test (#4058, @danfengliu) +* Handle namespace mapping for PVs without snapshots on restore (#3708, @sseago) diff --git a/changelogs/unreleased/3614-jenting b/changelogs/unreleased/3614-jenting deleted file mode 100644 index b0965c019..000000000 --- a/changelogs/unreleased/3614-jenting +++ /dev/null @@ -1 +0,0 @@ -Install Kubernetes preferred CRDs API version (v1beta1/v1). diff --git a/changelogs/unreleased/3641-arush-sal b/changelogs/unreleased/3641-arush-sal deleted file mode 100644 index 61681fa66..000000000 --- a/changelogs/unreleased/3641-arush-sal +++ /dev/null @@ -1 +0,0 @@ -Add Label to BackupSpec so that labels can explicitly be provided to Schedule.Spec.Template.Metadata.Labels which will be reflected on the backups created. diff --git a/changelogs/unreleased/3711-ashish-amarnath b/changelogs/unreleased/3711-ashish-amarnath deleted file mode 100644 index 298a775b5..000000000 --- a/changelogs/unreleased/3711-ashish-amarnath +++ /dev/null @@ -1 +0,0 @@ -🐛 Fix plugin name derivation from image name diff --git a/changelogs/unreleased/3734-ashish-amarnath b/changelogs/unreleased/3734-ashish-amarnath deleted file mode 100644 index b48e60b71..000000000 --- a/changelogs/unreleased/3734-ashish-amarnath +++ /dev/null @@ -1,4 +0,0 @@ -✨ ⚠️ Remove CSI volumesnapshot artifact deletion - -This change requires https://github.com/vmware-tanzu/velero-plugin-for-csi/pull/86 for Velero to continue -deleting of CSI volumesnapshots when the corresponding backups are deleted. \ No newline at end of file diff --git a/changelogs/unreleased/3789-alaypatel07 b/changelogs/unreleased/3789-alaypatel07 deleted file mode 100644 index d22f47c72..000000000 --- a/changelogs/unreleased/3789-alaypatel07 +++ /dev/null @@ -1 +0,0 @@ -use unstructured to marshal selective fields for service restore action \ No newline at end of file diff --git a/changelogs/unreleased/3792-sseago b/changelogs/unreleased/3792-sseago deleted file mode 100644 index e684e787c..000000000 --- a/changelogs/unreleased/3792-sseago +++ /dev/null @@ -1 +0,0 @@ -Add PVC UID label to PodVolumeRestore diff --git a/changelogs/unreleased/3803-2uasimojo b/changelogs/unreleased/3803-2uasimojo deleted file mode 100644 index 0c9ea2b1c..000000000 --- a/changelogs/unreleased/3803-2uasimojo +++ /dev/null @@ -1 +0,0 @@ -Support pulling plugin images by digest diff --git a/changelogs/unreleased/3805-dsmithuchida b/changelogs/unreleased/3805-dsmithuchida deleted file mode 100644 index e94787c80..000000000 --- a/changelogs/unreleased/3805-dsmithuchida +++ /dev/null @@ -1,16 +0,0 @@ -Added BackupPhaseUploading and BackupPhaseUploadingPartialFailure backup phases as part of Upload Progress Monitoring - -Uploading (new) -The "Uploading" phase signifies that the main part of the backup, including -snapshotting has completed successfully and uploading is continuing. In -the event of an error during uploading, the phase will change to -UploadingPartialFailure. On success, the phase changes to Completed. The -backup cannot be restored from when it is in the Uploading state. - -UploadingPartialFailure (new) -The "UploadingPartialFailure" phase signifies that the main part of the backup, -including snapshotting has completed, but there were partial failures either -during the main part or during the uploading. The backup cannot be restored -from when it is in the UploadingPartialFailure state. - - diff --git a/changelogs/unreleased/3823-dharmab b/changelogs/unreleased/3823-dharmab deleted file mode 100644 index d441a228c..000000000 --- a/changelogs/unreleased/3823-dharmab +++ /dev/null @@ -1 +0,0 @@ -Add --client-page-size flag to server to allow chunking Kubernetes API LIST calls across multiple requests on large clusters diff --git a/changelogs/unreleased/3845-sseago b/changelogs/unreleased/3845-sseago deleted file mode 100644 index 0abe890fe..000000000 --- a/changelogs/unreleased/3845-sseago +++ /dev/null @@ -1 +0,0 @@ -Fix CR restore regression introduced in 1.6 restore progress. diff --git a/changelogs/unreleased/3857-zubron b/changelogs/unreleased/3857-zubron deleted file mode 100644 index 9037a0bf5..000000000 --- a/changelogs/unreleased/3857-zubron +++ /dev/null @@ -1 +0,0 @@ -Use region specified in the BackupStorageLocation spec when getting restic repo identifier. Originally fixed by @jala-dx in #3617. diff --git a/changelogs/unreleased/3866-alaypatel07 b/changelogs/unreleased/3866-alaypatel07 deleted file mode 100644 index 7480bc062..000000000 --- a/changelogs/unreleased/3866-alaypatel07 +++ /dev/null @@ -1 +0,0 @@ -skip backuping projected volume when using restic \ No newline at end of file diff --git a/changelogs/unreleased/3877-zubron b/changelogs/unreleased/3877-zubron deleted file mode 100644 index c4020a4bc..000000000 --- a/changelogs/unreleased/3877-zubron +++ /dev/null @@ -1 +0,0 @@ -Skip the restore of volumes that originally came from a projected volume when using restic. \ No newline at end of file diff --git a/changelogs/unreleased/3912-ywk253100 b/changelogs/unreleased/3912-ywk253100 deleted file mode 100644 index 81352675c..000000000 --- a/changelogs/unreleased/3912-ywk253100 +++ /dev/null @@ -1 +0,0 @@ -Run the E2E test with kind(provision various versions of k8s cluster) and MinIO on Github Action \ No newline at end of file diff --git a/changelogs/unreleased/3919-jaidevmane b/changelogs/unreleased/3919-jaidevmane deleted file mode 100644 index 61925a651..000000000 --- a/changelogs/unreleased/3919-jaidevmane +++ /dev/null @@ -1 +0,0 @@ -Fix -install-velero flag for e2e tests \ No newline at end of file diff --git a/changelogs/unreleased/3926-jenting b/changelogs/unreleased/3926-jenting deleted file mode 100644 index ffb8f178c..000000000 --- a/changelogs/unreleased/3926-jenting +++ /dev/null @@ -1 +0,0 @@ -Upgrade Velero ClusterRoleBinding to use v1 API diff --git a/changelogs/unreleased/3941-sseago b/changelogs/unreleased/3941-sseago deleted file mode 100644 index 17751a5ff..000000000 --- a/changelogs/unreleased/3941-sseago +++ /dev/null @@ -1 +0,0 @@ -enable e2e tests to choose crd apiVersion diff --git a/changelogs/unreleased/3983-jaidevmane b/changelogs/unreleased/3983-jaidevmane deleted file mode 100644 index 967d4b6b3..000000000 --- a/changelogs/unreleased/3983-jaidevmane +++ /dev/null @@ -1 +0,0 @@ -Fixing multipleNamespaceTest bug - Missing expect statement in test \ No newline at end of file diff --git a/changelogs/unreleased/3990-reasonerjt b/changelogs/unreleased/3990-reasonerjt deleted file mode 100644 index 44a82e2c6..000000000 --- a/changelogs/unreleased/3990-reasonerjt +++ /dev/null @@ -1 +0,0 @@ -Bump up Go to 1.16 \ No newline at end of file diff --git a/changelogs/unreleased/3993-mahaupt b/changelogs/unreleased/3993-mahaupt deleted file mode 100644 index 9e0312c2b..000000000 --- a/changelogs/unreleased/3993-mahaupt +++ /dev/null @@ -1 +0,0 @@ -Fix restic error when volume is emptyDir and Pod not running diff --git a/changelogs/unreleased/3996-ywk253100 b/changelogs/unreleased/3996-ywk253100 deleted file mode 100644 index 2e9534315..000000000 --- a/changelogs/unreleased/3996-ywk253100 +++ /dev/null @@ -1 +0,0 @@ -Select the velero deployment with both label and container name \ No newline at end of file diff --git a/changelogs/unreleased/4007-ywk253100 b/changelogs/unreleased/4007-ywk253100 deleted file mode 100644 index 4e052ce95..000000000 --- a/changelogs/unreleased/4007-ywk253100 +++ /dev/null @@ -1 +0,0 @@ -Wait for the namespace to be deleted before removing the CRDs during uninstall. This deprecates the `--wait` flag of the `uninstall` command \ No newline at end of file diff --git a/changelogs/unreleased/4015-zubron b/changelogs/unreleased/4015-zubron deleted file mode 100644 index 937ba33da..000000000 --- a/changelogs/unreleased/4015-zubron +++ /dev/null @@ -1 +0,0 @@ -Use the cluster preferred CRD API version when polling for Velero CRD readiness. \ No newline at end of file diff --git a/changelogs/unreleased/4022-reasonerjt b/changelogs/unreleased/4022-reasonerjt deleted file mode 100644 index 5457fd7e2..000000000 --- a/changelogs/unreleased/4022-reasonerjt +++ /dev/null @@ -1 +0,0 @@ -Implement velero debug \ No newline at end of file diff --git a/changelogs/unreleased/4028-zubron b/changelogs/unreleased/4028-zubron deleted file mode 100644 index 79a32656d..000000000 --- a/changelogs/unreleased/4028-zubron +++ /dev/null @@ -1 +0,0 @@ -Add a RestoreItemAction plugin (`velero.io/apiservice`) which skips the restore of any `APIService` which is managed by Kubernetes. These are identified using the `kube-aggregator.kubernetes.io/automanaged` label. \ No newline at end of file diff --git a/changelogs/unreleased/4055-ywk253100 b/changelogs/unreleased/4055-ywk253100 deleted file mode 100644 index 4dd9a9c85..000000000 --- a/changelogs/unreleased/4055-ywk253100 +++ /dev/null @@ -1 +0,0 @@ -Change the base image to distroless \ No newline at end of file diff --git a/changelogs/unreleased/4064-kahirokunn b/changelogs/unreleased/4064-kahirokunn deleted file mode 100644 index 877a6e864..000000000 --- a/changelogs/unreleased/4064-kahirokunn +++ /dev/null @@ -1 +0,0 @@ -Updated the version of velero/velero-plugin-for-aws version from v1.2.0 to v1.2.1 diff --git a/changelogs/unreleased/4497-dkeven b/changelogs/unreleased/4497-dkeven new file mode 100644 index 000000000..63d246e42 --- /dev/null +++ b/changelogs/unreleased/4497-dkeven @@ -0,0 +1 @@ +Skip not mounted volumes when backing up diff --git a/changelogs/unreleased/4517-reasonerjt b/changelogs/unreleased/4517-reasonerjt new file mode 100644 index 000000000..570af0b5e --- /dev/null +++ b/changelogs/unreleased/4517-reasonerjt @@ -0,0 +1 @@ +Update doc for v1.8 \ No newline at end of file diff --git a/changelogs/unreleased/4545-mqiu b/changelogs/unreleased/4545-mqiu new file mode 100644 index 000000000..bc9eca647 --- /dev/null +++ b/changelogs/unreleased/4545-mqiu @@ -0,0 +1 @@ +Add E2E test of backups sync from BSL diff --git a/changelogs/unreleased/4550-dbrekau b/changelogs/unreleased/4550-dbrekau new file mode 100644 index 000000000..dea16b567 --- /dev/null +++ b/changelogs/unreleased/4550-dbrekau @@ -0,0 +1 @@ +Fix: OrderedResources in Schedules diff --git a/changelogs/unreleased/4584-bynare b/changelogs/unreleased/4584-bynare new file mode 100644 index 000000000..008c138c4 --- /dev/null +++ b/changelogs/unreleased/4584-bynare @@ -0,0 +1 @@ +Skip volumes of non-running pods when backing up diff --git a/changelogs/unreleased/4591-mqiu b/changelogs/unreleased/4591-mqiu new file mode 100644 index 000000000..fb68c9305 --- /dev/null +++ b/changelogs/unreleased/4591-mqiu @@ -0,0 +1 @@ +E2E SSR test add retry mechanism and logs diff --git a/changelogs/unreleased/4634-qiuming-best b/changelogs/unreleased/4634-qiuming-best new file mode 100644 index 000000000..b3569ea03 --- /dev/null +++ b/changelogs/unreleased/4634-qiuming-best @@ -0,0 +1 @@ +Fix E2E backup namespaces test diff --git a/changelogs/unreleased/4639-jxun b/changelogs/unreleased/4639-jxun new file mode 100644 index 000000000..a027b1b00 --- /dev/null +++ b/changelogs/unreleased/4639-jxun @@ -0,0 +1 @@ +Update image used by E2E test to gcr.io diff --git a/changelogs/unreleased/4680-jxun b/changelogs/unreleased/4680-jxun new file mode 100644 index 000000000..9d6b56ba2 --- /dev/null +++ b/changelogs/unreleased/4680-jxun @@ -0,0 +1 @@ +Support regional pv for GKE \ No newline at end of file diff --git a/changelogs/unreleased/4692-jxun b/changelogs/unreleased/4692-jxun new file mode 100644 index 000000000..b18f53f86 --- /dev/null +++ b/changelogs/unreleased/4692-jxun @@ -0,0 +1 @@ +Add GINKGO_SKIP to support skip specific case in e2e test. \ No newline at end of file diff --git a/config/crd/v1/bases/velero.io_backups.yaml b/config/crd/v1/bases/velero.io_backups.yaml index 9aeeb5d98..fec76fbee 100644 --- a/config/crd/v1/bases/velero.io_backups.yaml +++ b/config/crd/v1/bases/velero.io_backups.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.7.0 creationTimestamp: null name: backups.velero.io spec: diff --git a/config/crd/v1/bases/velero.io_backupstoragelocations.yaml b/config/crd/v1/bases/velero.io_backupstoragelocations.yaml index bdef97594..24ac9e945 100644 --- a/config/crd/v1/bases/velero.io_backupstoragelocations.yaml +++ b/config/crd/v1/bases/velero.io_backupstoragelocations.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.7.0 creationTimestamp: null name: backupstoragelocations.velero.io spec: diff --git a/config/crd/v1/bases/velero.io_deletebackuprequests.yaml b/config/crd/v1/bases/velero.io_deletebackuprequests.yaml index d267d7310..761036ddb 100644 --- a/config/crd/v1/bases/velero.io_deletebackuprequests.yaml +++ b/config/crd/v1/bases/velero.io_deletebackuprequests.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.7.0 creationTimestamp: null name: deletebackuprequests.velero.io spec: diff --git a/config/crd/v1/bases/velero.io_downloadrequests.yaml b/config/crd/v1/bases/velero.io_downloadrequests.yaml index ed98240fd..45f229c44 100644 --- a/config/crd/v1/bases/velero.io_downloadrequests.yaml +++ b/config/crd/v1/bases/velero.io_downloadrequests.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.7.0 creationTimestamp: null name: downloadrequests.velero.io spec: @@ -46,6 +46,7 @@ spec: - BackupLog - BackupContents - BackupVolumeSnapshots + - BackupItemSnapshots - BackupResourceList - RestoreLog - RestoreResults diff --git a/config/crd/v1/bases/velero.io_podvolumebackups.yaml b/config/crd/v1/bases/velero.io_podvolumebackups.yaml index 1e318061b..80364ecbe 100644 --- a/config/crd/v1/bases/velero.io_podvolumebackups.yaml +++ b/config/crd/v1/bases/velero.io_podvolumebackups.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.7.0 creationTimestamp: null name: podvolumebackups.velero.io spec: diff --git a/config/crd/v1/bases/velero.io_podvolumerestores.yaml b/config/crd/v1/bases/velero.io_podvolumerestores.yaml index 444c1fc98..2ca79c571 100644 --- a/config/crd/v1/bases/velero.io_podvolumerestores.yaml +++ b/config/crd/v1/bases/velero.io_podvolumerestores.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.7.0 creationTimestamp: null name: podvolumerestores.velero.io spec: diff --git a/config/crd/v1/bases/velero.io_resticrepositories.yaml b/config/crd/v1/bases/velero.io_resticrepositories.yaml index 4a67f1ee8..65ff08eb3 100644 --- a/config/crd/v1/bases/velero.io_resticrepositories.yaml +++ b/config/crd/v1/bases/velero.io_resticrepositories.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.7.0 creationTimestamp: null name: resticrepositories.velero.io spec: diff --git a/config/crd/v1/bases/velero.io_restores.yaml b/config/crd/v1/bases/velero.io_restores.yaml index 8335c2a55..09df605e5 100644 --- a/config/crd/v1/bases/velero.io_restores.yaml +++ b/config/crd/v1/bases/velero.io_restores.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.7.0 creationTimestamp: null name: restores.velero.io spec: @@ -205,8 +205,10 @@ spec: are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be - unchanged. The $(VAR_NAME) syntax can - be escaped with a double $$, ie: $$(VAR_NAME). + unchanged. Double $$ are reduced to a + single $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: @@ -221,12 +223,14 @@ spec: references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the - input string will be unchanged. The $(VAR_NAME) - syntax can be escaped with a double $$, - ie: $$(VAR_NAME). Escaped references will - never be expanded, regardless of whether - the variable exists or not. Cannot be - updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + input string will be unchanged. Double + $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string + literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of + whether the variable exists or not. Cannot + be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array @@ -244,17 +248,19 @@ spec: value: description: 'Variable references $(VAR_NAME) are expanded using the - previous defined environment variables + previously defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. - The $(VAR_NAME) syntax can be escaped - with a double $$, ie: $$(VAR_NAME). - Escaped references will never be - expanded, regardless of whether - the variable exists or not. Defaults - to "".' + Double $$ are reduced to a single + $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal + "$(VAR_NAME)". Escaped references + will never be expanded, regardless + of whether the variable exists or + not. Defaults to "".' type: string valueFrom: description: Source for the environment @@ -804,6 +810,30 @@ spec: required: - port type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds + the pod needs to terminate gracefully + upon probe failure. The grace period + is the duration in seconds after the + processes running in the pod are sent + a termination signal and the time + when the processes are forcibly halted + with a kill signal. Set this value + longer than the expected cleanup time + for your process. If this value is + nil, the pod's terminationGracePeriodSeconds + will be used. Otherwise, this value + overrides the value provided by the + pod spec. Value must be non-negative + integer. The value zero indicates + stop immediately via the kill signal + (no opportunity to shut down). This + is a beta field and requires enabling + ProbeTerminationGracePeriod feature + gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults @@ -862,6 +892,7 @@ spec: by services. type: string protocol: + default: TCP description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults to "TCP". @@ -1006,6 +1037,30 @@ spec: required: - port type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds + the pod needs to terminate gracefully + upon probe failure. The grace period + is the duration in seconds after the + processes running in the pod are sent + a termination signal and the time + when the processes are forcibly halted + with a kill signal. Set this value + longer than the expected cleanup time + for your process. If this value is + nil, the pod's terminationGracePeriodSeconds + will be used. Otherwise, this value + overrides the value provided by the + pod spec. Value must be non-negative + integer. The value zero indicates + stop immediately via the kill signal + (no opportunity to shut down). This + is a beta field and requires enabling + ProbeTerminationGracePeriod feature + gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults @@ -1017,7 +1072,7 @@ spec: resources: description: 'Compute Resources required by this container. Cannot be updated. - More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' properties: limits: additionalProperties: @@ -1028,7 +1083,7 @@ spec: x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object requests: additionalProperties: @@ -1043,12 +1098,14 @@ spec: a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. - More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object securityContext: - description: 'Security options the pod should - run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ + description: 'SecurityContext defines the + security options the container should + be run with. If set, the fields of SecurityContext + override the equivalent fields of PodSecurityContext. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' properties: allowPrivilegeEscalation: @@ -1217,6 +1274,25 @@ spec: is the name of the GMSA credential spec to use. type: string + hostProcess: + description: HostProcess determines + if a container should be run as + a 'Host Process' container. This + field is alpha-level and will + only be honored by components + that enable the WindowsHostProcessContainers + feature flag. Setting this field + without the feature flag will + result in errors when validating + the Pod. All of a Pod's containers + must have the same effective HostProcess + value (it is not allowed to have + a mix of HostProcess containers + and non-HostProcess containers). In + addition, if HostProcess is true + then HostNetwork must also be + set to true. + type: boolean runAsUserName: description: The UserName in Windows to run the entrypoint of the container @@ -1369,6 +1445,30 @@ spec: required: - port type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds + the pod needs to terminate gracefully + upon probe failure. The grace period + is the duration in seconds after the + processes running in the pod are sent + a termination signal and the time + when the processes are forcibly halted + with a kill signal. Set this value + longer than the expected cleanup time + for your process. If this value is + nil, the pod's terminationGracePeriodSeconds + will be used. Otherwise, this value + overrides the value provided by the + pod spec. Value must be non-negative + integer. The value zero indicates + stop immediately via the kill signal + (no opportunity to shut down). This + is a beta field and requires enabling + ProbeTerminationGracePeriod feature + gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults diff --git a/config/crd/v1/bases/velero.io_schedules.yaml b/config/crd/v1/bases/velero.io_schedules.yaml index fd76c5c3a..afb7b14a0 100644 --- a/config/crd/v1/bases/velero.io_schedules.yaml +++ b/config/crd/v1/bases/velero.io_schedules.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.7.0 creationTimestamp: null name: schedules.velero.io spec: diff --git a/config/crd/v1/bases/velero.io_serverstatusrequests.yaml b/config/crd/v1/bases/velero.io_serverstatusrequests.yaml index 7ac36c8e8..7efbd9694 100644 --- a/config/crd/v1/bases/velero.io_serverstatusrequests.yaml +++ b/config/crd/v1/bases/velero.io_serverstatusrequests.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.7.0 creationTimestamp: null name: serverstatusrequests.velero.io spec: diff --git a/config/crd/v1/bases/velero.io_volumesnapshotlocations.yaml b/config/crd/v1/bases/velero.io_volumesnapshotlocations.yaml index 4a63234c3..56ef139d2 100644 --- a/config/crd/v1/bases/velero.io_volumesnapshotlocations.yaml +++ b/config/crd/v1/bases/velero.io_volumesnapshotlocations.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.7.0 creationTimestamp: null name: volumesnapshotlocations.velero.io spec: diff --git a/config/crd/v1/crds/crds.go b/config/crd/v1/crds/crds.go index 82f613324..b23a26318 100644 --- a/config/crd/v1/crds/crds.go +++ b/config/crd/v1/crds/crds.go @@ -29,17 +29,17 @@ import ( ) var rawCRDs = [][]byte{ - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec<]o#9r\xef\xfe\x15\x05\xe7a\xee\x00K\xbeŽ\x04~\x9b\xf5x\x11a7sƎ\xe3<\x04y\xa0\xbaK\x12\xcf\xddd\x87dˣ\x04\xf9\xefA\x15\xc9\xfeR\u007fP^\xcf\xdd^0|\x99q\x8b,\x16\xab\x8a\xf5\xc5\"\xafV\xabՕ\xa8\xe43\x1a+\xb5\xba\x03QI\xfc\xeaP\xd1_v\xfd\xf2\xcfv-\xf5\xed\xf1\x87\xab\x17\xa9\xf2;\xb8\xaf\xad\xd3\xe5\xafhum2\xfc\x84;\xa9\xa4\x93Z]\x95\xe8D.\x9c\xb8\xbb\x02\x10Ji'賥?\x012\xad\x9c\xd1E\x81f\xb5G\xb5~\xa9\xb7\xb8\xade\x91\xa3a\xe0q\xea\xe3\x9f\xd6\u007f^\xff\xe9\n 3\xc8ßd\x89։\xb2\xba\x03U\x17\xc5\x15\x80\x12%\xde\xc1Vd/ue\xd7G,\xd0\xe8\xb5\xd4W\xb6\u008c\xe6\xda\x1b]Ww\xd0\xfe\xe0\x87\x04<\xfc\x1a~\xe4\xd1\xfc\xa1\x90\xd6\xfd\xdc\xf9\xf8\x8b\xb4\x8e\u007f\xa8\x8aڈ\xa2\x99\x89\xbfY\xa9\xf6u!L\xfcz\x05`3]\xe1\x1d|\xa6)*\x91a~\x05\x10\x96\xc3S\xae\x02\xc2\xc7\x1f<\x84쀥\xf0\xb8\x00\xe8\n\xd5\xc7\xc7\xcd\xf3\x9f\xbf\xf4>\x03\xe4h3#+\xc7D\U00048074 \xe0\x99\x97\x05&\x90\x1f\xdcA80X\x19\xb4\xa8\x9c\x05w@\xc8D\xe5j\x83\xa0w\xf0s\xbdE\xa3Сm@\x03dEm\x1d\x1a\xb0N8\x04\xe1@@\xa5\xa5r \x158Y\"\xfc\xe1\xe3\xe3\x06\xf4\xf6\xaf\x989\vB\xe5 \xacՙ\x14\x0es8\xea\xa2.я\xfd㺁Z\x19]\xa1q2\xd2ٷ\x8eTu\xbe\x0e\x96\xf7\x81(\xe0{AN\xe2\x84~\x19\x81\x8a\x98\a\xa2\xd1z\xdcA\xdav\xb9,!=\xc0@\x9d\x84\nȯ\xe1\v\x1a\x02\x03\xf6\xa0\xeb\"')<\xa2!\x82ez\xaf\xe4\u007f7\xb0-8͓\x16\xc2a\x10\x80\xb6I\xe5\xd0(Q\xc0Q\x145\xde0IJq\x02\x834\vԪ\x03\x8f\xbb\xd85\xfc\xab6\bR\xed\xf4\x1d\x1c\x9c\xab\xec\xdd\xed\xed^\xba\xb8\x9b2]\x96\xb5\x92\xeet\xcb\x1bCnk\xa7\x8d\xbd\xcd\xf1\x88ŭ\x95\xfb\x950\xd9A:̈\x91\xb7\xa2\x92+F]\xf1\x8eZ\x97\xf9?E\x01\xb0\x1fz\xb8\xba\x13\t\xa3uF\xaa}\xe7\a\x96\xfa\x19\x0e\xd0\x06\xf0\xf2\xe5\x87\xfaU\xb4\x84\xa6OD\x9d_\x1f\xbe\xe9_\xd1:\x99\r{\r\x90\xfa4:(\"\x85\x16^\x0f\xe8\x0ehH~\xf8\aޒg0\x81Yj1\xe7\x1d)^\x10D\xc0\x9e\xb7vQ@\xa5\xa3\x16\xb2\xb0=Ed\xd7g\x90~\xfe\x84\xf9\x1cy M\xf2\xce\x16\xf2q\x80lw\xea\xe0\x94\xa7.#\xb8>M|\xe3S\x1a7 \xe0\x05O\xdec\x11\n\x889\x82&\x9a\x88tΉù\x15\x16\xb2\x17<1\x98\x90,Y\x1c\x9d*\n\xbe\xbd\xe0)\xa5ۀ\x80\x84\x93\xb4!\tD\x94\xa4\x0fL\b\x8e\xadӉ\a\x9c\xf8\x8a\xbahyq\x90\xaeHb\x8b\xb4\u007f\xc32\x1b\xb6u\x92\x86\xcc\xd8\x0fֳ\x88v\xc1AV\x89\v%3\a\x16y\xb7\xc4\xd4׳(d\xdeL\xe4\xe5~\xa3\xa6\xbd\xe1~\xfb\xac\xddF\xdd\xc0\xc3WiC\xf6\xf1\x93F\xfbY;\xfe\xf2M\xc8\xe9\x11\u007f\x031\xfd@\xde^ʫm\xa2C7\x87\x96 ܾm|\x84װGZ\xd8(\x8a[\x02=8#\ua9db\xb7\x0f\xfdV֖\x93dJ\xab\x15\x9b\xca\xf5\xd8L\x9e؉ \xb5\xe9q\xe4\x1c\xb5fR?a\"\xd8'\xb2$~\xbc\xcf\xf1\x16\"\xc3\x1c\xf2\x9a\x89əI\xe1p/3(\xd1\xec\xe7\fG\xb7U\xa4\xdf\xd3PHԺ\xbe](ai\xa6=\xb6\xa0\xba\xf3edV\xb4s\x13zEf/v\x9dHHNw]^\x11\x9bX\xf6?\x16\xa9+\xf2\x9cϒD\xf1x\x81ƿ\x80\x17\xe7\xb6\xdf#\xe6-d)8\xc9\xf8?d\xe6X\xa0\xff\x17*!M\xc2\x1e\xfe\xc8GC\x05\xf6Ɔ,Vw\x1a\x9aAZ \xfe\x1eEq\x9e\xea\x1eY\x9c&݂\x857\xe4zw\xe6\xb1\xdc\xc0\xebA[oSw\x12GS\xaa\xfd&-\\\xbf\xe0\xe9\xfa\xe6L\x0f\\oԵ7\xf0\x17\xab\x9b\xc6[Ъ8\xc15\x8f\xbd\xfe-NP\xa2$&u\xe3#\xb8TW\x99b\xc9\xe8\t\xd0\xc0\xe6܉\xdc\xdc9\xac\x93\xe4\xb0\xd2\xd6%\xa3\xf2\xa8\xad\xf3\x99Ş[zI\x16\v\xbc\f\x85\xec\x15\x88\x9d?\xf9\xd3&\x9e\xe9\x90\xda\x1b$\\\x89kv^\xc3\x12\x1b\x9b\x8c\x98\aJ\x81\xd5u\xbb\x83\xbd>\xbd\xf6\a=<\x89\xc8عX\x84[\x19\x9d\xa1\xb5\xf3\"\x92\xa0\xad\x17\x92\x84M\x82P\xf8\x00\xc6\x1f\x98\xcc'%cKwH\x89H\x17\xba\xf2\x0f_;\xd9K\xda\xfc\xf4\xf7\x92\xf0]\x8a\x17\xf0\x9e-K1<\x19LB\xf1ޏ\x8c\xdb$\x00\xf2\xa1\x81\xd9\u05fc\xd5\xd3=\xc8 H\xbf\a3]J\xb5\xe1\t\xe0\x87w7덒ķ8\xee\xf7qlK\xf4\xe6\x03\xef\xdeT\x8fHs\xe6\xde`\x8fs\xe7ynr\x14\x13A*\xed\xba\xe9\x04\x82[\xe9\xfc\x83\x85\x9d4\xd6u\x11M\x15\x8aza\xf7\xb7\xed\xd2\xc8I=\x18\xf3\xa6\xc0\xe9/~d'\x91uЯ\xf1|u\xf20s\xac\xf1\xa1\x10\x82܁t\x80*ӵ\xe2\xf4\vmu\x9e³\xc0+\xe8d\x92\xa5)\bj\xa8\xea2\x8d\x00+\x96:\xa9f\xf34\xdd\xee?\tY|\v\xb69Y\xa2\xaeg\rg\xdbzl{\xf2#{\a\xe5\xa5\xf8*˺\x04Q\x12\xe9SÞ\x9d/\x8e\xe9q\x1c^\x85tl9\b.\x9b\x11\xa7iSU\x05\xba\xd4\x1d\xb9ŝ6\xbc\x9f\xaḏ1\xccA\n\xb4\x02\x01;!\x8b\xda$jȋh{I\xac\x11\x94\xc5\xfb\x05\x11i\x93\xaf\x98\x14\t\x89\xd8Dgq^[W&\xddU|4\x98\xe6\x9e-%\xa5\xa3{V\x19I\xb2\xa4\xdf\xdbC\v\"&\xd4黋v־\xbbh\v\xed\xbb\x8b6پ\xbbh\xcb\xed\xbb\x8b\x16\xdaw\x17-\xb6\xef.\xdaw\x17m\xaeۜ\xb6^\xc2\xc8W\xdcO\xfc\xb8\x88E\xc2\xf1\xf4\x1c\x8a3\xf0C5Ž\xaf\xbeO\xad\xb0܌\x8f\x1a\xa9\xab\re\xfd+\xbe\x910&\x01m\xd1EkJ\x9a\x92K\xda Q\xbc}\x01\xf1B\x11fR9\xe5x\xf5mJ\xc1\xcfR\x99O\xbfδ)\xb3\x89\x85\xa6:N2B\x87x\xb3\x81\xdc\xden\rI\xbf^\x87\xfd܈\xe9߽\x065\xa1\x14g\xa1\x00g\xbe0w\x8e^\x83УO0\xd3+\x18\xfd\xdd\xd0k\xa1Jf\xba6&\x9c\x04\xa1\x13\xc7\x1f\xd6\xfd_\x9c\x0e\x952\xf0*\xddad)\xaf\aT|\x86\xa5\xf6ݲ\xd7(o\xe1\x8aɐ\x8e\xa0\r(Y09g\xa4\xb5G^\xf8K\xe5C\xb8\x8b\xf7\xe5|\xf8\x91VK\xf3\xe6\n\x9a~\x85̄\x8a\xbe\xf4\xc8(\xbdP8\xbdFf\xbe\xa8\xe5\x92ʘa\xdd\xcb$\xd0\xe5z\x98\x94\xc8q\xa1\xf6\xe5\r\x15/\x89Վ\xbf\xf9`,\xa5\xa6\xe5M\x95,\x8b\x05\x81\x89\xf5+\xfdʔy\x90\x17T\xad$\x11g\xb9B\xe5⺔P\a2\xbb\x8e\xe4j\x94\x91:\x93Y\xc0\x935(s\xd5%\vY\xa9\xf3ʓ\xf4\x9a\x92Y\xd0\\o\xb2\\I\xf2~\xf5\xa2\xef\xe1\x03O\xab\x9a\xc5j\x90E\x1fy\x1e\xbf\xc5z\x8fK\xaa<\x16)\xf6Ɗ\x8e\xa6bcb\xdeK\xeb8\xfau\x1a\x13@S\xaa7&\xaa3& \xce\xd6l\xa4\xd6dL\xc0^0\xbb\xb3R2\xf3\xe3\xf8EHX\xb4o\xc5\xdfJ\xa2\u07ba0mr4\xb3\x1ez*\x9a\xb3(\xf63^\x839\ae\xf6\xf1\xe2$\xf5\xeaz\xfdc,\xd7MIx\x06?K\x95{9!A\xef\xf8\t|Q\x98\x8bb\x1aw\xa5\xf5\xf7Ɓ\x0e\"\r\x8b\x950|\x93|{\xf2\xd9\n\xbb\x86\a\x91\x1d\xfa\x1d\xe1 ,Ť\xe5\xa8\x1bv݄i\xb7q\x14}\xb9^\x03\xfc\xa4\x9bH\xb8{\xcd\xcaʲ*NP[\x84\xeb\xfe\x90\xb7\xc5\x1c\xa3\x12`\x95\xa8\xecA\xc7;\xb0\vaǗ~\uf448>ހ\xcd\n]\xe7\r\xf4\t\xe6\tu\x82\xc7g\xf6}\xf8\xee`\xd6ޣ\f\xfeM\x8c$\x86\xd7,\u007f|\xff\b\xdf:m\xc4\x1e\u007f\xd1\xfe2\xf2\x12%\xfa\xbd{7у\x0e\x8b\x19\xb7X\x90%F\x88\x10\xaeE\x0f\x80\xb5\x89\xf4\xb0\x1b\xda\xe4\aa9\xa6\xdef\xf6\x9fs\xc5\xc2b\x9e\x9e~\xf1\vp\xb2\xc4\xf5\xa7\xdagSV\x950\x16\x89\x9aqa~Ж\xfe{Яc\nO\x875\xff8\xc4\xdb '\xeb9is\x11\xf6\xfe\xdat\x14\xbcH\xa2%A}\x1e\x1f\xd5\t\xf4:L\xf2\xbb\\\x8f\x9dKL\xc1\xe9\xbc.A\x81\xb5/\xb6{\u07fb\xbfS\x1e\xcb\xd4\xfd{'\\m\x97o\xe0s\xb7\xf8\xdeF8\xf2\xa9\r_\xdc\xf5 \xfcE\xd77]\xc2\x0f\x19\xea\xde\x1b(\xf3|\xba?\x1f\xc1/]\x98ܣƹ\xf1\xe66\xfd\xab\xb0M\x16|\xd4ķ\xe0\xfcH\xf6\xa0\t\x1a\xe6\x80GT\xa0\x15'\xbd\xf9J\xac\u007f\x8de8f,\x99ԁ\x12\xb2\xeauUh\x91\xc7\x1d\x1emVx\xc1\xe3\x89\xf5\x979\xa2\xf9`g`\xf2\x8b\x01;mƈp\xae0\xbda\xb9\x83\\8\\\x8d\x02M\xd2}\xa3\xc2\xc6\xe7C\x8b\xf7湓\xdfA|\xb8\x14\x1f7\xf0\xa7K%Z+\xf6\xf1\xc2\xfc+)\xb0=*d\xbb9\xb2\xfe\x10ϴ\xa7\x11\xfd\xeb\xe2>\xa5\"2W\x8b0A\xcc&uz}\x183+\x85\xde\xc3N\x16\xdc5<\xed\x114\xfb\x94ڑ\xca\xe1\x1e\x87\xee*~\xad\xa4I\xb1\x04\x0fMG\xa2\r\xe7\xd3X\x1b\xb4O\xe0`!\xf7\x92\xd4(1{/\xccV\xecq\x95\xe9\xa2@.]8\xc7\xeb[\xf2\xda\xc3\x1e}\xe2\xe6li?u\xfbF\u007f*\b\xbb\x87\x13_\xbc\xb9\t\x16z\xdcA-\xc5_\xb5\xb9\x81R*\xfa\x87\xdc0\x8eL\xe3\xe0\x8b\xec\x01\xbf.\xb0\x80\xf7#\xf5i\x0e\xb4;\xda\r\xa3\x98M\xf9\x0f㇘+\xf8\x8c\xe7\xe6ΟKbι\x97\xb1w}\xa8\xcbF=\x1a\xbd\xa7\xd0i\xe4\xc7\xfb\xa8\xcbF~{\x14\xc6IQ\x14'?\xc9\xe4\xec#?|B\xd2&\x93&e\x9c\xac\x01\xcb%ʆnm\x9c&\x95\x97\x04>Q\xdc\xea\xda\xf56h\xbb\xc1G\xc4\"ι\x86\xcf\xdaaL\xe3\xc9>L\xb2\xabh\xdd\nw;m\x9c\x0f\vW+\x90\xbb`\xa2F\xe0\x92\x86\xe7Ե\u007ff\a\xa4k\xd3'\xad\xf4\xb2\xf7iPX\x96^Ǐ\xfd\xf0\xe9\x92\xc82\xf2\x80\xf0\xd6:Q\x8ch\x8dߔ\xadf_\x80\xa4\x0f\xf3\u007f\x1b1\x8eg\x04\xdft\xfb7%\xfeu\xb9\xf5\xd1\x11\x83\xf3\x94\xe3\xd3\u007f\xaf1\x8b\xa9<\xca\x16Q\xc1\xab\x91Α\x96\xea\xe6\xf6\xc1\x91^*\n\xb0\x1avb\xe2A\x879}ɿk'\x8a\xcdtN\xa9\xeft6\x9d\xe3\xb2x\xf8\xf9\xe24\xb1e\xcb$\x98X\x96\xaf.\x936\x8e%Vf\a\xa1\xf6$TF\xd7\xfbC\x94\xcb\t{3\x95\x89\xaf\t)\xa8\x8azO\xa2\x1er\xe3\xae6\xaa\x13\x17\x87ly\xdeAWd/\x93\x98\x86\xec`|\xea\xed6<Ѱ\xda\x19]\xae\x02/8op\x13\xe2U#59e\x14]M\x00m\xefB\xb3\x18T\x15*\x106\xe0\x93P\xfa6\xcfֹ\xe0\xd1\t\xe3R]\xbd/\xbd\xce\v^\x1eC\x1e\xc7\xf7K\x88\xc6}\t\xe0\xfd\xf0\xd1=\x8a\x9bU|e\xceG\xfb^\x14,9\u007f\x069\x80\x1a=\xaf8s\xdbzNZ\x1f\xfd\xbf\xad\u007fvl,\xccC\x8a\xa7\xf6<\xe8>8G\xa5]\xdeB\f\xde\xd5\b=\xfe w\xfe %#\xac\xff\xf8w?\x1f=&\xf9,\x1ff\xdd\x15\xf6D\x1a\xbf\x03>ae0\xa3\xdd;\xb6\x8c\xc7\x02ɏ\xb0\x88}O\xe8\xc3E\x8ed?\x80\xb5\x1f\x9dò\x1a\x9dq&\x82m\x87M)K\x11;\x8c,$>\x80\x18\x81\x85Ң\x99\x90\xf5\x82\x055N\xcce\vj\x86M-\xc8\xd6\x19)\xad]=nΚ8\xf0\x9dW\xf7*\x8c\x92j\xbf\xb4\xc7\xfe=t\x1b\x89\x87\x02\x84\x91\x88hd\x19M\x8c\xb4\x18\x11u\x02\xa2\x88\xe3Ļd\x83 \xe9\x9dB\xa2Q;p\xf6\x91\x15h\xde\xd9\xdba\xa6\xf0\xa5\xcdR\x88,C\x12\xd7\xcfÇN\xaf\xaf\xf9\x8f\xf8\x96)\xff\x99i\xe5ͭ\xbd\x83\xff\xf8\xcf+\bi\xb0\xe7\xf8h)}\xfc\xbf\x00\x00\x00\xff\xff\bB\tb\x14V\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4YK\x8f\xe3\xb8\x11\xbe\xfbW\x14f\x0f}\x19˳\xd9K\xe0K\xe0\xf1$\xc0 =ۍq\xa7sH\x02,M\x96lnS\xa4BR\xf6:A\xfe{P|H\xb2$\xb7݃d\x97\x97n\xf3Q\xac\xfa\xeaM\xcd\xe6\xf3\xf9\x8c\xd5\xf2\x19\xad\x93F/\x81\xd5\x12\u007f\xf1\xa8\xe9\x97+^~\xef\ni\x16\x87\xefg/R\x8b%\xac\x1b\xe7M\xf5\x15\x9di,\xc7OXJ-\xbd4zV\xa1g\x82y\xb6\x9c\x010\xad\x8dg4\xed\xe8'\x007\xda[\xa3\x14\xda\xf9\x0eu\xf1\xd2lq\xdbH%\xd0\x06\xe2\xf9\xeaÇ\xe2\x87\xe2\xc3\f\x80[\fǟd\x85γ\xaa^\x82n\x94\x9a\x01hV\xe1\x12\xb6\x8c\xbf4\xb5\xf3Ʋ\x1d*\xc3\xe3]\xc5\x01\x15ZSH3s5r\xbazgMS/\xa1[\x88\x14\x12[Q\xa4\x8f\x81\xd8&\x12\xbbO\xc4º\x92\xce\xff\xf9\xf2\x9e{\xe9|\xd8W\xab\xc62u\x89\xad\xb0\xc5\xed\x8d\xf5?vW\xcfa\xebT\\\x91z\xd7(f/\x1c\x9f\x018nj\\B8]3\x8eb\x06\x900\v\xd4\xe6\xc0\x84\bZ`\xea\xd1J\xedѮ\x8dj*\xdd\xde%\xd0q+k\x1fP\x8e\xb2@\x12\x06\xb24\xe0<\xf3\x8d\x03\xd7\xf0=0\a\xab\x03\x93\x8am\x15.\xfe\xa2Y\xfe?\xd0\x03\xf8\xd9\x19\xfd\xc8\xfc~\tE\xfc\xe0J\xa2\x1e\xc2\xef\x9am%=\xe9\xfd\x9f\r:O\xba*`\x1d2\x16l\x11\x9a:\xf8}\x01\x9f5\xacY\x85j\xcd\x1c\xfe\xdf\x15@H\xbb9\x01{\x9b\n\xfa\xc9v\xb89\xa2\xd6[ȹ\xf0\x82\xbe&\xbdxS#?\xf3\x1f\x81NZ\xb2p\xcf<\x06\xbf\x18\xe0\x9a\\\xfcr2\xcdcڹi0\xceѹ/F\xe0pe\xc0\xf2\xaa\xddx\xc6c\x8d\xb6\x92.\xa4E(\x8d\x1df\f\xd6F\xe0\xfeȑ\xaa\x18\xad\xa1n\xaa1#s\xf8\x8aL\x86\x8d\x19\xe3x\x8cxi\x1c\x86\xe0{\x8d\r\xb8n㜭\xd1\xde\xc2\xcbzE\x1b\xdb4\xc9`\xbd\x82m\xa3\x85\xc2\xcc\xd1q\x8f\x9a:\x04Y\x9e\xa6\xef\xa2\xf1t\xbfɨ\x86\n#\xd5\xf8\x19\xdbi\x19b\f_\xc2\xf64Q\x13\xdc dm\xb1\x94\xbf\xdc \xe4cؘ\x01\xaf\x99߃\xd4N\n\x046\x01\u007f,\xd6.\b\xda\xe6\xff\x87\x14E\xbeA=\xafy{d\xe7-\x0e\x9f1\xbe\xe2?\x8fi[\x8bB\xfe\x9d\"\xffy-xɏ'%:\xb4\x0f\x06\u007f\x8a\x15\x16\x9fH\x95g\xcc<\x8fO\xbcR\xa9\xe5g\x8b)g\xa6\xba\xc0X\x8b\xae6ZP\xf3t[\x9dֱ\xfc\xbf\xab֦\xd5:?\x8fr\x83\xb5\xac\x85\x9bZ\x95\xf0D\xf3\xe6f%>\\\xf5[\x01\xb3u\xd4)v\xfd\xca@\xc6_\xa5My\xd7\xebS\xa8\x1f\xd6\xd0\xe8P\xa9\x85\x8c_\xc0\xdf5|\xa2ޖ\xb2\x93X\x12\xdfv\xca\x00\xa4\x03m\x8et\xbcG/\x90\x00\xa3c\xbe\xa6n\x8di\x91\x9a\xe1\xb0t\x94JQƶX\x99\xc3dƦBӢ:\x01sd:\x87\xdf\x15\x1f\x8aw\xbfY\x17\xa4\x98\xf3\xd4Ԡ\xf8\x8a\a9~\xe5\x19\xa3{?:\x91\x1d\xbfu\a\xfa\xf1Sn\x96\x176m\xfbi\x02\x8cR*\xaa\x05'\xe2DW1\x8c\xdf#?n\xee\xef\\(\xe1Q\xfb\xa9\xb2\xef\x88\x16CDŽ\x82\xaax\x93\xde%\x1a\xe7\xd1N\x18@\xab\xbd\xa0sPF\xef\x06\x8e\x13Gz\xa5\xa0\n-\x1a\x94\xb1 \xd0Sj\xd2;\xe0{\xa6wؽB%\xfe_\xe7\x94\xccg`3\x9d\x85H}\xc9\xa6-n\x13\xf5\x0e\xb9\x14\x9f\x8e>|h?\xb5\x1f\x1a\x00\xcbXҿҀ\xa2f\x88\x1b\xf0\xa9\xef\x1b\x00o\x06܀\xc3\x1e\x15\xb7\xc6>\xa6\xc8\xf8gBQi\x0f\xd8#\x87\x96B#\x11m>x\xcf!\xc5\r\x1c\x1d5\u007f\x04U/\xf4\xb9\x94\xfa\xa9\x94\xba\xab\xa5\x8a\xb7'џ_\x8b\xf8\x85ƨ\xd8'6\xfd2\xa0\x12 \xe4\xf7\xa97\xbc\x18\xd2\x00\x88\r\x117p\x93aEc\xd15\x00#\x1f\x05\xe6j\xbc\xf1\xe1c-g;\x1cL\xc5\x0f\x10\"\xfa\x1fo\xaf\x1f>ݟ\x99\x01\x1c\x8ae\x8aZX]\xc0\x0f$``D\x01\x1aFp\x10\xa3\xacQ\xe0rϠ\x80v8\xdd\x14\xddx1\b;Ў\x04\x18#\xa3\xa0\xaf]tV\x18r\x90\xf1\x10\xb6\u007f\xa0\xd5\x16\xee\x91s\x19\x90.\xa4\xde\xe5V; +0ڰ\xf7\xf4\xf7Km\xc9\xf7̇\xf6F'\x91\x8f\x0fyE\xf6\xa6\x87\x83\xe9\x13~\x0f\xc6;\x18\xcc30\xe6S \xf9\x93z%DZ\xf85\xd3D~\x176ЩF٬\xd7{\xd2idl\x18\x86\xe4I\x9fץ\xfbi\x9b4\xb0\xac\x1d\x1e\xb0_\v\xedW\x86mG\x8aV\x13\xe3\xdaDZ\x15辌M;\xb8\xefx\x1c2y\u007f\x86U\x9fsÈ2\xf9\xfd\x89\xa3t\xf37\x14Ƚ\\e\xaf\xa9\xf5\x16G\xa2\xb3)\xb3s\xf7\xe5\xfe+LG\x171\xe6\xec\x17ޏ\x89r\x94 \x13F~\x87\\E\xdcq\x18JM\xf4.\x06\xf2Z^lO\xe8\xe7\xf4K\xda\x0e\xa42\xb5d֪\x85\xab\xb2G`\x8b\x90\xa23\x8a\xae\x85k\x0fWf\xc0\xfe\xca\b\xfe\xef\x02d\xa6e\x95\x89}\x9b\x04\xa7+p\x1e\\Y;qL;\xea\x15\xbd\x16\x86\xf6>\xa2\xcd\nf\x12s6\xedȖ\xf1\x80]`x\xea\xc8v\xd3\xd0\xce\xd8}\x19\xf0\xf6̱<\xd0\xf9\xa9e\xf2R\x9a{^\xbd<\x14\xed\x88qօ\xab\x93bo\xe2E\x8d&\xf9\x8f̔\x9c\x89\x1b\x9b\x98\xd1\xebX\xa9l\x8b\xa5\xa4\xb7r\x81́/\xac3P_JP\xf9\xce\x19\xf2\x02\xc6?\x8f\x89\xa0\x9dQxB\xcec`C\xca{\x06\x1d\xb8t\xc1\xdfHK\x87U\xac,l\xe4`Q\xa4\xbd\x88#\xc5a\x01\xd37\xd4\xc9O\xfe\x86\x9am\x8f\x1bPN\xf8\x8a\xb2\x86\xd9<\xcf|\xb13\xb2\xd0\ng\x14\xdc\xe6\x98%\r\xb0nu\xfcw\x11\n\xdd>\r\x97'\xad\xe0\x06\x9f\x16\xac\xd7\xfe\x96ÞQ\xe6-\x9f\x9d\xb7\x95\xbd\xf2M}#K\x8bMya\x94\xbc\xef\xdc\t\x8b\xa2\x81\xcd~\xe2\xf5\xd8\xc2\xc6Z\x8c\x8a\xeef\xfe\xd7\xf1\xee\xdd\xd9\xefCy\xb5\xc1;\xaa?M\xf0\xdb\xefM\xad\x8a\xeea\xfa\x1b\xc8\xc6\u007f\x02\x00\x00\xff\xff~\x96\x80P\xae\t\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4WMo\xdc6\x10\xbd\xebW\f\xd2CZ \xd2&ȥ\xd8[\xeb\xe4\x104\r\x02;\xf5\xa5\xe8\x81K\xcdJ\xac)\x92\xe5\f\xd7q\x8b\xfe\xf7bHi?\xb4Z\xdb9T\xb7%\x87\xc3\xc7\xc7\xf7\x86\xb3U]ו\n\xe6\x16#\x19\xef֠\x82\xc1\xaf\x8cN~Qs\xf7#5Ưvo\xaa;\xe3\xda5\\%b?\\#\xf9\x145\xbeíq\x86\x8dwՀ\xacZ\xc5j]\x01(\xe7<+\x19&\xf9\t\xa0\xbd\xe3\xe8\xad\xc5Xw蚻\xb4\xc1M2\xb6Ř\x93O[\xef^7o\x9b\xd7\x15\x80\x8e\x98\x97\u007f1\x03\x12\xab!\xac\xc1%k+\x00\xa7\x06\\C\xeb\xef\x9d\xf5\xaa\x8d\xf8WBbjvh1\xfa\xc6\xf8\x8a\x02jٴ\x8b>\x855\x1c&\xca\xda\x11P9̻1\xcduI\x93g\xac!\xfeei\xf6\xa3\x19#\x82MQ\xd9s\x10y\x92\x8c\xeb\x92U\xf1l\xba\x02 \xed\x03\xae\xe1\x93\xc0\bJc[\x01\x8cgϰ\xea\xf1t\xbb7%\x95\xeeqP\x05/\x80\x0f\xe8~\xfa\xfc\xe1\xf6\xed\xcd\xc90@\x8b\xa4\xa3\t\x9c\x19\x9ca\x06C\xa0`D\x00\xec\xf7\xa0@9P\x91\xcdVi\x86m\xf4\x03l\x94\xbeKa\x9f\x15\xc0o\xfeD\xcd@\xec\xa3\xea\xf0\x15P\xd2=(\xc9WB\xc1\xfa\x0e\xb6\xc6b\xb3_\x14\xa2\x0f\x18\xd9L,\x97\xefH\\G\xa33\xe0/\xe5l%\nZQ\x15\x12p\x8f\x13?؎t\x80\xdf\x02\xf7\x86 b\x88H\xe8\x8a\xceN\x12\x83\x04)7\x9e\xa0\x81\x1b\x8c\x92\x06\xa8\xf7ɶ\"\xc6\x1dF\x86\x88\xdaw\xce\xfc\xbd\xcfM\u0090lj\x15Or8|\xc61F\xa7,\xec\x94M\xf8\n\x94kaP\x0f\x101\xf3\x94\xdcQ\xbe\x1cB\r\xfc\xea#\x82q[\xbf\x86\x9e9\xd0z\xb5\xea\fO\xa6\xd2~\x18\x923\xfc\xb0\xca\xfe0\x9b\xc4>Ҫ\xc5\x1d\xda\x15\x99\xaeVQ\xf7\x86Qs\x8a\xb8R\xc1\xd4\x19\xba\xcb\xc6j\x86\xf6\xbb8ڐ^\x9e`\xe5\a\x91\x19q4\xae;\x9aȚ\u007f\xe4\x06D\xf5E0ei9Łh\x19\x12v\xae\xdf\xdf|\x81i\xeb|\x19s\xf6\x8br\xf6\v\xe9p\x05B\x98q[\x8c\xe5\x12\xb3\xf2$'\xba6x\xe38\xff\xd0֠\x9b\xd3Oi3\x18\xa6I\xccrW\r\\\xe5J\x03\x1b\x84\x14Z\xc5\xd86\xf0\xc1\xc1\x95\x1a\xd0^)\xc2\xff\xfd\x02\x84i\xaa\x85\xd8\xe7]\xc1q\x91\x9c\a\x17֎&\xa6Jv\xe1\xbefV\xbf\t\xa8\xe5\xf6\x84@Yi\xb6Fgk\xc0\xd6GP\a\xe7\x8f\x046'\x99\x97\x9d\x9b\xc1\xa9\xd8!\xcfGgX\xbe\xe4 \xd9\xfe\xbeW\xa7\x85\xe6{l\xbaFj\x05\x8d@J\xf5\xf8\xa19\xcbx\x19\x03,\xaaw\x11\xc9$b\xa1Ax\x95R E\xea\x18\xd3\xf9\xd6\xf2\xa1K\xc3\xf2\x065\xfc\x9c1\u007f\xf4ݣ\xf3Wޱ\xc8\xfdѠ[oӀ7N\x05\xea\xfd\x13\xb1\xd33\xbb\u007fz\xce\x03\xafQ\n4^\x866\x06\\#%{a\xbb\vb\x9d\xbe\xfc(=ͼ/\xcc\xd7&\xf5\x88\xc8k\x93\xdd\xfd\xed\xf0E\xed&\xe2\x82t\xea,\xa9\x85a\x01\u007f6|\xc1\xa3\x976\xa8G\xdf<\xcb\xe7\xac8\xd178=\xc7OT\xeb\x14#:\x1e\xb3\xe4\x97o\xbe\xe0\xb9V\x9f\xfc\xf1\xdb\xf5\xc7'\xfc\xfe\xee\x10\x99{;e\\A\x13\"\xd6d:y\xafeN\x1c\x9f\x9dxNF\xf9N\xfb\x87S\xa2\x16o\x14\xbf\x06\x13s]{\x02\xe2\xfb}`)K\xe8ʓ3\xef\x90rB\xa4\xfc\x9ck5o$\xe4\xdb \xb4h\x91\xb1\x85\xcdC\xa9\xaf\x0f\xc48\x9c\xe3\xde\xfa8(^\x83`k\xe7\xa0;\xa5&\x00\x1a[\x9a\x835beT\xd7\xd2\x12\xcb\xe7\xce\xfa\xe9\x8a\x1493\x95f\xe2-\x95|i\xedLg\xe7\xb0\xdbHg\xb3@I\x99G#>F6\xef\"\x9b\xb8\xa3\xa4\x0f\u007f\x1f\xdb\xfdM\xfa\x10)\xac\xea\x1c\xaac!⦗\xba\xee\x14\xba\xa3\xed\t\x80/\x8d\xa59<\xb0\x18\x16K\x12\x13\x80\xac{\x14\xab\xc8ڭ~L\xacʆZL\xf2\x02\x18K\xfa\x97\xc7\xfb\x8f?/\x0e\x96\x01\xac3\x96\\\x90\xbdj\xe9\xdb\xf3\xe8\xde*\x80 _:iC\xb4\xf753LT ؕ\xe4!4\xd4\vE\"\xcb\x00\xa6\x82\xd0H\x0f\x8e\xac#O:9\xf7\x8010\x11j0\xcb\u007fR\x19\xa6\xb0 \xc7l\xc07\xa6S\x82#`E.\x80\xa3\xd2\xd4Z\xfe{\xcb\xdbC0\xf1R\x85\x81\xb2\x85w\x9fԁ\x9cF\x05+T\x1d\xdd\x00j\x01-n\xc0\x11\xdf\x02\x9d\xde\xe3\x17I\xfc\x14~7\x8e@\xea\xca̡\t\xc1\xfa\xf9lV\xcb\xd0Griڶ\xd32lf1(\xe5\xb2\v\xc6\xf9\x99\xa0\x15\xa9\x99\x97u\x81\xaeld\xa02t\x8efhe\x11E\xd71\x9a\xa7\xad\xf8\xce\xe5\xd8\xf7\xd7\a\xb2\x86\r\xfb\xd6\a'u\xbd\xb7\x11\x03\xed\x8c\a8\xd4@z\xc0|4i\xb134/\xb1u>\xfce\xf1\x04\xfd\xd5\xd1\x19C\xebG\xbb\xef\x0e\xfa\x9d\v\xd8`RW\xe4\x92\x13+g\xdaȓ\xb4\xb0F\xea\x10\xff(\x95$=4\xbf\uf5ad\f\xec\xf7\u007fu\xe4\x03\xfbj\n\xb71\xbdaI\xd0Y\x81\x81\xc4\x14\xee5\xdcbK\xea\x16=}u\a\xb0\xa5}\xc1\x86}\x99\v\xf6+Ӑ8Ymo\xa3/\x1f'\xfc5\xa8\t\vK%{\x8f\r\xc8'e%˘\x1aP\x19\a8$\x9f\x1e0\x1eO\\\xfeR\xc5X\x04㰦\xdfLb9$\x1aH\xf6n\xecL/\x1bו\x94Ĕ\x99\x83O\x94GL\x01T\u007fxݐ\xa3xƑ\x0f\xb2\xe4\xe02^\x06\xe36̘9\x90\x98\x1eq8\xe1\x06\xfe\xb4\x11tA\x8f\a#hLl>\n\xa1\xc1\x14\xad\x8f&f\x8d\xeb\xb4>\xbe\x85?\xa3_%\x985\xe2\x82\\\xf9F\x04G\x159Ҝ\x85\xa9pY\x13\xcb[@\xa9\xfblM\x85\x1f\x82\x19\x91l\x99\\@\x02\x86\x01\x01g\x83\x02\xceT\xf5Q\x89\u007fy\xbc\xef+yo\xc4,{8\xbe\xf7\x82}\xf8\xab$)\xf1\x88\xa1y\xc1\xdd\xd7\xf7U\xba,ִ`\x00\xc1J*\xe9\xa0I\x80\xd4>\x10\n0\xd5(G\x06\x12\xc0\x89\xef(\x9f\xb8I\x15,\x97\xca]ka\xdb\x03r\xed\x94\x02\xfe\xb6x\xff0\xfb\xeb\x98\xe9\xb7Z\x00\x96%yf\x84\x81Z\xd2\xe1\x06|W6\x80\x9eՐ\x8eĂw\xa6-jY\x91\x0f\xd3|\a9\xff\xe9\xa7\xcf\xe3\xd6\x03\xf8\xd58\xa0/\xd8ZE7 \x93ŷe\xb9\x0f\x1a\xe9\x939\xb6\x1ca-C#\x87\xcdtk\x01\x0e\xaf\xac\xf6:\xaa\x1b\xf0\x99\xc0du;\x02%\x9fi\x0eW\\~\xf6\xc4\xfc\x0f\xe7\xce\u007f\xafNp\xfdCJ\xed+&\xbaJ\xc2m\xfb\xf0~\xd2\xed\x84L\x99\xe7d]\x93\x8b\xc0e\xec\x8bM\x85K\xf5\xf7`\x1c[@\x9b=\x16\x911{/\x15J\x12GB\u007f\xfa\xe9\xf3I\x89\x0f\xed\x05R\v\xfa\x02?\x81\xd4\xc96ֈ\xef\xa7\xf0\x14\xa3c\xa3\x03~\xe1\x9b\xca\xc6x:eY\xa3ՆunpE\xe0MK\xb0&\xa5\x8a\x84\x83\x04\xacq\xc3V\xe8\x1d\xc7\xf1\x86`х\xb3\xd1ڣ\x9f\xa7\xf7w\xef\xe7I2\x0e\xa8:Vb\ue695d4\xc30&\xf5\xe2\x18\x8dGͼ\xff|\x97\xc2'\x18(\x1b\xd45%}\t\xaa\x8e\xbb\xe3\xf4\xfa-y|\fI\xfao\x04\x9a\f\v\xc7\xff\xad\xb9\xbfP\xb9\x88\xa0_\xa0\xdc\xc3^\x94\x9fU\x8eg\x15\xa7)P\xd4O\x98ҳj%\xd9\xe0gfEn%i=[\x1b\xf7,u]ph\x16)\x06\xfc,\x8e\x1b\xb3\xef\xe2?o\xd6%\x0e\n/U(\x12\u007f\v\xad\xf8\x1e?{\x93R=\x86}y\x1f\xbb^dd5<\xcbi\xb1nd\xd9\xf4\xc3I\xae\xb1'\x92I2\x12\x16\xa94\xa3\xde|\xf5Pf\x83v\x8e%\xda\x14y\x00.P\v\xfe\xbf\x97>\xf0\xfa\x9b,\xd8\xc9\x17\xa5\xef?\xee\xef\xbeM\x80w\xf2M\xb9z\x02\x80\xa7\x18\xb1\xe6^\xb0)+I\xee\x020\xfbp@\xdcC\xc7\x11ĺ\xa5y\x152\fX\x8f@1\x14\">{\xa0z<\v\xd8\xceZ\xe0@\x8d'\xac=\xa0#@hѲ\xe7\x9eiS\xa4\x16oQr\u007f\xe6\x16\x9c1ϒ\x00\xadUr\xb4\x15\xe7F\x9eAh\xc6\xfb\xea\x87\xc4\xe1\x82\xfdӀ3\x06ٳ\x00\t\xdfla{0\xb0\x1cK\xd13\xa0\xf8\xa4\x15y.e\xb4v(b1>@\rhx\xa0\x18,Y#\x06+\x87\x918\xd8L\xfa\xbdh\xaa\f\x18:\xff\x8a\xb92\xd2\xf76MU$d.\x11B\xbfu\xb2,\r\xa3\xd3ç\xb5\xf3\xee\xbd=>\x11\x1fq\x9cH\xc2\x05\xd9r\xcc\xe6([\xa3\xef\xef\x18\x1b\ra\x8f]:\x19\xeb6s#\x11\xa1##\xdb\n\xa5\"\x01\xfd\xdb\xde\xf0\xcc\b\xd7}.K\xaa\xb8\xc8uV\x19\x14\xfd@\x96\xc5\xdb\xc23\x9e\xd7\xe3\xebȵ?ó\xf3$\xe2$?b\x84c\xc8V\x19\xd7b\x98\x83\xc0@\xc5(S\xdd)\x85KEs\b\xae;\xde>S,Z\xf2\x1e\xebK\xa9\xf8{\xa2Jsj>\x02\xb84]\xd8\x0e\xaa\aE\xe1\xda\xe7\x98zݬ<:\x02\x1e\x8632D\xf7\x19\xaa*\x15\xcf\xec\x17\x82݃p\x94jI\xe3\xad\xee-5\x01\xc06\xe8/\x99\xea\x91i\xc6\x12l[\xbd\xcef\x18\u007f\xa4\xbb\xf6\xf8\x96\x02\x1eh=\xb2z\xaf\x1f\x9d\xa9\x1d\xf9\xe3\xc0)\xfa\xf8\x1a\xa9\xe6\x05\xfc\x1a\xb3\xe1U\xfa\xe7\x8b.\x99 \x93AcT\x9f\xcc&\xa0\x02ݵKrl\x87\xe5&\x90?,\xe7c\xaf\x12q\x9aٙq\xef|\xef\xbf\xc4)\x0fh%\xea\xf8z\xc8\xd9\x15\f\b\xe9\xad\xc2\xcd\b\xe3^\x91\x88X8\xb9\xb8\x04\xec\xe2\xb9OjK.n\xbd\xf65%\xcatg\xf4\t|\xdd\xe7\xb3\xd4\xe1O\u007f<\x83o\xa4\x0eT\x0f\x9aC\xdegs\xbe\xe3[\xbe\xce\rgZ\xb7\xd7h}c\xc2\xfd݅(Xl\t\xfbl\xd8\x01\xa5X\xfb\xe2\xdbf&ʡ0\xe6\xaammyU\xaa\xfa\x80.\xbc\xb4\x15-\x0e\x88/t\xa1\xc8y\xbc\a-Ȣ\xe3L\x8f/\xe1\xb7\xc3ߚn\xc0\xcb\xf8\xbc\xc7x+\x01\xb04|{nN\f,\x8d\xa3\x91\x92\t\xc7m堉\x1c\x8a\xff-\xfb\xc7h\x9c\x1c-F\xc9\xc5\x1e\xef\xfcD\x9cWv\x18\x06K\x9e\x0eH<\f\u007fO\xbbJ\xaf7\xfd\x0fd\xf1\xcf\xd2\xe8\x04\x95\xfd\x1c>}\x9e@~6\xfe\xd8\xff\xeeŋ\xff\v\x00\x00\xff\xff~\xe4\xff\xab\x84\x1c\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4YKo\xe3\xc8\x11\xbe\xebW\x14\xbc\ag\x81!\xb5;\x1b\x04\x81n\xbbv6P\xb2\xeb1F\xce\\\x06sh\xb1\x8bb\xc5d7\xd3]\x94\xac\x04\xf9\xefAu\xb3\xf5\xa4\x1e60\x13^l\xf5\xa3\xfa\xab\xaf\x1e]E\x8e\xb2,\x1b\xa9\x96>\xa1\xf3d\xcd\x04TK\xf8\xc2h\xe4\x97ϟ\xff\xecs\xb2\xe3口g2z\x02w\x9dg\xdb|Do;W\xe0=\x96d\x88ɚQ\x83\xac\xb4b5\x19\x01(c,+\x19\xf6\xf2\x13\xa0\xb0\x86\x9d\xadkt\xd9\x02M\xfe\xdc\xcdq\xdeQ\xad\xd1\x05\xe1\xe9\xe8\xe5\x0f\xf9O\xf9\x0f#\x80\xc2a\xd8\xfeD\rzVM;\x01\xd3\xd5\xf5\b\xc0\xa8\x06'\xd0Z\xbd\xb4uנC\xcf֡ϗX\xa3\xb39ّo\xb1\x90S\x17\xcev\xed\x04\xb6\x13qs\x8f(j\xf3h\xf5\xa7 \xe7c\x94\x13\xa6j\xf2\xfc\xf7\xc1\xe9\xdf\xc8sX\xd2֝S\xf5\x00\x8e0\xeb\xc9,\xbaZ\xb9\xe3\xf9\x11\x80/l\x8b\x13x\x10(\xad*P\x8f\x00z\x02\x02\xb4\xacWq\xf9c\x94UTب\x88\x19\xc0\xb6h~~\x9c~\xfai\xb67\f\xd0:ۢcJ\xea\xc5gǬ;\xa3\x00\x1a}\xe1\xa8\xe5@\xfa\xad\b\x8c\xab@\x8b=\xd1\x03W\x98@\xa1\xee1\x80-\x81+\xf2\xe0\xb0u\xe8\xd1D\v\xef\t\x06Y\xa4\f\xd8\xf9?\xb1\xe0\x1cf\xe8D\f\xf8\xcav\xb5\x167X\xa2cpX\u0605\xa1\u007fod{`\x1b\x0e\xad\x15c\xcf\xf1\xf6!\xc3茪a\xa9\xea\x0e߁2\x1a\x1a\xb5\x06\x87r\ntfG^X\xe2s\xf8\xdd:\x042\xa5\x9d@\xc5\xdc\xfa\xc9x\xbc N\xee\\ئ\xe9\f\xf1z\x1c<\x93\xe6\x1d[\xe7\xc7\x1a\x97X\x8f=-2劊\x18\v\xee\x1c\x8eUKY\x80n\x82K\xe7\x8d\xfe\xce\xf5\x01\xe0o\xf7\xb0\xf2Zl\xebّY\xecL\x04g;c\x01\xf16 \x0f\xaa\xdf\x1a\xb5\xd8\x12-C\xc2\xceǿ̞ \x1d\x1d\x8cq\xc8~\xe0}\xbb\xd1oM \x84\x91)\xd1E#\x96\xce6A&\x1a\xddZ2\x1c~\x145\xa19\xa4\xdfw\xf3\x86X\xec\xfe\xaf\x0e=\x8b\xadr\xb8\v1\x0es\x84\xaeՊQ\xe705p\xa7\x1a\xac\xef\x94ǯn\x00a\xdagB\xecu&\xd8MO\x87\x8b#k;\x13)\x85\x9c\xb0\xd7aZ\x98\xb5X\x88\xf9\x84A\xd9J%\x15!6\xa0\xb4\x0e\xd4\xd1\xfa|O\xf4p\xe8\xca3W\xc5s\xd7\xce\xd8:\xb5\xc0\xdfl\x94y\xb8\xe8\x00\xdb/C{\x128\xc9,1\x8c\xb1\x17\x0e>\xae<\x12\nP\xa7ͫ\n\x1d\x86=\x92Ũ\x10\xf7\xb2\x9eغ\xb5\b\x0e*\xe9\xfcH\xc2\tC\x04\x95\xad\xbe\xa0ƣ\xed\x03\xc2a\x89\x0e\x8d\xb8{\xcc\x10\xad\ry\x84\x15\x99\x14\x161\xc5\x02\xdb\x01-\xe6\x11\xf50\xc4\xd3\xd4Ù\xec9\b\xf8\xe7\xc7iʘ\x89\xe1\x1e:\x1f\x9f{\x81\x1eyJ\xc2Z?*\xae\xae8\xfbvZ\xc6\xc3B\xee`\v\nZ\xc2\x02\xf7\x921\x90\xf1\x8cJ\x83-\a%ʭ\r\x12`\x0e\xfb\x1d\xefb\xa6\xe8S\xd26\x85\v\xf5\xa0$G\x91\x86\xbf\xcd><\x8c\xff:\xc4\xfcF\vPE\x81^\x04)\xc6\x06\r\xbf\x03\xdf\x15\x15(/j\x90C=\x93\x99\xbcQ\x86J\xf4\x9c\xf7g\xa0\xf3\x9f\xdf\u007f\x19f\x0f\xe0W\xeb\x00_T\xd3\xd6\xf8\x0e(2\xbeI\u007f\xc9g\xc8G:6\x12aE\\\xd1\u1975a@\xbc\xabW{\x15\xd4e\xf5\x8c`{u;\x84\x9a\x9eq\x027\x12\xe5;0\xff#\x81\xf5ߛ\x13R\xff\x10\x03\xe8F\x16\xddDp\x9b\xfbn7\"\xb7 \xb9R\f\xech\xb1@\x17\n\x84\xa1'$oI\x89߃u\u0080\xb1;\"\x82`\xb1^\xccG\xa8\x8f@\u007f~\xff\xe5$\xe2}\xbe\x80\x8c\xc6\x17x\x0fd\"7\xad\xd5\xdf\xe7\xf0\x14\xbccmX\xbd\xc8IEe=\x9eb֚z-:Wj\x89\xe0m\x83\xb0º\xceb\xbd\xa1a\xa5\xd6\xc2B2\x9c\xf8\x9b\x82V9>뭩\xcax\xfap\xffa\x12\x91\x89C-B\xbe\x93۩$\xa9\x1a\xa4\\\x88w^\xf0ƣK3=\xbe\x8b\xee\xc3\x16\x8aJ\x99\x05F}\x11\xcaNn\xa1\xfc\xf6-q||\xf5\xa7g\xa0\x048L\x1c\xff\xb7K\xf4J\xe5B\xa5z\x85r\x0f;^~V9i\f\x9cAƠ\x9f\xb6\x85\x17\xd5\nlُ\xed\x12ݒp5^Y\xf7Lf\x91\x89kf\xd1\a\xfc8\x94\xf6\xe3\xef\u009f7\xeb\x12\n\xf2k\x15\n\x8b\xbf\x85Vr\x8e\x1f\xbfI\xa9T+^\u007f\x8f\xdd\xce\xfa\x02\xe6p\xaf\x84Ū\xa2\xa2JM@\x9fcO\x04\x13IũcjVf\xfd\xd5]Y\b\xed\x9c Zg}\xb7\x99)\xa3\xe5\u007fO\x9ee\xfcM\fvtU\xf8\xfecz\xffm\x1c\xbc\xa37\xc5\xea\x89B7\xfaHk\xa7Z\xa8,\t݅\xba\xec\xe3\xde\xe2TW\x0eԅ\x9b5\xaf*\f\xbdQ\xad\xaf,O\xef/\xe0\x98m\x16&\f[\x03\xf4\xe5`\x92%\x8e{\xb6\n<\x83'\x8a\xba\x80%\xd6\xf6C5v\x8f$\xd6\x1caD\xeaڀg8X_\x8bPZ2)\xa0\xf6\x11fÝ\xc3\xc1\x9a\xd6ꃑ}O8\x98ܚ\xe6`\"*yU[Ŋ;\xff\x9a\xc6*lH\xcc\xc6\xf8\xe6^L(n\xdf\xdcZ\x15V\n\xc7\xfdWL\xe7\xad|w\xbc#\xbc\xc7p:\xa2cj0\xf4+\x01\a\xac\x94O\x87\fY\x14v\xe4ŭ!\xa7\x8a8ԡ\xac\x93\xaa\xb3TT\xa3\x86\xcdK.x\x92\x0e34\xf4\xb7CUL\x12\xd4yԡ\xf7\x1c\x00}\xbc\xaf\xb4\xaeQ<\x01i\xe33\x11q\xb4\xc2tu\xad\xe65N\x80]w<}&\x80\x1a\xf4^-.E\xd0\xefqU\xec\xf8\xfa-\xa0\xe6\xb6\xe3M\xcbׇROŭ\xef\xbd\xe0umg\xa5\xfc%(\x8f\xb2f\xc8\xe36A}\xde\xe5\xe4A\xd35\xc7\xc7d\xf0\x80\xab\x81ѩytv\xe1\xd0\x1f[&K\x06\x1ch\x022\xf85xǫ\b\xe8\x0f\xba\xc4A\xbf\f*['ﶬj0]3G'D\xcc\u05cc>1\x92R\xc3P\x0f\x1dj\xef-\x93[\t)\xdbEQ}7Q(\x13^)\x89\xff\xb2\x05M\xbe\xad\xd5z@n\xd2$\\\xaf\xe2\xbe\x12G[\x8fIQ(\xe1\x1f\xe6^\xdb\xfb\aP\xf7֜\xa8\x06SȐ\xe1?\xfd\xf1\xccmL\x86qq\x90J\xfby!\xf4\x179\xe5\xeb\x9cp\xe6\xc2\xf7\xac\x1c_\x9b\xf6f{\x8b/e\xbc z8\xdf\xed\xa6\xae\xe3D\xb5\u007f̷\xccQ\x83D\x1d\r\x06\xe4zGv\xffެ\x1f\xd9\xdel\xaa\x90b\x0e\xf5\xc3ᧆ\x9b\x9b\xbd/\a\xe1ga\x8d\xa6\xf8\x99\x04>\u007f\x19A\xff.\xedS\xfa\x1c \x83\xff\v\x00\x00\xff\xff\xb9\xf7H\xe3\xa0\x19\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4W\xc1n\xe36\x10\xbd\xfb+\x06\xdb\xc3^*y\x17{)tk\xd3\x16\b\x9a\x04\vg\x91K\xd1\x03E\x8d\xeci(\x92%\x87Nݯ/\x86\x92bYV\xe2\xcd\x02\xab\x9bə\xc77\xf3f\x86\xf4\xaa(\x8a\x95\xf2\xf4\x80!\x92\xb3\x15(O\xf8/\xa3\x95_\xb1|\xfc)\x96\xe4\xd6\xfb\x8f\xabG\xb2M\x05W)\xb2\xeb6\x18]\n\x1a\u007fŖ,19\xbb\xea\x90U\xa3XU+\x00e\xadc%\xcbQ~\x02hg98c0\x14[\xb4\xe5c\xaa\xb1Nd\x1a\f\x19|\xb0e\xf1\xff\x00\x00\x00\xff\xff\x1b\xa0}D\xf4\f\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec}}s#\xb7\x91\xf7\xff\xfe\x14\xa8\x8d\xeb\x91t\x11\xa9\xdd\xe4.u\xb7\x95z\\ʮ\xec\xa8\xecղV\xca\xfaq9>\x1f8\xd3$q\x9a\x01&\x00\x86\x12s\xbe\xef\xfe\x14\x1a\xc0\xbc\x90\xc3\x17`\xa8}\xb1\aWu\xf1R3=@\xa3\xd1\xe8n\xfc\xbaA\v\xf6\x1e\xa4b\x82\xbf$\xb4`𨁛\u007f\xa9\xf1\xfd\xbf\xab1\x13\x17\xcb\x17_\xdc3\x9e\xbe$\xafJ\xa5E\xfe\x0e\x94(e\x02\xafa\xc68\xd3L\xf0/r\xd04\xa5\x9a\xbe\xfc\x82\x10ʹ\xd0\xd4\xfc\xac\xcc?\tI\x04\xd7Rd\x19\xc8\xd1\x1c\xf8\xf8\xbe\x9c´dY\n\x12\x89\xfbO/\x9f\x8f\xff8~\xfe\x05!\x89\x04|\xfd\x8e\xe5\xa04͋\x97\x84\x97Y\xf6\x05!\x9c\xe6\xf0\x92HPZHP\xe3%d Ř\x89/T\x01\x89\xf9\xd8\\\x8a\xb2xI\xea?\xd8w\\G\xec \xde\xd9\xd7\xf1\x97\x8c)\xfdm\xf3\xd7\xef\x98\xd2\xf8\x97\"+%\xcd\xea\x8fᏊ\xf1y\x99QY\xfd\xfc\x05!*\x11\x05\xbc$7\xe63\x05M \xfd\x82\x107&\xfc\xec\xc8\xf5z\xf9\u0092H\x16\x90S\xdb\x1fBD\x01\xfcrr\xfd\xfe\x8f\xb7\xad\x9f\tIA%\x92\x15\x1a9\xe3\xfaF\x98\"\x94\xbcDZ\x99\x0e\xe0$\x10\xbd\xa0\x9aH($(\xe0Z\x11\xbd\x00B\x8b\"c\t2\xb1\xa2H\x88\x98Uo)2\x93\"\xaf\xa9Mir_\x16D\vB\x89\xa6r\x0e\x9a|[NArРH\x92\x95J\x83\x1cW\xb4\n)\n\x90\x9ay\xc6\xda\u0590\xa3Ưkc91õO\x91\xd4\b\x10\xd8.;\x96A\xea8dz\xab\x17L\xd5C[\x1f\x8e\x1b\x12\xe5DL\xff\x1b\x12=&\xb7 \r\x19\xa2\x16\xa2\xccR#wK\x90\x869\x89\x98s\xf6ϊ\xb62\x035\x1fͨ\x067\xdfuc\\\x83\xe44#K\x9a\x95pN(OINWD\x82\xf9\n)y\x83\x1e>\xa2\xc6\xe4\rN\x0f\x9f\x89\x97d\xa1u\xa1^^\\̙\xf6\xeb'\x11y^r\xa6W\x17\xb8\x14ش\xd4B\xaa\x8b\x14\x96\x90](6\x1fQ\x99,\x98\x86D\x97\x12.h\xc1F\xd8u\x8ekh\x9c\xa7\xbf\xab\xa6\xed\xa4\xd5W\xbd2\x92\xa7\xb4d|\xde\xf8\x03\x8a\xf9\x8e\x190\x02oeɾjGQ3\xda\xfcd\xb8\xf3\xee\xea\xf6\xae)gL\xads\x1f\xf9\xde\x10\xbez\n\f\xc3\x18\x9f\x81\xb4\x93\x88\xd2fh\x02O\v\xc1\xb8\xc6\u007f$\x19\x03\xbe\xce~UNs\xa6ͼ\xff\xa3\x04e\x04Z\x8c\xc9+T*d\n\xa4,R\xaa!\x1d\x93kN^\xd1\x1c\xb2WT\xc1\x93O\x80\xe1\xb4\x1a\x19\xc6\x1e6\x05M}\xb8\xfe\xb0\xe5Z\xe3\x0f^ym\x99/\xb7\xfao\vHZ+Ƽ\xc6fn\x99\x93\x99\x90-\xe5`^\x19\xb7\x88v/Z\xd3\xec\xea7\x1al\xfd/k]\xf9K\xf5\xa0\x91\x1fӉ\x92\xb3\u007f\x94\x80*ήX\xd8P)\x1b$\x89\xef\x1f\x8a\xc5x\xe3\xef[xj\x1a<&Y\x99BZiۍ\xb1\xac\xf5\xf8j\xe3\x05\u070e(\xe3F\xfe\x8d\xfa7\xdd\xe6\xf5_\x8d:\xed\xe81\x95@\x8c\x042n\xe9\x11\xc6q\xb0\x9d\x9c6\x8di\xc8;:\xb7st\x04\xf79:\xcd\xe0%Ѳ\x84-\x9c\xa1R\xd2\xd5\x16\xc6\xf8\xbd\xf9P\xbeT\xcf;\x85\x90\xb1\x04\x9a\x1b\x85e\x8d\xddd\xa8\xdc\xec\x11\xf9\xa4\xb9\xb2\x10\xe2~\x1f'\xfej\x9e\xa9u\x18I\xd0\xc6!SX\xd0%\x13ҍ\xddm)S \xf0\bI\xa9q\x9bߠ[\xa2:\x15\x92\x14B\xe9\xed\\ؾ\x12\x89]\x1cۦp'\v7F\xe6\x14\x87\x9fb3Ж\x12\x11\x1cL_s\xb3\x12\xebg\xa5(\xed\xb3\xebھ\xc1\xf0n\x8e\x90)U\x90\x12\xe1d\xa0\xcc@\xb9o\xa5\xa8\x9e\xeaUv\xbe\x95t5x\xbb\xefft\n\x19Q\x90A\xa2\x85\xdc\xe4\xe4!\xfc\xb4\xed\x10ͱ\x85\x8f\x1d:\xc4\xe9^\xa7\x89\xeb\x81\xed I\x8c\xd1\xf1\xb0`\xc9\xc2n\x89F6\x91\x0eI\x05(\\F\xc6l[m\x1b$\xd97\xf7\xee#\xbb\x16R\xdd\xf6,\xa9uz]\x8b\xabn\a(\x9f\xba\xedQCm\xce\xd6\xf6j\xe7>R\xb7_'c\xbd^\x8d\x10\xda\xeb\x8dW\x8f+\xb4\xe8b\x18\xd3\xf7zF /\xf4\xea\x9c0\xed\u007f\xddG\x91fY\xe3\xfb\x9f\xf1ĄK\xfc\xf5\xfa\x9bG\x95\xf8\x9d\xb3\xb2\x8f\xa2\x99\x95\xea\xf3\x9f\xe1\xa4\xe0fq\xeb\xf6\x8a\x83'\xe4\xbb\xe6[\xe7\x84ͪ\tI\xcfɌe\x1a\xe4\xda\xcc\xf4Z/\xc7`\xc6!\xfb\x9di9\xd5\xc9\xe2\xea\xd1X6\xaa\x0e\xc7\x1cȗ\xf5\x97\xad\x81\xe8-\xe6\xf6Ƽ\x87.Ag\x8eIȭ\x93x\x87ܬ\u007fA\xeb\xfa\xf2\xe65\xa4\xbb\xd8C\x0e\x93\xbc\x8d\x81\\\xaeu\xb6\xf9ig\xf5\x1e:\fg\xfaT\x1e\x84u\xff\xcf\t%\xf7\xb0\xb2\x16\v\xe5\xc4L\x0e5\x1f\xda\xe2Kl2\a\xe3\x10(d\xf7\xb0B2.\xb0\xb0\xf7\xedCE\xc1\xb6{X\x1d\xf2\xd8\x1a\x03M\x9f\x9c\xbbg9i~@F\xa0\x1bz8\xf3\b\x06\x89\xbc.\xda?8r\xb8\"\xf1\xcd\xf3>b\x98մ5\x82i8\xb1'\xcaN\x91Y\x05\vV\x1c8P\x8c\xa5)\xc0\xd5\xe2\xc3D\xefi\xc6\xd2\xeaCV\xee\xaf\xf9vk\xb8\xddn\x84\xbe\xe6\xe7\xe4\xea\x91)\xd35\x9e\x92\xd7\x02ԍ\xd0\xf8˓\xb0\xd3v<\x82\x99\xf6E\\^ܪmÇf\xbc\xe9\x00\xe1\xb6\xedچ\x15\xaa\xe9a\x8a\\s\xe3\xb88~`\xf4\xd0~n\xf7\xfe\xd0ny\xa90\xa0\xc4\x05\x1f\xe1V9\xee\xfa\x92e\xf6\x81$\x85l\xcd\xc8fת\x8f\xda\x0f\x1eH\xf6\xce\xec$\xf6}\x1b\x0f\xcdh\x02\xa9\xf761\x8aG5\xccYBr\x90\xf3]\x1bG\xb3\x15F\xbf\x1fօ\x03\xb5\xaem\x81\x12v\xd8\xd6\xee\x9bS\xdd\xe9\xfeΌ\xcc\xca=\xe0)?\xd9{\x1f\xdd\x12\xbc\xdb\xfe\xe8\xfe\x11\xe1\x16\x8b\xf6\xc7^\xee\xd24œ\x16\x9aM\x024~\xc0\\l\xee\xfd\xb6cv\x87\xccia\xd6\xef\xff\x98m\x0e\x05\xfa\u007fIA\x99<`\r_\xe2\xa1I\x06\xadw]\x98\xa8\xf9\x19\xf3\x05\xa6\x88\x99\xdf%\xcd6\xc3\xc2\x1d\x83\x13F\xb7@f7r1۰X\xce\xc9\xc3B(\xbb\xa7\xce\x18d]!\x9bvc\x8a<\xbb\x87ճ\xf3\r=\xf0\xec\x9a?\xb3\x1b|\xb0\xba\xa9\xac\x05\xc1\xb3\x15y\x86\xef>\xebc\x04\x1d(\x89\a=\xc6;\x83\xbeuk\x89E3\xf0[G|\x9d\x99\xbb\xab\xd7\a\xc9a!\x94\xfekw\xc0nK\u007f&\xfe\x8d\xb6m\xda\x11\xf7\xdak\xb3\xbb\x18V\xa5T\x8d%7\xd3 ]\x10\xcf*Z\xef\x01\xf4\xf4\x8d\xf6\x05\xe9\xaa\x00\x1d\xad\x02\xe5\x86\xc1{\xa4\xc2\x1e\x00\x1c\xd2\xc5\x10\xab\xd1\xf0%\xd0\u07bezl\xc4\x18\xcd\n5\xffn\x0e\xe4\xd8Vm\"\xf2\x9c\xae\x1fy\x1d\xd4\xd5W\xf6M/ӎ\x90\x9d}9/q]\x1en\xeey\x19\xc2î\a\xa6\x17\x8c\x13\xea\x97?H'P\x94\x14b\xbf&\xb2mA\x15\x99\x02pϾ\xbd\xaa\xc1\xb6\xa7ݯsƯ\xf1\x03\xe4\xc5\xd1\xf7wR\xb3+j:=\xab\xab\t\xad~\xc0\x1d\xe7P\xd3H\xa4\xe4a\x01\x12ZR\xb1\x19\xf06\x16\xe3\x81$\xb9\xd0\u0378\x82\xa1[\x88\xf4D\x91\x19\x93J7;z\xa8\xc0\x95\xeaPq\b\x9ca3\xba;\x96\x83(u\xc4\x1c\\\xd5o\xb7N+s\xfa\xc8\xf22'4\x17\xe5\x01\x9b\xbbmf\u007fayu\xa4\xe8f\xe0\x812\x8d\xea\xce\xd0\xc5\b\x8b\x16f\x96\x8a\f\xf4\xa1S<\x85\x99QG\x89\xe0\x8a\xa5 \xfd\x91\xb7\x9dY&\xcc\u009dQ\x96\x95]\xc77]-\xd4M\xe5WRFy\xa9o훍\xa8\xe1B<\xb4\x19t0\v\x16t\t\x84\xcd\b\xd3\x04xb\xe6\x05\xa4U\xd9\xf8\t\xc7\fd\xcd\xc1by\x98\x827\rx\x99\x1fƀ\x11\xael\xc6w\x06Ś\x8f\u007fMY\xf6\x14\xd3f$/~i|_\xbf\xfdA\x96F\xa5T\x0e\xdf¦@\xde\x01MW~}P\xad\x8d\xab\x8a2 \x88,yS#>\xc1\xca\b\xf1\xef\\/\x8e\xe9\xb81\xce\x0e\x98صx>\xd3MkǐxRk\xc7|\xa0\xda\xe8bB3\xd7-\x02f\xab\xf4\x863\xf6\xbd\x92\x9a\x00\xcbg\n\xc6A\x85\xd4\x06\xbd\xcc\xf6\xe9\xech\v\xe4\xd9r\f\xde9\xba\x10\xd3e-\xcc\xeb\x1c\xcd\x06\xf8-x\t\xb8\x00\xefJ\x94\xe4\x81r텾2\xe6\nq\xa0ԇΪmT\xce\x03\x9e\xde@\xd6y\x93\xd5\xc3ۀk\xb9B\xb8ա\x9d\xb6\u0378\xa6\xa9H\xee\x8d9\x92\xd39\x9c\x9c(\xf2\xea\xcdk#*\xc6\xea0[F\xc0\x8e`\x1b\xb3'\xb1\x85\x14K\x96\x1a\xd3\xe9=\x95\x8cN3\xe3\x04\xcf@\x02O@\x91/O\xdf_\xbe\xfb\xf9\xe6\xf2\xcd\xd5Y\x10q\xe3:\xc3cA\xb9\x91\xc1R\xf9ݼ\x9a}3\x00\xe0K&\x057\f\n\xe3\xc6\xf5\x8cP\xb2\xf4\xbdM*$\x9aq\xb5\xb2\xa5\xb3\xe6\x82(V#\xf6\x81\x10ƋR\xfb\xe8\xe8\x03\xcb22\r\xa3X\xf2dA\xf9\xdc\xf0\xd5L^\x83\x8fD\xad\xb8\xa6\x8f\xa6\xdfA\x14\x8d\x99\xab\x12Z@\x8a\xd2O(IEi\x18\xf0\xe5\x97\xe7\x84\xc1K\xf2e\xe3#a\f\xbdrt\x1b\x13\x8fc\xe6\xb0\x04i\xedk;\x95\x87\x06\xc6m\x930\xa72\xcd@)\xa3\xcb\x1e\x16\xa0\x17 ]\xb8\xc6M\x1e\x84Dsm\x13\xd2\xc8m'\x02\xb1\xc6\x1c\x06Q\xf4\xf8\xc4\xfb\n`;f\xe2\"\x15\x89\xba\xd0Tݫ\v\xc6\xcdV5J\xa9\xa6\xa3\x862\xbb\xb0\xbb\xcc\xc8\xed{#\uf84e*1\xbf\xf8\x9d,9g|>\xa2\xd5S\x8c\x8f\xe8H- \xcbN\x02z\x19\xa4\x86m\v\xf6#\x9b\xaf\x85\x04\xee\x03\x1d~\xdbښ\xf2\xaaR\x8c\xf6\xcbcr#\xf4.d\xd7\xf6Vm\r\xc8\xe3q\xa7\uef3a\xb9{\xf7\xc3\xe4\xed\xf5\xcd]\xa8\xcal\xaa\xdb\xed*4N\xf9\xb4\xd4m\x87\n\r\x9b\xfd]궭B\x83\xe8nQ\xb7\x1b*4\x8c\xb1\x1d\xeav\x87\n\r\xa2]\xab\u06dd*4\xac\xbf\xeb\xeav\x9b\n\r\xa2\xba\xa9n\xbbUh\xb0\x1c\xac\xa9\xdbM\x15\x1a\xb6\xb5m\xaa\xdbA\x85v\xbc\x16\xa6B\x81/\xa3\xd5\xe7w\xce]h,\xf1j\xce\xc36W-\xf0\x84\x9c\xf1\xb6\xfe\xe8\xdam\x9f\x96\xf3\xed\x10\x16_\xbe\xa7m\x18\x00o\x0e6\x882\xa9\x97\x83G\x18\xe3vQy`a\xb6S\x8cWa۾\x93\x9e\xee\xb6y\xfe\xe3P\xfe\xf1\xfc \r\x9e\x8c\xc9\x1bw\"Nɫ\x9f\xaf__\xdd\xdc]\u007f}}\xf5.\x8c)$~\xed\x10\x0fr\xe8ɚ\x93\x0ew&\x82/\xbbw\xe4\xe0\x8dζB\u0092\x89\xb2\x06c\xf7_\xb8\xb6\xad\xaf[\a\x80Z\x11\x05rɒ\x98\xbevv\xad\x8f\x01a\xdb^3\"~\xf4{\x8c\x89\b\xc2\xdb=\xb8\x86I\x11A\xf7\xd8~\x9cm\xfb\xbc\xb9\x18!8\xa2Ab\xdb.\xb3\xe45\xcch\x99\x05\xba\x85\x8e\xac Ϟ\x8dC\xac\x02\xf7^Oe\xf5\xb5\x14\a\x86Λ\xad\xa5\xb0nmz\x8f\x8f\x15\x1fG\x9d\x9f8Hdk\x03WQ\xc2\xca\x1cj\xce{=A\x88\xa9\xba\xc5\xef\x97\xc4\x1eF\xce\xd8\xfc\r-\xbe\x85\xd5;\x98ŐXg;\xa2%\x1d\xb0\x90\x88C\x8f\x0f\xd7\x1b\xda\x0f\xb6k1k\xb6/_H\b\x96\xb4\xab\xb5xr\xe7p\xafh\r\x1a\xf6\xc4\r\x89\xf4[X\xbe\xc5\xd9I\xbe\xb5\x8d\x82\x86\xc1\x14M\xb1\xf2\xd8\xf5\xa1.P\"x\x02\x85V\x17bi\xf6ax\xb8x\x10\xf2\xde\xf84f\a\x18ٓ\x10u\x81\t\x18\x17\xbf\xc3\xff\xe9ѻ\xbb\xb7\xaf߾$\x97iJ\x04\x06\xdcJ\x05\xb32\xb3\x80\xab\x831\x9e]\xadN'?ǔ\xe6sR\xb2\xf4\xabpe\xeb\xdb\x11dC\x14\x16\x83w$\xf9\xb8\xc53\xdcU\x8f}\xcd7\xa3\xc2+\x8d`v6\xa6\xd5A\x00\xc8\xed\xcdCU\x9d\xd1ؓ\xedS!2\b\fA\xfbv\xf8\xa1`W;\f\b\xda\xd5\x02\x0e\x0e\xbb\x1a\xae\x80\xe3\xec\x1a'\xf5\xb6q\x18\x90\xb1\xbb9\u05ed\x10\xe9K\xa2ʢ\x10R\xab*U}l\x14AX\\\xaan-\"\x98\xd5uN\xfe\xab\xfa\x11\xb3\x06ԏ''\u007f\xfe\xf6\xea\x87\xff{r\xf2\xd3\u007f\xc5~\xa7\xa6٨2r\fª\x80d\xccE\nFe\x9f\xdb\u007f:/\xe62Ah\xc4M\x0f\xf6(Mu\xa9\xc6\v\xa1\xf4\xf5\xe4\xdc\xff\xb3\x10\xe9\xf5\xa4'I\xa4\xa1\"LPr\x14#`[ɏ\x90֒t_\x1a\xa4\xf7\xa6骈\x18y\xff\xda,\x99\tՋ\xc3\xc1U]\xedA2\xad\x81\xa3\xdf\a27\x1eɹю\xb1\u0383oƉX\xbex\xf6Q\x8d\x9e\x99gё\xa6\x11\xb9\xed\xd4M\x1f\x8dE|\x90Ш?\xefqW8\xba\x1eD/'\u05fe\xe4\xccGd|ߝ\xad\x9a\xb6\x8f\xb1\xbfy\xa8\xf1\xd7O\xb2\xcfy\xea\xfd\xb6\xba\x1aMj\xd1\xf7\x87\xe5pno\x19\xc3\xda4\x94\xa7u}\x9aS\xfb\xe38)\xcaXe\xee(\xe4\x90\v\xb9:\xf7\xff\x84b\x019H\x9a\x8d\x94\x16\x92Σ\xb7\x1f\xdfU\xecb\xfd/\xfb\xb9X\xcd\xdf`\xc1fO\xc3\xce\xcc\x1aT%\x90\xa4\x94\xc6\xdb\xc9V\xdeF\x81\xf4\xa3\xedo\x95\xfct\x17\xc79\xb4\xb5\x85\xbc\x86)\xf7\xf35k\xfd\x81a\x9c\xa5\xc8\xca\x1c\xd4y\xe5\xa5\xf4 l\xe8\x01_\x92%\x95\xea\xa3z\\)[2u(P\xb6\xabQ\xbez\x1b\xa9\x9a\b\xeaX;\b\xc65\xcc{\xb8h\xa3c0\xa3\xd3}t\x85\x03zL\xb6(uQ\"l4\xa7\xba:\xdfy,D\\\xe4ηF)\x17o%\xa1\xad\xf3\xe2Y4тj\r\x92\xbf$\xffy\xfa\xf7\xdf\xff2:\xfb\xea\xf4\xf4\xc7\xe7\xa3\xff\xf8\xe9\xf7\xa7\u007f\x1f\xe3\u007f\xfc\xcb\xd9Wg\xbf\xf8\u007f\xfc\xfe\xec\xec\xf4\xf4\xc7o\xdf|s7\xb9\xfa\x89\x9d\xfd\xf2#/\xf3{\xfb\xaf_N\u007f\x84\xab\x9f\x0e$rv\xf6\u0557\xd1]~\x1c\xd5\x11\x9a\x11\xe3z$\xe4\xc8\n\xc1\xde4\xff]\xcd3\xf7X:靷D\xfan\xbc\xa4i\xb1}D\xd5\xd1״\xeaņ\x9e\x96\x95\x82D\x82\xfe\xf4bζ_\xde\f\xb7\xf9+\x95\xc3\xff\xab\tC\xf7w=-\x9bj\xbf\x05\xcb\xdc\x11<\xea\xeeA\x16\x0fɗXA\xc0}\xe1\x1e\"ND|\x1bB\xe5C\xa8|[\xfb\xb5\x87\xcao\xed\xfa\xa9\xe3\xe4\x18\xed\xee\xb76\x878y\xd4n\x17\xfdr\xdchm5\xe6\xa0W\"{\x18\x89\xca\v=\xda\xefD\xe6\xd5%\xc4H!\x8a2\xa3\a\xe7EV\xfd\xd8D\xe1l\xc2\xf4\x82h\xba\xed\xb5F!\xd5\xc8i\vE\b^\x82\x9b\xa81r\x99e\x84q\xbbI\xe2ǂQ\xb1\b\x11\xb2Q\aB-\xf2oi\xd8\xf0\xb0\x00\x1e\x95Ug\x1bSDi*5\xe3\xf31\xf9\xdeв֘â0N\xf22Ӭ\bD7U\x1eVU\x95\x82P\xa5D¨\xf6\xa8\xe3\xd0\r5\xa3J\xfb)A \x8f\xa6\xf7\x88]L \x05\x9e\x00V1*\x03\xf7A?\xe7ӕ\xe1\xe8\x15_V`\xa4҂s!X\xfbt\xf7\xedc\x03G\xcd\xf2uК\x1a?\x1a\xa8\x141\xc6mi\x88Y]D\xaa:\xdf\r\xa3\x17obW\xe8\x97(7dö\xaeϧ+\xcb8\xe6TU\x8a\xfc\x03\xa3\x80\xe2\xcdܭ&nm\xa8F\xda\v\x9f\x9ay\xfb$\xa6\xed1\xcdڞ&m?sv\x97)\xdb\xc3\xe3\xa9W\xd41\xc0\x1a\xfd\f\xd0\x1eF`!a\xc6\x1e{j\x98K^\xcd\x11a)p\xcdf,\x0e**L\x87\n\xe0\x98\xcd\f4Y\xd8\x02\x88\xbc\r\xb2\x89\x91\xe9O\x00\xebn#\a\xc7Q\xe8\xb7kq\x8eA\x9b\x0f\xda|G\x1b\xb4\xf9\xee\xe6\x96\xd3g\xac\xca?\xa0\xa7\x8c\xb9\xb5\xf1\t\xc0\xaf\x1b\x19\xba\xa8\x11\x9e\"\x9b\xbbZ\xafuI\x8d\v\xfcbز\xc4\xf2\x9f\xb8\xf4\x8c\x17Ymr\xb6\x8c\xbax \v6\x0f\x8d\x88e\xb0\x84\xcc\xd9\xf7$\xa7\x9c\xcem\x11E-\xfcQ\x1d9\xb8\xa0\xafmF%I\x96\xb6Jn\xe1P\xcd\xc6i\xd4T&h\x98,\u05f7\x86)\x92\xb1{ \xaf\xa1\xc8\xc4\xca\xd5J\xe4)\xb9\xd5T\x1b\xb5t\v:\f\x00\x17\xa5\x9e\xcfa\x95\\7\xdbZ\xf5A@\xd1@0\xa5\b߶|C\\\xb00\xe6\x8b\x11\xaa\xba\xb2f\xb5\xd1\xc49\x19Xz\xb6}\x9f\nE\xb5\x89\x15\xb1\xe8\xfd\x81E\xc8ڭ/F!\xa2ZM\xb3\xed(X\x1b}\xd6\xe8K\x9af\x8cC\xb3r-\xc3j\x98\xf1G\x98\xcd\x15l\xf5\x91\xf3P\xe3\x0f\x81\x99\xc4\xdb%V\x8d\xaa\x86\xb6\xef}\xc0\xfcR\bMNO.N\xce6\x0e\xb5N\xe2\xa9\xceX\x06vw\xb5%d\x92j\xb2\xa2I*\x96\x17\xd9\n\xe7\xe7$Ż|\\:\xac,\xe3Έ\t*53˾d\xd19Q\x82hI}}\xf9\xf8\xbe\x1aj\x86\xb8\x96\xa5\xb3UNO~99'\xa0\x93X<0!\x0f\x82\x9fh\x14\xa31\xb9\x13\xa4Tuǣi\xaeDI8\xd8r\x85\xf0Xd,a:[\xe16\x1fMS\x94ږ\xddë\xf1\xb0\x14\xd4\xd5#\xd3.O'\x9e\xec\x8c<\xc7\xd5nM\x05B\x8d3\xb4\x84\x8b\x05\xd0L/\xe2\xf3\xfb\x8c\\r\xc1G\xff\x04)\xb0\xd0\x14w\x14c\xa3;\x11gg\xcdv\x04\xc4IL\x18a\xfd\xedH\x10\x831\x10\xbe\x81`\x93\x93l\xdcEyw7\xf9\x06t{\v\x8bb\x87\xe9\x91\xc7\xe7cH\x1b\xe4LȎ\xcbV\xf7\xb7\xbe\xfb\xdfB\xa8(ΐ͛:\x95\xb6\xf7\x0eX'\x85\xc7ŜmӢ\rKv\x88Fr=\x89\ab\xfd Jí)\x9df\xab\xaa~\xa8\x02M\x9e\x99\xae\xc7Þ\x19\xc7\xf9\xfc+\xd0\x14+\xb6r\xa5\x81F\x9aHGXj\x8d\xbe\x1cǨ\xb17\xae.,\xc9\x1e3\xda,&\xe5d\u007f\x8ck*^M\x1a\x13\x15\xefܱ\xea\xd7\xf5\xf1#)\xc9\r]ag\xc1\xfd>\xed\x85M\xa4\xfe\xe2[;DW\xd57\xb2x\x87o\x8cc7qQ\xf4\xe8]\u007f\x940\xe9\x8dV%]'`\x96W\xbdh\xba\xdc\xcbpX\xdaz;J~It1\xacf{J6a\xf7>>\x9f\xfaA-I\x1c\x10\xb1\xfdz?N\xf4\xccP \xbd\xed-L\xe6\x89N7\xdeL6ւ\xd0$\x01\xd5\xc3Զ\x1b\a*,\x05r\x19\np\xac[o\x11+Dx\xfcҷ^\to\xc7Iw;J\xb2[G\xe5AIx\x99O{h\x92\xaa\x00\x86Ե\xc0\xb8\x89\xef\x11L\xa9\xcaV\xde`\xf7\xfc\x11n\x1f{\x0fM\x18\xca\xe7@^\x98\x9e\xfe\xe9\xdf\xfe\xed\x8f\xff6F6DS\xf5\a˔\x93\xeb˛˟o߿ªo\xb1R\xfe$\x99mX\xb6!z\xffi\x9f\xcc#)ýRa\xa1\xb3>3l|\r\x17\xff\xb6\xc1eկr\x9cmZ\xa0\xba\xf9Hz\xa6\xcf&6\xc2E\xf4\xa1\xfdl\x9d\x14\xb7\"\xb9?\x82\xa7}r\xf7jbI\xd5\xcev\xd4,P\xeeČ/E\xb6\xb47\xc5ݽ\x9a \x83\xe2fּ\x8d\xe7\x03\x18\xea[\x99>\xfaLx\v͉\xa2\xca\xf2\xc2ݕH\x89\x04\x9a1\xa5Y\x82ߪ\x8e)\"\xfd{q\x1f\x83\xe2\xf9d\xe2\n'o=\x1c\bC\fѫy-4\xd1\n1\xf4Q\x11\x8d\xd0DlJ\xd7`\x91\x1c\xdd\"\xb1[\xbd\x90\xfd\xec\xf8\xc1\"\xf9\xb4-\x92\xcfm\x8f\x8c~\xb5\x90p\xabE\xd1\x135a\x89\x1c\t3\xe1\xef \xdb\x06j iĔ\xda[\x83/'\xd7Ut\\\xb4\x80\b\b^\t\xa6\xaa\xcad\xe1\xcff8(u\x81\xf0\x88\xb2\xb0\x91/\u007f\x95`\xf8\x89U!\x01\xef_\x13\xfc\xbc\xaaH\x80\xec\x00n\u007f\x04\x9d\x84\xaf\x16\x8c\xc98\xec\x88;O\xf4\xd3\xd5\x17\x86\x91H\xaa\x16\x80ŕ\xe1\x91i\xe5\xef9\xa6Jp{\x84릏\x89\xf0\x03L\xa6HA\x95\xb2\aw\xba\x1e\x84\xfd\xc8D\xa4'\x11\xa7\xb7\x8d\x0e\x91\xb9\xa4\t\x90\x02$\x13)\xc1\xaa\u007f\xa9x\b\xef\xe7\x14\xe6\x8c+/\xbf\xa6\xa3~a\x18[\t\xa2N\x84\xeb;Gߵjb\xe3EեND\x84\x1ev\xaf;.\xae\x03\x88\x82S'\xb1\x9bf\xf9\x944\xcbV\xf5B\xf5\x99\x9e\xfa\xf8\x93\xb4\x89$\x8aeB=\xeeu$Q\xf8i`\vyd\x96B\x8dJ\xea#\xfe-\xe9dʬ\xaad\xd1\xe3\"*\x12\x0e_\x1e\xa0M{\xdb\x00m:\xb8\rЦ\x01\xda4@\x9b\x06h\xd3\x00m:\x88\xc4\x00m\xaaz4@\x9bv\xb4\x01\xda4@\x9b\xf6\xb7\x01\xda4@\x9b\xea6@\x9b\x0ej\x03\xb4\xe9\x806@\x9b\x06h\xd3\xd66\x1c$\x0eЦ\xdf\xe2A\xe2\x00m\n{}\x806\x05\xb4\x01\xda4@\x9b|\x1b\xa0M\x81m\xb0H\x06h\xd3o\xd1\"\xf9\xdc\xf6\xc8^\xd5\xc5\x02_\xf38\x9e\x89\x14\xd3\x1ee\xc6&xV\xcf\x12\a\x03\x12\xb3\xe8\xda:\xb6;c\xf2\xaa\x05\xcfp\x17\xda\xdb*-A\x14\x1dЧ\x86'\xf5\xad\xd7\x13\\\x93\xc9\x17\x05S\x17\x85\xb0\xff\xaf\xc6\x144\xc0\x046\xbe\x16\xb29\xc4n\xbe1(\x82}\b\x82(]\xb7\x1b=\x80H\x80`\x9a\xc7D\x0e\xf4\xb1nz \x06v\xa0\x05<\xd9(\xe5ڍ\x14X;\U0004f1828\x94\xc0\xe6i\u007f\x1f\xc0\x05B\xe16N\xfa#)V\xe8\x80m\xa7\xfcq&\xb9:\xfe\t\xff\x13\x9c\xee\x1f\xffd\u007fǩ>Y\x892\x8a\xe6\x96\x13}w2\x1f)\x9b\x9d\xa7\xf9\xfeT>\x8ef\xf7I~\xebD>V\x98z\x9d\xe2\xf78\x9c\xeai\\\xc7G\x92\xa3-%\a6\xbe[HP\v\x91\x05\xebږ\x9e}\xc38\xcb\xcbܨ\te\xd4#[Vh\xe6p\x19\xf18'ku\xd8c8C\x98\xa5\x80\x97XR\x96Ŕ\xaa\xc3\xd2z\v\x8a\xf1\tU&\t@j\xf6\xc9\xd7\xf5\x11x0\xcd?\x8e\xab\x91\xdb[5\x98\"/B%\xcf^\xa8\x88\xfe\xdd\x1f\xff\x105\xfb1\x9ea$`c?X\x03)\as\xb2?P\xa3\x8f\xb9\x11\x1bHy\x1ap\xc6\x0e`\x06\xf9!rk\xd8\x01\xca ,n\x9b=\x12 \xa3\x97\xe6\xec\t\xc4\xd8\x01\xc2p<\x8a4@6\x01\x18\xeb@\x8a8\x96ǃ/z\xecmO\x05\xba\xd8\x0e\xb8\x88\x15I\xd2\x1bl\xd1\x1fhq\xc4\xdb\x0ek\xe4@\xef\xdb\xf1{\x85\xe8\x8e\x109\xec\t\xaax*\xb6\x1c\x03B\xf0\x11o\x9f\x8d\x9e\xd5>\xe0\x89\x9e\xc0\x89>\xa0\x89X\xc0\xc4\x0e\xb0D\x9fHsO\xa0D/\xf1\x89=\x8e\x88>\x8a\xe8\u007f\f\xd1\xfb\bb\a \xa2ϝ\xb0\x9dG\x0f\xb1\xf7\x17\xfa\xd6>vX;>\x884\v\x9bG\x0eG=:8\xfa\xb1A<\x88a7\x80\xa1\x01D\x88\xe5\xe1&x\xa1\x0f\b\xa1\x87D\xc7*\xff\xa8C\x95h\xa5\xcd8ӌf\xaf!\xa3\xab[H\x04O\x83-\xa3\xb5\v\x91\xaa\xf5\xaa,9뙇#!Z٘\v\xean΄\xd4'\xd4\xfaӐpՊ\xe6#\xa1xNaF\xaf\xdbٓ\x1f\xf7܂|\xb4\x90\x81M)=\x86\x10\xfcU<\x101\xd3\xc0\xc9)\xe3^\x0e\xc2\xe3\xa8u\xb0\xa0\x8e\x17U\xcb\xda\xfc\xf5\xc5\xf3p\xcde;\xf3\xf9\x06v0\xb4\xa5\xd4\xd3\xc5\xf5\xdc\a\x8e\x1f\xd8s\x84gex\x8c\xbe\x15ܳ\x01¶~\x0f\x9e\xbc\xfa\x1a\xbe\x17\xd8o\xafM0J\xed\xca6D\xd0\xfcL\x85*\x1av\xb6\x17rF\"n\x1e\xdb\x057\xab\xa1c\xc1d\xb7@\xcdj\xd8XxG\xb7\xc1̢ c\x1f=¹\x06\x13\x8bw?\xb7@Ĝy\x16i\xc4G\xc3\xc3\x06?,\xb0퀁\r~\xd8'\xe4\x87}\x1e\x1e\x86f9\x88R\u007fR\xce\xc5Â%\x8b\xa6\xad\xc2rPD\x94} \xef\xc6\nq\xdd\xea\xb4.\x9f\xfa\xea\xa9_\x9dG\x12%q\xa1\xe1\xf9\x8e\x98\xd3z%\xa0\nt\x154n\xaa\b%\xafon\u007f\xfe\xee\xf2/Wߍ\xc9\x15M\x16\xcd\x12f\x9cP\xb3\xa9\x05\xd1D]\xb4\xa0K \x94\x94\x9c\xfd\xa3\xb4\xb7\x96\x92\xd3\xea;g\x1e\xbf\x17D7\x0e\xeb\x17\xb5\xcb\x18\xcd\x13\xa4\x04Z\x13\xf4\x1dSxI\x1cRqP\x18\xa1 \xfc\x02\xe9\xf6\xceC\xae\f\x19ki\xe2.\xb5\x00\tdΖ\x81N\x90\xa1\xea.V\xa4\xa9\a$\xe1\x126\xab\xc7شt*\xca@\xf5\xb9\x00\xc2A\x9b\xd5]E\xc7\x04W\xadzx\xa5\x02\x15\x86M\x9b\x96\b\xc3*$˩d٪\xd9I\x9a\x8dɍ\xf06\xfc*Ԇh\xb2\xf0\xf5۫[r\xf3\xf6\x8e\x14\x12K\x82Y\xbcQ\xf0\xeec\xa6\x97L\xc1L\x90\x9d\xf0tL.\xf9\xca~\xc8\xea\xf2@\xac\x921ځ\x1b\x82\xce\f\xf1\x17\xde>{>\xc6\xff{ffP\x86\x86\x97*hZ\xb2\x01еV\x0f\x9b\x06\xe6\xa0\xe0\xd0\x1b2\xd0\x13\x9f\x1bqL\xbc\x06\xb8t\xe3\x9a\x18\xd6K(읮\xa1\xbb%\xadD\x1a\xa7\x10\x95\xa1Y\u007fY\xb4=\x18\xef<%\xcd!\xf5\xbc\x0f\xbf\xb6O\xbc\xb1k\xe55\xc2VtrY\x88\xf4D\x91\xeb\x89\x17DZ\xbd\b\xda\xec\x04\x11D\x8d\x1c\x1a\x9b\x84\xa5\xb6\x83\xf6p\xe5\x9c<'\u007f&\x8f\xe4\xcf\x11\x14\x8d\xa9\xfc\xa7p[\xb8\x8f=\xd1'\xd0a=\xe5\xebI\xcfy\xfeި1C\xc9̌\x16dʢ\xf0\xb1f\x82\xe1Q\x834ۄ\x93\x98p^\xf6\xf0\xf6\xcc\x10>I\xb1\xb7'8׳Ɲ\xd3D/\xa2\x00\xa9\x95\x03\xb7E\xf0c\xfc8\xf2g'\xf8\xa6\x8b\u007f\x15J\xdf8u\x16\u007fK\xb6m\xda/n\x92S\x9d,\xda\xfaָ\x10Q˾.\xddHR\x81q0\v%^\xb0\b\xac\xd1\xc7[\xbaqЛ\x96\xa4nJT\x1fU\xba\x16\x12\xc0ر\xb3\xcbm\xb1\xd3H\x9dP\x88\xd49\ff\xc8ic\x93\f\xf5\x18l\xdb\xe27\xb8\bG\\\xe2x\x9d\xd4gtaB\xb9\xcdB\x9a\x81\x94\x88R\x8fa\xe9\n\xd1\x16,\x81\b\xb1\xec\xa1\x05\v)\xb4HD\xd6S\xb6&\x8e\f\x9e'\xd8`\xf5\x9bh\xd9\xfa\xdb\xeb\xc99\xb9{59'B\x92\xdbWw\x93~HfB\x9eݽ\x9a<\xfb\x80l\x8d\vN\x8d\xda\xe6X\xe0\xbb~\"\xc3\x1c\xac>\t\x85\xa18\xa7V\x04\xd08!\xa3\x9c\x16\xa3{X\x05\x99\xad\xf1\\\x8a\xe2\xd1f\xa7\xed\xe0sz8\x10\\\x02M\xd9'\x94K\xe9\x14Mݯ\xee\xa4\xca\\,\x03\xe3>\xe8\xb0y\xea\xc0\xd3B0\xaeUW\xa6e\x10\xd9M\xaf\xef\x93A,\f\x99\x96\x1dmȴ\xdcچL\xcb!\xd3rȴ\x1c2-\xbbېi9dZ~Vة!ӲцLˈ6dZnkC\xa6\xe5Amȴ\xdclC\xa6eg\x1b2-;ڐiyh\x1b2-\xab\xf6\xebA\xf8\x0e\x99\x96\x9f*\xc2wȴ<\xa4}\x1e8\xe8!\xd3rȴ\xf4|\x192-\x83ڐi\xb9ֆL\xcb\xcfU\xa8\x86L\xcb!\xd3rȴ\xec\xfe\xeeo\xdd\x0f\x1b2-?U?\xec\xf3\xf00\x86L\xcb!\xd3\xf2\xc3eZJP\xa2\x94I\xd8\x1e\xda\x16\xb2W\"/J\r\xe4\x9d'U-\xb4\xa0\x81OW\x16\xc9\xddH\xe8\xfb\xb0\x97\x18$\x82\xcfؼ\x94\x98>w\x91SN\xe70J\xec\xe0F\x15\x9fFU\xff.>\x04\xbc.c9\vK\xb54\xad\xce[\x9c\xf40\x92\"\xf7\xe4\xbe;r\xcf\xfd\xb8\xa0Z\x83\xe4/\xc9\u007f\x9e\xfe\xfd\xf7\xbf\x8cξ:=\xfd\xf1\xf9\xe8?~\xfa\xfd\xe9\xdf\xc7\xf8\x1f\xffr\xf6\xd5\xd9/\xfe\x1f\xbf?;;=\xfd\xf1\xdb7\xdf\xdcM\xae~bg\xbf\xfc\xc8\xcb\xfc\xde\xfe\xeb\x97\xd3\x1f\xe1\xea\xa7\x03\x89\x9c\x9d}\xf5epW\x8f\xbc\xbf\xb5\x97\xe5w(9\xf5\xa9$\xaa\xef\x9c>\x1a=\x1b.\n\xb9(\xb9\xb6@]\xbbګ\x05aOrC\xd7&\xf9\xc4\xd6'\xe9\xb3e\xbb\x93\xeda\x99\x06\xb4a\x99\xdae\xfa\xce\xc9N{\xa1\x06\xf71w\x06Ԏ\x85\x1a\x05B2\xdb8\xe6\xc7U\xfdd\x8a\x88\x9ci\riT\xceQ#\xab\x1a\xc1\xadMgת\xacp\x9bofq\x99L5\xa1\x9e\x8d\f1\xa1\x17 \x1fXD*\xa3q\xbfx\x1d\xf1@\xd53Ja\xc68\xb8K\x9e\u007f\xb3j/\xea5\x05I)\x99^\xbd\x12\\\xc3cP\xa4\xa0\xbdln\x1d!\"\n\x8b?\xf6\xa80\v\xfe\x0fS\xdf%\xc7L\xbf\xe0C\x8dBd,Y]\xf8A\xe1\x1c\xc0\xa3\xbex\x02+USu_\xcb\x02\x8c\x8c\xbbRO\xf9F\x0f>\x84i\x8a\xfb\xfeD\xb2%\xcb`\x0eW*\xa1\x19\xcah?\x97\xf5r\v\xd5\xe0efX!E\xa6\xc8\xc3\x02\xcc\xfa'Ԍ\x14C)\t\xe5dN#@U\xb9\x99\xab\xc2w\xce\b\x9d\xd1\x0eZ\x91\x82J\xe0\xda\u007f \\'`9\x80\xa9\x10\x99\xcbx\xc8Vu\xffY\\\b\x89\x8b\x9f9<\xfclz\xab\xc8,\xa3\xf3*!J\x81\x8e\x85y\xd4'\x8c~\xa8\xe4h\x13\xc6\x14\ue984f\x0ft\x85Ӷ\x16\xf1\x8a\xa0\xf8\x92\xbc8\xc3\xf5M\x15\xa9\xfa\x98\x92?\x9c\xe1\t\xe9\xab\xcb\xc9Ϸ?\xdc\xfe|\xf9\xfa\xcd\xf5M\x9c\xda4s\x06\x811\xfb\x84\x16t\xca2\x16c\xeem \xbf\x9a\xc4p\xbbJӋT\x8ap\xc81\xf2[\x96\x1c\xeb\xa9Թ\xe6\xfd\"<͢,(v\xb3V\x87\x83I\xce%\xe5x\xa6\xbdZ;\xf0\x96%\xd7,\xff\xa0IY4ퟐu\x99\xa6\x90\xf6c\xc9\U00070b2f|7VuM\x98(\xaa\x84L\xde\xde^\xff\xbf5\xd9\\\x15}\x00v\x1f!\x97\x84\x10\xb3\x90z\xcf\xf1;\x9b\u007f:\xcc\xf2\xce\xf6\xf9e\fUv@?L\xc1\xbb\x92\xb7+\xca\xd5t#\xec\x93\x14\xc6db\xb7fPmj\r\xa5\x1e\xae\xeb$\x10C\x92kF\xb3lE\x8cg\xb8\xa4\x19\xd8\xe4\t)\x02KՑ\xf5\x1a1Ml\u008cf*X\x91\xc7\xef\xc6Ɛyc\x9c\xe6^\xb3XQ!)p\xa1]\xb8-j5\x88\x19R#֓o\x80\xfdZ;^\x94\x91Yo\xc6Ly\x9eO\xaa\x9e\xe3)O0\xd5R\x81\xeaތ\xe3o\x957\xa3\x95@S\xcc\t/\xa8^X\x9cIN\xd5=\xa4\xf6\x87H\x1b\xdb\xc54l\x8f\xab\xa1߭\n 3\xa0\xba\x8c8rB\xdbڢw\x80\xd3i\x16\x1e\n\xedQ\x9e\x83\xa6oy\xb6z'\x84\xfe\xbaJC\xee%\xc8\xdf;o\xa9}\x16\x13\x0e\xb0\xc0ʙ\xa6\u007f#\x9cDL\xbbnfJ;\xe9\x8b\xf1\x12>\xb0\x82\x90%\xbfT\xdfHQ\x06\x9b\x02\x1b\xc6\xfa7ׯQ_\x96V\xf7\x01\xd7r\x85\xa5%b\x94D{\xcdU\xfe\xd8\xdf\xcczt+0\xdc\xe0\xf1\xeaaFJ\xae@\x8f\xc9\x1b\xba\"4S\xc29\x8e\x11\a\xb2d\x82(\xc9f\xdcgL\xb0\x86\x13\xe8\x98̦\xa9\xd0\v\xb2F\x10\xd5\xc3\xe6w\xc2\v\x10\x18\xa6\xdaC\xe8*~h\x86\xb0F7\x9c,\xbd\aE\n\t\t\xa4\xc0\x93`\xe9m\x9c'\xff\xe9_?\xd0y\xb2\x93\xfc\x1b\xc1\x8dz\xe9%\xfb\xd7\xa1\xfb\f\x96\x10Q(t\xfd\x1e\x03C\xc58\xa4^j\x90lT\xf43\xa3SȬihW\x8e\xda\\9\xd1Q\xd1Ƞ\xaa\x14Y\xff\x94\xd5w\"C\x9d\xc9\xd8)\x1e|\x95\xfa\xb3+\xf3Ÿ\xee\nW\xea\xf2Mu\xad\x81a8\xf04\xb6\x92CA\xf5\xe2\x9cH\xc8(&%;\x85vo\x03Z'q\xf3\xd4\x18\xb0\xd7\f~\xe2\xd0\xcef\x82\xc7%\xa1\xe1\xa8\xf1X\xc0[\xc43ԁF\xc1?\xfb\xce\v[D\x9d\xf3Oa7\u007ffGRE<\xe3N\xd5L\xbbg|\xe4\xce\x1f\xac\x1d\x85\xd7x\xc0\xe6\x88\x19\xcd趴X\xc7g\xf3\x90n\xe3\xd0-\xce\x03Z?\xa8\xf3NPt\xa0hǁ\x1da\xe1\xb1zb\x0fƏzhG\xb6\x1c\xdc\xc5\x1fa\x93\xa7>\xc6&Ow\xa4@\x8ep\xac@>J(/.\x99Ζ\xdd\xeay\x17\xd3m\x83J\xc3m\xd5\v\x1a~\xc5\xf1D\xa4\xb6\xf4`U\xba\f/%ƒ\x89쟡\xc0\xc8\xf65\u007f\\\xd8<\xcef=Bw\xf9L\x98\x97e\xfc\xb5\xccG(\xf3\"3\x16B\xab\xc7\xc1\x90\x17\xa4ո\x14\xea\xbcbF}\a\x95\xab\xc6\x18\xb6^\xfe\xbbT\x9a\xd0\xea\xde)_tmR}ʆ\xb8\x02/\x1eq\xd7\xfcaeq-|P\x95\xa4l6\x03\x9f\xc6\x16\xe8e\x17T\xd2\x1c4HE\x1c\xbek\nsfs\x89ČPÌ\x93\xc00TU\x8d\xe5\xdc\xe6&1Mr6_XS\x9cP\x92\t>'\xc1(G-H&hJP\x8b\vI\x1e\xa8\xcc\t%\tM\x16pn\x13\xfa\xd22x\xe1c\xb9\xff\xd5Hi\xaa\x8d+\x056[\xd7]\xf3\x9b\xf82&ab:\xdc\x106\xdc\x10vp\x1bn\b\x1bn\b\x1bn\b\vm\xc3\ra\xc3\ra\a\xb7ᆰᆰ\xe1\x860ۆ\x1b\xc2\"\xdapCض6\xdc\x10vP\x1bn\b\xdbl\xc3\ra\x9dm\xb8!\xac\xa3\r7\x84\x1dچ\x1bª\xf6\xeb\xa9L?\xdc\x10\xf6\xa9V\xa6\x1fn\b;\xa4}\x1e\xf5\xfb\x87\x1b\u0086\x1b\xc2<_\x86\x1b\u0082\xdapC\xd8Z\x1bn\b\xfb\\\x85j\xb8!l\xb8!l\xb8!\xac\xfb\xbb\xbfu?l\xb8!\xecS\xf5\xc3>\x0f\x0fc\xb8!l\xb8!\xec\xc3\xdd\x10\xa6tʂj\xfa\x1fPN2&9\xce\x17w!\x94L\xcb\xd9\f$Z\x99\xd8;\xaf\xa4\xe2\x00\xe4\xbet8֎3\xddth\x1a\x05\xfa\x1c+Xڤ\xa80\x9f\xa2\xb3[\xbeJ\rַ\x97\xa0B\xeb_2N\xae\xde~];l1\xb50\xe3\xcau\xe1x\xde\xf2$\x1eg\\\vBG\xf9\x9e0\xdeZ\xf0^\x92\te\xd1d\x96\xd9ɂr\x0e\x99\xf3lX\x18g\x17T\x91)\x00'\xa2\x00n\xd3o(Q\x8c\xcf3 Tk\x9a,\xc6f\x04a\x96\xb7\x13\x02w\xadA\xddS\xa5%\xd0\xdc\n\x83\x84<\xf4\"\n\xd3EB\x13)\x94\"y\x99iVT\x9d$\n0\x97*\xf0\xc0\xf3zVO0\x02\xcc\x00\xcd\n,\xa4X\x8d\"\xb8\x8f6O\xbe\xb1\xe6\x8d\x0fx\x8e\xb7\xf4\xe4\x85^Y\x1cv\xd8Ʒ\x002cRi\x92d\f\xb8v\xa3\xb6\x05<\xb0\x9f\xe7$\x14W\xa7\xf1\xda\x03\x9c\x05\xe5X\xcbS\xb4\xa8\n\xad,J8\xae\xa3\xae\x8b)S\xcerW\xe7\x84j\xbfQ\x06\v\xbd\x97%\x14\xfb\x14;\xe9{\xed~\x8a\xecf]\xf3Q\xd50\xf5Z\x19\xce2\x1af\xf1z\xa5t\u07bc\xea\xc9g\xe7\xf8\n\xa0\xa8V\x83\xc8b^\xaa\xe5\x02.\x1c\x0eK\xa3? \x01\xb6D[\xd2h\xc6 \x8a\xebZ\xf4ɕ\xa8\x06\x993\x8e\xc8\xf07\xa0\x14\x9d\xc3$\xf0xn\x9bs\x89't\xb5p\x05\xba\x13\x98\xae\xaeEÆ\xab\xc1\xab'\xaa\xd9\xed \xb2\xb9\x1dc\x95\x03\xf1 \x99րB\x8c\xa5\xcd\x11\xb5\x10\x88!\xdb\xe8\\\x13Z\xfb\xc6\u007f\xd0~(Tj\x8d=\xc5S\v\b\x9d\x02\x99J\x0632c\x9cf\x0e\xc3\x19\x06VƂ\xa7T\x19ѤJ\x81\xc4\b\x8c\x83\xf8yބ\t\xec\xf7\x8e\x91Z\x96<\xa1\x8dkP\xb0<\x02\x9b\x919\xe2D\x03\x03U\v\xcaɿ>\xff\x8f?\x91\xe9\xcaX\xc1\b\x16\xd0BӬ\x9a\xc0\f\xf8<\xb0\xf8\xa3۞ډ\xea\x95$०\x81\x88\x1e\xe3\f\xfc\xe1~\xda\x0eR^\xa4\xb0\xbch\xc8\xe7(\x13\xf30\x9en^1\x1bb\xc8G\x85\x05:\xd4\x00\xde#\x16\xad\b|ue\xb2\x10\x0f\xf6ځ\x9e+\xb6N\x8f(DQf\xf68\xe7k_z$\x88d\xa9`3]\xbaS\x0f\x86J\x83\xef\xda\xda.\xe6\xe0\xd6n(aF\x8b\xabL\xe0\xc2\xeaUQ\xe5R\u0098|M\xb3lJ\x93\xfb;\U0005d62b\xb7\xfcJ\xca\xc0\xdb\x0fQ\xfa=?2j\xac\x98E\xc9\xef\xed]\x8dU1*\x11\xb6ۊR\x17\xa5\xf6\tb\r\xceV\x93\x19\\0\xa42\xd0\xcc\xf8\xdb̅G\xb3n\xf1\xb2\xbc0u\xc0\t\x18~Y풉y\xd5o\xe5\x95A(\x9a\xf8\x0f\xcf\xff\xf5߭\xca\"B\x92\u007f\u007f\x8e\xe9&\xea\xdcnbh\x1b\x18C6\xa7Y\x16\x1a\bk*\x18#\xf4=\uf84e\xd3\x11:^\x1d<\x89\xcb}w\xf7\x03\xfa\xdbL+\xc8f\xe76o\xd9_\xa6\x11D\xf4\x04\x8d\xb8\x13\xb7\xcb\x1a\x1f\xe3c8\xb4K\x91\x959\xbc\x86%\xebs\xf7y\x8b\x8aϙʘ\xd2D\x84e\x90N3\x91ܓ\xd4\x11j\xe0:\xd7/:\v\xe1L\x04\x82u\xeb\xe8\x1a\xd7ӆV\xab\xceiQTy\xa0\x92>\xb4\x06\x8b\xba$\x18\xbcJc\x83\xdc\xf1'B\xb6\xbb\xa1\x06\xbb\u007f\xb7\xc1՚\x90\x17\x98\"t\xf7\xb3\xcdemmT,\xf7W,D\x9dS\xb89\xb1v\x9a\x999\xd4\xcc\xe1\xa1\xf9\x1e\a&qx\xe0\x16\x8fyuΐS\xed|\x9aȳ7\x94\xda\x02\xa4b\xca\x180\xefqM\xbc\xca(\xcb\xe3\xb3\x03c*V\xf6\xb8#(\xe6LcԐ\xd3\xc0\x17\x83\x19ݫfB\x18.֪4\xbc\xf3)^\xebOD\xea\b\xa1\xaa\xb6\xf7t\x19W6X\x1c\xb6%\x89\xf628\xfa\xaa\xfd\xf75\x8f\x9aZߎ3|9\xe3\x02\xb24\x9d\xb2\xff\x18\xea\x1b;\u007f\x04\xed\x8dz\xdb\r\xa3o͚f\xc0\xc6\tT\xc3\xf5r1\x92\xb1\xc5\xd5D\x907\x12\xe4\xbaGN^\x9e|P\x1dn\xd9-EA\xe7QwB\xafq}\x9d\x1cI\xc1\xba:QI+\xc6MF\x8aUQa\xa4\viU\xfc.\x8a\xa8Mƫ\xf7a\xef>aՒ\b\x8a\x0ftE\xa8\x14%O\xed\xd9C}(\xf5f\x8d\x1d7\x82\xc7t\x99\xa9\xaaH\xae1\xbc\x19d\x98\xf7?\x05M\xcd\xde\xf6b\xfc\xe2\xf9\xe7\xb6\xf1\xe3H\xd66\xfe\xc8\xca`\r\xbd\xf5A\xb9\xe0\xef\xf4\xebɉ7.\xc4Z_\xc1\x17U\xb2ʞ\x01!\x91\aɴ\x93\xe6\a\xa6\x80\x9c\x86F\xcd}\x13\xb2Y\a\xeb\xec\b\xb7\u007f\xf6\xb9\x85\x88\x10UN\x9f`g\xb0\n=\x82=\xa8\x84\xbab\xf1*\x9efǶ\xd2d\xfa\xb3\x98R\xb0\xa7\xb67'\xb6b\xc6\xd9\a]$nʮ\x1e\x8b\x88\xab\x05Z\xd3v\xf5XP\x8c\xfa\x17\xf5\xfcŬ\x93z\v\xdf>\u007f\x11t\xb7\x9b\x05\u007f\x81\x05]F\xed\u007f\x8a\xe5,\xa32[\x99\xa9\xbf\xb5\x9c$\xd3R\x13\xe0K&\x05\x8fBn\x12\xb2\xa4\x92\xd1i\x06D\x02֑J@\x91/O\xdf_\xbeCtWL\xd1\x0f[\xe2\xcb\xcdO\xa9\x18\x9f\x1f\x81\xa3\x8dA\xae/\x82Z\xa4#\xe8\xdaE\xe0\xf9i$\x13-\x00\xcf_\x1a\x01U\"$/ui/\\~L\xb2R\xb1\xe5\x87܋b=\xc7\xca\xd6\xfe\x159\x8e\xae\xdc\xd0k\x16\xa4o\xd6J+Պ|\xa3zQ0`\x03\x8d\xc1\xba:^'\xac&P\x8e}\x05\xf3f\xd9f\x17Pw\x95\xd7l\xfa\x81\xbb\x14 \x14\xba\xb4\x866\xc8\xe9\xfc#\x84\xd6Ce:H*\x83\xe51L\x12\x1d\xee\U000f0bb7\xcdb\xfb\xa6+\xcao\xa3\x8e9}D@%\xc5\xe5z\xe0\b\xc5\f{A\xdeC\x06R\xf8m\xe9\x812]\xe5\xaa0\xce\xf4\xabЛ\xe6\xd1q\xb2\xb5\x18\x0f\x13\x80\xa0\xa9?x^\x0e|p\xff\xb4\xed\x13\xb3\x9db\xb5\xb7\x17\xbb\xbe\xbf\xe3eƓ\xacL\xe1UV*\r\xf2\x1d(Q\xca\xceӏ\xb5\xd3\xe5η\x1a\t)\x0f\xee\xc0)\xb1\x8f\x8cT\"\x8aN\xf5 \xeb\x97+{\xc6u*\xf5ɪ\x98\xdbD]\xba\x8a\xab\n\xa2\xb4\x90\xb0\xa5*'/\xb3l-!B\x96\x1dBd\x9e3\xd6\xc9\x16\\\xf8.\xff\xc1w\xd18\x92\xaa\xa0\a\xb3\xac\U00042f7aJe,\xc1\x80=\xf7\u007f\xb0\xffez\xed>\xd21B;\x97\x16\x84\x8a\xb8,<\x9d=Gp\x05\xaf\xbf`\xb3/\xed\x877\x87\xbf5(\xb8s!\x1dĴ.9\xf4\x1d\t\x14\xb2\xfa\xf95\x86y\xc99\x84_\x9bb\xd3\xe4X-\x83\xee\xb9)M\xee\xcb\xe2\xd3b\x1f^Qv\v\x19\xda\x06{X\xf7]\xf3Y˶\x1c4]\xbe\x18\xb7\xffb|k\x96iD!w\x9af\x0f\x16\ti\xb8fL\x13\xc6S\xb6diI\xb3\x96\x046xV\xb3\x16\x93YX\xd6\x05\x90\xc2rz\xee\xfd\x16\x8f\x89\x87\xaf\x05\xaf\xd5\xddQ`\f\xfc\x18\xf3\xdbAa\xbbUp;\xbc\xb8\xf6\x8a\xe5\xa2;\xc7u\xf7\xc5)\xcfG\xa7ڍ\xff\xb0\x15f{\xe7\x8a\x19\xf9\xe7p\xe4\x977\xaf\xb7\x997;C\xf6\xad\xae^\xee\xe8\x8e[3Մ\xef\xaa\xe0\xec\f1e\xf3)\xd49\xa1\xe4\x1eV\xe7\x16\xfc\xca]\t[G\xc4^+\xe5̆{\xd8n\xaa\x98\x97-\xbdm\x8c9$\x80\u007f\x0f;c_-v\xdcê:vG\xbe\x98\x1f\xaa;\xe6+V\xb8{\xfev\xef\xfd;O9\x0f\xb27<\xd7\x0e\xee~\xc5f\tF\xf8\xac\xa8\x981\x9c(w\x97\x97\xe0j\xc1\x8a}\xe0\x18\x8a\x90m1\xf3ܯnw\xb2\xe4\xad\xfc]\xf3sr#\xb4\xf9\x9f\xabG\xa6\xf6$䘹|-@\xdd\b\x8dO\xf7f\x8e\xed\xda\xc1\xac\xb1\x8f\xa3Hs\xab#1I\t\xbfQ\r\xf3z\u007f\xee\\\xc5b\xa6\xc857\x8a\xca\xf1\xa0\xaaA\xaf\x1cy\x9f\x97\xc6\x05\x1f\xa1V\xdbm\x89\xdao\xb7\xe8[\xb6\x9ao49\xd7\xfc\xd4n\x96\xb7\xbaa\xbb`#\xda\xf6/\b\xd0.2\x9a@\xeajT\x9b\x89גj\x98\xb3ݥ\x8bs\x90s\x04\x1a$\x8b]\xa3:\xe0\xe8\xf0@\xc3\xfb\x18&\xf2vU3\xaa\xd8\xfe\x14&\xb4\xdbCp\xfb\xdc\xc2\r\x9a\xfa긓\xbd\x1am/\xc76\xf7\"\xfbi\xb7\x99\xd3\xc2H\xfe\xff\x18\xf5\x8cB\xf4\xbf\xa4\xa0L\xaa1\xb9t\x19*[\xbe\xdb|\xc3\xd9:M\xe2\x86.S\xc4\xcc\u0092f`\x8b\x1cRN`g궘m\xec\x96\xe7\xe4a!\x14\xee\f\xf5!ҳ{X=;o\xad\x90-\x14\xcd\xc3\xd7\xfc\xd9yu^\xd6Z\x94\xd5>\x85G\x18\xcf\xf0o\xcf\xc6\x1b\x1b\xec\x16\xda{\xb6ݝR\xb2㏕\xd5\xfd\xc6B\x9b6g\xfeP\xf9\xd8)\x1b\x1b՛\x9a\xdfl\tG\xd38n\xb9\x15]\x9f\xa4r\x0e\xba\xcb\x05q\x163B\x19\xc6䒯6\xe8bb\\\xa7\xc9휸JΊ\xd6\xd5\aB:\xb0\u007f\x93\x94\x03.\xa9nG\xd8<\xb89k;&\x057P\xb9\x84\x1b\x91\xc2D\xc8.|G\xfb\xbcf\xfd\xf9\x0e\x8f\xb6\xc1\x14\x91\xa5\x88\xb3\xc7G;\xfa\x8b\xb6\xb1\xb3\x8b\x8f\xe9|\xba\xefO\xde\xef\x1bϻ\xea\xc1\xdd\x03\xa1X\x97\xdb\xceW\xc78\xcc\xfb6׆\xd3B-\x84&\xa7KF]F\x93(Sw[\x84\xec8\xef\xe91J\x95, -3辰h\xa3ĕ\u007f\xd4\xdb~%g\xff(\xdbw8\xf9\b\x95{\xbak!\xd4<\xa9\\\xebFDը\xa3\xbf\xe0|\xfa/9/\xd2Q\xde\x02\x85o\x92\xb4kA(\x8dyJ\\7\n\xb6x\xb73q\xa5\xbd\xdd\xe3\x9div~\fۖC\x87\xfa\xe8\xde\\G\xee\xab\x1b'\xe2[\x96\x95\xc5ҷit\xc9ܭ\xc5\xdc'\xb4Х\xf4\xf7\xa3\x95\x12\xef\x12\xa9˟S\xcf9Ǣ\x16\xd9펁\x8b\v2\xc1\xefX\x0eJӼ\xd8#!\xaf6\xdf0\x13 dj\xbb\x86\x11\xccF\x88\xa0\xbe\xb0\xa3\xcb+\xa6\xf551\xe9\xb8AےA\x9b̐\x86\x94\xc0\x128qI>x\xf2n\x03\x10\x1dD\xede\xdbr\x89g\x01\x9e\x0e\xe2}gB\x12\xbc\x91\xa7\xea\xfa\xa6D\xf8\xd4\xf1\x94j\x18uf\x12\x1e\xb4\x12;w\x1d\x84\xe9\xefS5\x98\xfb\xe0̓\x04qbfz\xb3̾\xed3\x0f\\\xaa\xdf\x03H s\xe0\x86ŝ\x1aǙ\xb2\xf6\x9a\b\xc3X\xb7\x82\xab\x00Ν\xbdY\xa3\xa4\xee\x03֝\xa9v\x95\xad!3|\xa43\xcbjW\x02\xbd\xcb\xf8x\aTu\xe1\x96Z\x8c\xf8\xba\xf9\xac\xf3U,\x0fp\xe8\t\xb5w\xec\xd8\xdbΘ\x84N\xd1w]\x12\xf8\xe5\x805NH\xb1\xa0j\x9f\xba\x9c\x98g\xaa\xeb<\x1a\x8b\xb2Ҕ\xef\xb6\xf4\tx\x99o\x12\x1f\x91\x1bx\xe8\xf8\xf5k\x14z\xf4;\xbb\x97҈\\\xf3\x89\x14s\xd9Uan\xe4\x17V\x87\x84\x8cȄJ\xcdh\x96\xad\xbe\xee\xaed\xef\xbf\x1e\xc4;ו\xbdV\x82}\xac6D\x19\xb7\xeb\xcfH*\x9d\xe2M\x11\xb5\xb0\x9e\xa8Z\x8e;&\xd9\u007ftl\x9cP\xf0\x81\n\xd6&\x8a\x10,\xa5G0\x9b\t\xe9\xee\xc9\x1e\x8d\b\x9b9\xfd\xd9e|P\x96\xa1\tg\x0f\xd1\bӵ\x83XmHB\x12\xcaWD\xa2\xb0\xe2\xb5!9]\xd9\x10\xd3+ \x87N\xa5\x13\xb2-\xde^;\x16\xdc|\xbe\xba\x9e\xb1\xaaS\x82\xe4,\xeb\x16t\xe95C\xb6\xcdUĬ\xfe\xca\x06Uf\x95u{\"\xfb\x8aj`N\xe4\xf5v\a\xb9}\xe0V=\xec\a`S*7\x86!\x9a&\xf2\xf6p\"S\xfeU3g\xc9\x02\xab\xf1\xe8\x85\x14\xe5|\xe1Ep\x9b\x02\xdd\x16V,1\xe9\xb7\xc8ʹ\x11k\x17\xe8ӥ\xe4\r\xefŅ\xfeҺ\xbb\xbb\x88\xeef\xe1\x0e\xdb]\xb5v\xbc}\x06`\xeb\xe1~;{\x95\xee\xfe\xe9\xee\xc8\xcbJ\xa5^\x1d\xb27\xbf_{|\xed \xc5\xec\xd25E\xb7\x9fv0\xe7\x94\xcdl\xd441\xbd\xde\x04\xef|\xe0\x03\x91\a*9\xe3\xf3}\x83\xff\xde=\xd6a\x9a8\n\x1d\xc6I\xc7 *s%\xc88\xf1\x9d܂\xf5\xa9\f\x96\x1e\xe6I\xe7\x1a\xda\xf8\x11\x059m0\xd9}\xc9\xfdR\x9b\xf5\xb6̅;\xa8\xb4\xbc\xbdg<}\xe9\x01\x81EVJ\x9a\xb9\u007f&\x82[\xb5\xa0^\x92\x1f\u007f\xfa\xc2\x0f\xe8=HU\xfd\xf8\xff\x03\x00\x00\xff\xff}\xb9\x01\xa8\u007f\xcc\x01\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec<\xcdn\xe4\xb8\xd1w?E\xc1\xdfa\xbe\x00\xee\xf6\x0e\xf6\x12\xf86\xeb\xf1\"\xc6Nf\acǗ \a\xb6Tm1\x96H\x85\xa4\xda\xee\x04y\xf7\xa0\x8a\xa2\xfeZjQ=\x1ed\xb30/vKd\xb1\xfeXU,\x96x\xb6Z\xad\xceD)\x1f\xd0X\xa9\xd5\x15\x88R\xe2\x8bCE\xbf\xec\xfa\xe9\x8fv-\xf5\xe5\xee\xfdٓT\xe9\x15\\W\xd6\xe9\xe2+Z]\x99\x04?\xe2V*\xe9\xa4Vg\x05:\x91\n'\xae\xce\x00\x84R\xda\tzl\xe9'@\xa2\x953:\xcfѬ\x1eQ\xad\x9f\xaa\rn*\x99\xa7h\x18x\x98z\xf7\xc3\xfa\xc7\xf5\x0fg\x00\x89A\x1e~/\v\xb4N\x14\xe5\x15\xa8*\xcf\xcf\x00\x94(\xf0\nl\x92aZ\xe5h\xd7;\xcc\xd1\xe8\xb5\xd4g\xb6Ąf{4\xba*\xaf\xa0}\xe1\a\u0558x*\xee\xea\xf1\xfc(\x97\xd6\xfd\xd2{\xfcIZǯʼ2\"\xef\xcc\xc7O\xadT\x8fU.L\xfb\xfc\f\xc0&\xba\xc4+\xf8LS\x95\"\xc1\xf4\f\xa0&\x8c\xa7^ը\xef\xde{\x18I\x86\x85\xf08\x01\xe8\x12Շ/\xb7\x0f?\xde\xf5\x1e\x03\xa4h\x13#K\xc7\xec\t聴 \xe0\x81\t\x04S\x8b\x02\\&\x1c\x18,\rZT\x8ez\x94\x06W\x01ô\x01\t\xa0\r\x94h\xa4Ne\x02?\x89\xe4\xa9*\xfd`\x9b\xe9*Oa\x83`*\xb5n\x06\x94F\x97h\x9c\f,\xf4\xad\xa32\x9d\xa7\x03\x8c\xdf\x11Q\xbe\x17\xa4\xa4+h\xc1e\x18\x18\x83i\xcd\a\xd0[p\x99\xb4-\xfe,\xfe\x1e`\xa0NB\x81\xde\xfc\x1d\x13\xb7\x86;4\x04&`\x9dh\xb5CC\x1cH\xf4\xa3\x92\xffl`[p\x9a'ͅ\xc3Z\xaem\x93ʡQ\"\x87\x9d\xc8+\xbc\x00\xa1R(\xc4\x1e\f\xd2,P\xa9\x0e<\xeeb\xd7\xf0gm\x10\xa4\xda\xea+Ȝ+\xed\xd5\xe5\xe5\xa3ta\xa9$\xba(*%\xdd\xfe\x92\xb5^n*\xa7\x8d\xbdLq\x87\xf9\xa5\x95\x8f+a\x92L:L\\e\xf0R\x94rŨ+^.\xeb\"\xfd\xbf Q\xfb\xae\x87\xabۓ~Yg\xa4z\xec\xbc`\x85>\"\x01\xd2l\xaf0~\xa8\xa7\xa2e4=\"\xee|\xbd\xb9\xbb\xef*\x93\xb4C\xee3\xdf;\x1a֊\x80\x18&\xd5\x16\x8d\x17\xe2\xd6\xe8\x82a\xa2JK-\x95\xe3\x1fI.Q\r\xd9o\xabM!\x1d\xc9\xfd\x1f\x15ZG\xb2Z\xc35\xdb\x0f\xd2êL\x85\xc3t\r\xb7\n\xaeE\x81\xf9\xb5\xb0\xf8\xdd\x05@\x9c\xb6+bl\x9c\b\xba\xa6o\xd8\xd9s\xad\xf3\"\x98\xa9\ty\x855~Wb\xd2[24Nne\xc2\v\x03\xb6ڴ&\xc0v\xadYh\xe3\xab\x16j\xd3C݇ϏZ\x9bk\xa3\x15\xe0\vɾ]ͤ;\xcf\x19*Za\xa6R\x84\xe7\x01L\xa8M\xcc\xfa\xe0\xcd\x047\xf9\x15\x16%-\xd7\x19\x14\xef\xebn\x84\"\xf1(mܑ7(ؘ7][580*<]\x86į\x9dLk\xabq\xc0\xcd\xe3\x1c\xf5\x88mE\x95\xbb\a\x9dW\x05\xda{\xfd\x15\xad\x93\xc9X\xcf\x01\x11\x1fG\a\x06y\xa3%\x0e\xbb\f\r-N~\xc1\xf6n\x14.\U0001ac58\xb2\xc9\x13O\b\x026\x9e\x03d;\xf3\x1cJ\x9d\xc2\xce\xcf\x04\x9b}@\xfaP6\xad|6Z\xe7(Ƹ\x86/I^\xa5\x986.o\x94/\x03jo\x0e\x06qp \xa4\"-#WL\xa8\xaa\xe6\xed\x04\x9d쯄A C!\x95\x87\t\x92U\xb0&y\x9c(鰘\xc0\xf3\xa8F\xfaFA\x88\xd8\xe4x\x05\xceTc\xba\x1e`\bc\xc4\xfe\b\xcfB\x00\xb5\x84e͘ڜ\xe72AbVc\xb4\x99k̚\t\xfa\xfe\a\x19\x96i\xfd\x14ä?Q\xbf\xd69A\xc2q*l0\x13;\xa9\x8d\x1dF8\xf8\x82I\xe5pj\x1d\t\a\xa9\xdcn\xd1\x10\xac2\x13\x16m0)ǘu\xdcDP3\xc7\x05\u007f@W+t\x12\x1esc\x8a\x146œP\x81\x11'\x8b]\x95 U*w2\xadD\x0eRY'T\xe2\xe9\x13\r~\xe3\xf4\xc1\x9cB\x1c\xe0\xef\rp\xa0\x82\xa4\xd4\xf3lZ!\x85\xa3\x856\xe3\xca\x11\xda!\x98i6l\x04Y@=\xe5\x8e\xdaf(\xa2\xafQI٥\xb6v碕\x94\x0f\ns\xb1\xc1\x1c,\xe6\x988m\xa6\xd9\x13\xa3\x04\xbe\xc5\xda\xcf\tΎX\xd2\xd6g\x90\xa2\xce\x1aѶ9\rϙL2\x1f\xbf\x91\x961,H5Z\xb6\x18\xa2,\xf3\xfd1\xa2!F3\xea\xc9\xe6\x8cF\xdb\"\xcc\xc7\x10\xee\x94!i[\xa4\rnی5\xees\xbdQ\x9b7\xa6\xf7\xd0Tߤ\xec\xb7\a\xc3__ىݒ\xf6w\xb7[\xc0\xa2t\xfb\v\x90.<\x8d\x81J\x01V\x8b\xc7\xefLp\xa7\xad\x96\xdb\xe1\xe8W_-\xaf\"\xb5\x06\x8d߉\xd0\xd8Y\xddվj\x91\xc0>uG^\x80\xdc6\x02K/`+s\x87\x1cK\xcd!\xda\ttf%\xf7\x9a\f\x8a\xf5\xbd\xd4\n\xe1\x92\xec\xa6\xd9\xd2F\x8c\x18\xf0j\b\xc0\xc7\xe5a\x0f\xc32\x88\x00\tMP\xc1Y\x10i\xb0\xf0ٕ{^\x1f\xed\x13\x8e\x00?|\xfe\x88\xe9\x1c\xcb ^S\x0f\x88\xfa0\x88t\xba(0\x81Q ;Dq\x98\xd6\xec\xf1|\x0e\xed\x02\x04<\xe1\xdeGV\xa3\x9b˱F\xa2\x15\rH\x83\x9c\xd0c3\xf2\x84{\x06Ug\xe8\xa2\xe0-Q\x15ߞp\x1f\xdbu\xc0T¯\xceQx\xee\xd2\x03\xa6\"f)\xb5\xadaj\xbdv\xc0\xe98ba\x99Q\n-p\xfcD\xb2\x1b\x81\xf5\xd2\xd2O\xb8\u007fg\xbd\xf8h\xd5d\xb2\\\xc0\x012\xd8`\x91WX\xc8\xc7>\x88\\\xa6\xcdd\xbcN\x16@\xbcU\x17\xf0Y;\xfas\xf3\"-\xa1\xa8R\xf8\xa8\xd1~֎\x9f|W\x16{\"Nd\xb0\x1f\xcc\xcbRy\xb7@|Y4\u007f\x8b\x03\xbbPR\xd1Fl\xd2\u00ad\xa2\xfd\x99\xe7\xcf\x121e\x18\x90\xf3h\x15\x95匮\xd2j\xc5n:̶\x00h\x17\xafZT\xda\xf4$u\xb1\x10\xe2(\x8a5z\xf7\xe4\xad\xfc\x9b\x83\\\xf8\xb1f\xb0\xccE\x82)\xa4\x15\xe7\xdb9\xf1.\x1c>\xca\x04\n4\x8f\b%\xf9\x8dx\xa5Z`\xc9};A\v\xe3C\x8b\xd0j\xb7\x90\xc6!\xb6\xa2U\x1f\xd93\x889\xaa\xfbD\x96\xfdx\xf78*ٽs<\x14\xc5}\x91\xa6\x9cz\x16\xf9\x97\x85\x9ee\xa1\xbc\x0ec\x10\x8f\xa4\x0f?\n\xc1\xc9\xde\u007f\x91{e\xf5\xfew\x9c7\x14\xd2\xd85|\xe0\xc3\xcd\x1c\xbb\xe3C\x96\xb03U\x14H\xc2DZ =ى\x9c\xc2\a2\xde\n0\xf7\xc1\x84\xde\x1eDPq&\xe69\xd3\xd6\xfb\xfc\xadĜO\xb7Οp\u007f~q`\xbd\xceo\xd5y\x1cL\xb2\xf9\aF\xab\x89Z\xb4\xca\xf7p\xce\xef\xce90[\xb2DN\b\xde\x16hutW>~^\xb2\x15\xa0\xbdv\x88ZhpsHK!\xfc\x1c\x15\xd1:]j\xeb\x16\xa1\xf5E[\xe7\x13\x80\xbdp{$C\x18\xb3\xfb\xab\xb3\x86 \xb6\x0e\rX\xa7M8\x10%\xb3;H\x90\x93\xe4\xed\xbc\xecI\xd4M6\xd2\x03\xa6M\xe6yk!\xbcM?\xf7'\xa5\xf4\xff<̄\x83%\x86]\x1a\x9d\xa0\xb5\xf3\xaa\x14\xe99f\x12\xb6M\xb2V\xf8\xcd\xdb6\xca4Ǥ\x92C[\x16\x8a\x13kO\xd8\xd8ܼt\xf2\xced\x86\xe8w\x8c*\x9f\x82#p\xd1KQ\x88\xe1\xe1|4\xba\xd7~tX\x8050\xbfa2\x8f\x15\x1b\x95eqs\xad\x92\xbf\xb5\xc0\xa3\x90\xea\x96'\x82\xf7\xdf-X\x81`\xca\xf1ԭ\xccu\x18\xdf\n\xa4y\x10\xbb\u007f\x85pԬ\xf9\xac\xc6`O\xb2\x87'\x19\xf1\x92\x02\n\xa6\x95v\xdddM=\xd3;\v[i\xack\x11^\x00UZ>N\xfe\xbe{Luc\xcc\xc9[\xcc_\xfd\xe8NZ1\xd3\xcfuaĒ\x8du`~&v\br\v\xd2\x01\xaaDW\x8a\x13^d.h\x9a\x05\x10\xbd\x10\xbd3\x89\xf4\x99\x9d\xc1\xaa*\xe2\x19\xb2b\xed\x94j6;\xd6\x1d\xf2\xb3\x90q\xd9)8M\xacN\x16\xa8\xabY\xd7\u07b6~e\x87\x1fݫ\x80)ċ,\xaa\x02DAbY\xb2o\xdc26M\xb9\x8c\x97\xf5\xb3\x90\x8e\x0f\xfd\b6\x89g\x995MtQ\xe6\xe8\x106\xb8Ն큕)6\xe1C-\xff\xd1z\x93\xa9&`+d^\x99\x056z\xb1d\x96\xee\xdbj\xf3\xf4\xfa\x9b\xb1xDV\xcc\xccȤ\xfb\x82\xa0y\xde\u007f\x94fY\xc8\xfc\xc5\xe0뇦\xa5\x91\xa4\xa5z.:\x9d\x85\xc9\xd1k?:\xad\x95W\xa8\xfdTx:\v\x951y\vO\x9b\xf6\x16\x9e\xbe\x85\xa7o\xe1頽\x85\xa7o\xe1\xe9[x:\xde\xde\xc2\xd3N{\vO\xa3\xfdG\f\x86\xfe\xab\xa3#\x1d\xa2\xb0\x8a,\xc1\x98C{f\xae\xba\xd2\xe8:\xaf\xacC\xb3\xa4B\xfav|\xe4H\r}⻬\xf8k\xad)\xadiKWZ\xa7הLӒ\f\x8b\xc9\u007f\x8b\x11\x11\x85G\x97AOW\xdb\xc7\x16\xd0͕\xcd\xf5kǛr5\xff\xdf\x04C\x9c\x0e\xd3\xd7\xd2\xf3\xdf\xf8tk\xae\xfa\xb5o\xbc\x0f\b\x18\xff&\xeb\xca#\xcb\xdaf\x8aَ\x17\xe2Oy\xf8\xc0\xcb\xc1\xe1B\x9f\x99\xa6W\xf8\xfd\x9b\xe6eD\xb5\xd9t\x8dY}j\x89N\xecޯ\xfbo\x9c\xae+\xce&({\x96.\xf3\xdf\x1f\xd1\xd6U=v\xcbڃ\x9e\xd6\xdf\xfe\ry<\x01Q\x1bP2\xf7\x02\b\x10z\xec\x87_K\xbfE>y\x9d\xcfo\xd4\xe2\xebҖV\xa35\xf5C\xf3^\xe5\x1bjЖ} 0[o\x16\x834\xc4T\x99\x8d\u05cf\xcd@]R[\x16\xbb\a\x8f\xa8#\x8b\xaf\x1e\x8bc\x0f\xf0\x17\xbb\xb15c\xd1Q[l}\xd8\xf7\xa9\n\x8b\xac\x05\xebTx͂<\xb1\x02,\x9aaq\xd5^\xd15^\x9dʭyn\x1d\xa9\xec\x1a\xafך\x059V\xcf\x15S\xa5\x15\x85ktmVSq5\x9fI\xfc\xa6\x8a\xacׯ\xfd~\xcd8\xffx}UTUU\xd4^`\x1e稺\xa9\xa5\xd5RQ\\]Z\x19\xd5T=\x1d\x998\xaa\x1e\xea\xb0\xd6\xe9\x18)\xb3UP\xd3\x15N\xc7\xc0\x8e\xd5>E\xd45\x1d\x01٭xZ\x1c\x06\xccj\xd3L\x87\xf1\xaf\xeaC\x9b\xf7\xb5\xf9\u007fC\x03\xbf\x95hmR4\xb3\xbb\x92%\xa8Ϣ\xdd\xcfW\x0e\xe6\x1f|\xae\x13>&\xa7^\xdd\x1d\xcfT\x14\xa5\x9b\xcfG\x12\xf8E\xaa\xd4k\x12-\x96NL\xc37Tp\x81Y\x13fMWܶ\x11\xed`\xb7e\xb1\x14d\xd4S\xd8\xec}VȮ\xe1F$Y\xd3q\x02\"Ϝ\tK;\xfbB88o\xb6\xb1\x97a$=9_\x03\xfc\xac\x9b\fB\xe7\xf3\xd0\t\xb8V\x16e\xbe\x87\xca\"\x9c\xf7\x01}\xdb.lRw\xac\x12\xa5\xcdt\xb8I b#v\xd7\x1f1\x92/\t\xf7\b$\xb9\xae\xd2f\x86#\xe2\x16j\x0f_\x1e8\x92㯧\x93\xf6+\xf3:R\v\xfb\xaa\xc1G\xe8\x13 \xa7.\x8fXIJ鬊uڈG\xfc\xa4\xfd\xbd\x1a1<\xeb\x8f\xe8]\xadR\xdbՐc\xad\x8b&'5\xd9\xd36\x04\xd8\x1e\xbdԫ\xadMB\x11\xb6S&wf\x9d;\x97G\x10w\u007f\xff\xc9\x13\xe4d\x81돕\xcfp\xadJa,\x12\xa7\x03\xa1~\xd0f\xda\x12f\xfa\x19r]\xf3\xe1\xa7!\x1d\x06\xf9\xa8\x87\x93i'Q㯰\b\xea\x1bX\x17\xa3\xf2\x0f\xe3#;\x9b\xe5\x8e\x10\x8f\xe5\xc4\xf4v\x12\x96\xb0V'\x92m\x11\xa7(\xf8\x84\xe5\xfbݱp,\";b2*\x8b\xbf>+4_\xc3B\xb5\xb7\xcaKj\xe6֕\xbf\x1c\f\f\x02\x1e3\x1fd\xff\x06\xdd\xc7\xfc\x9e\xaa\x19d\xfd\xdd_ތ3\xe3\xc2=4\x87\xac\x9bY\xff\xd3k\u007f<\xa2^\x8d_\xfd\xb2jn\xa39\x8b\xe0\xacu\xc2U6\xe6\x82\x1f\xee\b\x89(]ej\xf7\x9aT\x86/\x9c \xe8\xefc8튟\\X\x17%\xcbOM\xc76\xc3`\x9d?\x8f\n\x06\n\x9e\x85\xe5\xbbs\xfciԨkn.\v\x9a\xbc=\xc7{\xd7+H\x85\xc3\x15\xc1?M\x9c\xa3\xeb\x80/蘡\xf4\v\xf5iN\xc7kF\xf3\xc0p\xb1\xc7\xdd\x14\xea\xe3ǝ+\xf8\x8c\xcf#Oo\x14\x11q\xb8\x19\xf5g\x9a\x98r\x86b\xecz\xb3\xa3$\xee\x9aQ|\xa0\xfc\xe8\xde\xfew~\xce?\xc2\xe5~\xfc3\xd1\xca\xc7\xd4\xf6\n\xfe\xfa\xb73\xa8}\xedC\xb8\xc1\x8f\x1e\xfe'\x00\x00\xff\xffa\r\b,+Q\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4V͎\xe44\x10\xbe\xe7)J\xcba/$\xbd\xab\xbd\xa0\xdc\xd0\xc0a\x05\x8cFӫ\xb9 \x0en\xa7\xd2mƱCU\xb9\x87\x06\xf1\xee\xc8vҝN\xd2̀\x84o\xb1\xeb端~REY\x96\x85\xea\xcd\x13\x12\x1b\xefjP\xbd\xc1\xdf\x05]\xfc\xe2\xea\xf9\x1b\xae\x8c\xdf\x1c?\x16\xcf\xc655\xdc\x05\x16\xdf=\"\xfb@\x1a\xbf\xc3\xd68#ƻ\xa2CQ\x8d\x12U\x17\x00\xca9/*^s\xfc\x04\xd0\xde\tyk\x91\xca=\xba\xea9\xecp\x17\x8cm\x90\x92\xf1\xd1\xf5\xf1C\xf5\xa9\xfaP\x00h¤\xfe\xc5tȢ\xba\xbe\x06\x17\xac-\x00\x9c\xea\xb0\x06F\x8aJ\xa2$0\xe1o\x01Y\xb8:\xa2E\xf2\x95\xf1\x05\xf7\xa8\xa3\xe3=\xf9\xd0\xd7py\xc8\xfa\x03\xa8\x1c\xd06\x99\xda&S\x8f\xd9Tz\xb5\x86\xe5\x87[\x12?\x9aA\xaa\xb7\x81\x94]\a\x94\x04\xf8\xe0I\xee/NK`\xa6\xfcb\xdc>XE\xab\xca\x05\x00k\xdfc\rI\xb7W\x1a\x9b\x02``*\xd9*\a.\x8e\x1f\xb39}\xc0Ne'\x00\xbeG\xf7\xed\xc3\xe7\xa7O۫k\x80\x06Y\x93\xe9%\xf1\xbd\x12\x19\x18\x06\x05\x03\n\x10\x0fJkd\x06\x1d\x88\xd0\td\x94`\\\xeb\xa9K9:\x9b\x06P;\x1f\x04\xe4\x80\xf0\x94(\x1f\"\xab\xce\"=\xf9\x1eI\xcc\xc8Ơv\xa9\xbe\xc9\xed\f\xeb\xfb\x18N\x96\x82&\x96\x1dr\xf24P\x82\xcd\xc0\x00\xf8\x16\xe4`\x18\b{BF's\x94\x89\x9f\x16\x94\x03\xbf\xfb\x15\xb5T\x03\x0f\x1c\x93\x15l\x13\xab\xf5\x88$@\xa8\xfdޙ?ζ9\x12\x12\x9dZ%c\x9d\\\x8eq\x82䔅\xa3\xb2\x01\xbf\x06\xe5\x1a\xe8\xd4\t\b\xa3\x17\bnb/\x89p\x05?y\xc2Df\r\a\x91\x9e\xeb\xcdfod\xec:\xed\xbb.8#\xa7Mj \xb3\v\xe2\x897\r\x1e\xd1n\xd8\xecKE\xfa`\x04\xb5\x04\u008d\xeaM\x99\xa0\xbb\xd4yU\xd7|EC\x9f\xf2\xfb+\xacr\x8a\x95\xc5B\xc6\xed'\x0f\xa9!\xfe!\x03\xb1\x1dr}d\xd5\x1cŅ\xe8x\x15\xd9y\xfc~\xfb\x05F\xd7)\x19s\xf6\x13\xef\x17E\xbe\xa4 \x12f\\\x8b\x94\x93ؒ\xef\x92MtM\xef\x8d\xcbե\xadA7\xa7\x9fî3\xc2c\xed\xc6\\Up\x97F\x11\xec\x10B\xdf(\xc1\xa6\x82\xcf\x0e\xeeT\x87\xf6N1\xfe\xef\t\x88Ls\x19\x89}[\n\xa6St.\x9cY\x9b<\x8cc\xeeF\xbeV\xba{ۣ\x8e\x19\x8c$Fm\xd3\x1a\x9d\xda\x03ZO\xa0\xd6T\xaa7!I\x1a\xff\x12\xcb0I2\x9a\xd9|\x89\xfd\xf9:\x9a\xf5q\x92^\x0e\x8aq~9\xc3\xf4\x10e\xe6\xfe\xadiQ\x9f\xb4\xc5l\"O\x13|\x1dJ<\xe8B\xb7\xf4Y\xc2=\xbe\xac\xdc>\x90\x8f\x935\xcd\xf5\xebs\xa36 \xffo\xf6\xc6-\u009dG\x96\xa5\xd2?l:\xaa'\x03z0\x04\x14\x9c\x8b}\xbb\x98\x90\t\xc8|\x92/d\x8c`\xb7\x82f\x15\xcfg\xd7\xfa\xb4\t\xa8\xe8XI\xee'\x1c\x92=\xf8ɸV\f\xde\xceu>\xcb\xe1\xf5&B\xf3I\u007f\xd2\xff\xa6\x1cǍ!\\\xf5]&T\xab\x0f\xd1\xe3\x1a\xe3\xeb\xfd5\xa0\f֪\x9d\xc5\x1a\x84\xc2R;\xeb*\"u\x9aW\xcdXj\x97}\xea\x95\x02Z(\xc4>y9\xa0\xbb\xd5\r\xf0\xa2\xe6S\xfe\xca3\xecN\xb7T\xef\xce\xcbᲥr\xe9\xd6\x10gw)f\x85\xb37\x91\xb2\x9a\xbd\\ҫ\x9bǂ\x90\xedTv\x9c\x19W\xad1.\"\xcb\x18nBXM\xf6\xe22\x99o&\xe1\xb1xR\xfbi\xc0\x1cv\xe7?\xfd\x18\xc80\x92\xe1Ͽ\x8a\xcbt\x8e\xcb\\/\xd8\xdcϷ\xe0w\xef\xae\xd6\xd9\xf4\xa9\xbdkL^\xe2\xe1\xe7_\x8a\xec\x18\x9b\xa7q\a\x8d\x97\u007f\a\x00\x00\xff\xff!\xec@\xb2>\f\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4VM\x8f\xdb6\x10\xbd\xebW\f\xd2C.\x95\x9c \x97B\xb7\xc0\xed!h\x1a,\xe2ԗ\xa2\a\x9a\x1c\xd9ӥH\x963t\xbb\xfd\xf5\x05)j\xfd\xb1\xf26\v4\xbaq8|\xf3\xe6͇ݴmۨ@[\x8cL\xde\xf5\xa0\x02\xe1߂.\x9f\xb8\xbb\xff\x81;\xf2\xab\xe3\xdb果\xe9a\x9dX\xfc\xf8\x19٧\xa8\xf1G\x1cȑ\x90w͈\xa2\x8c\x12\xd57\x00\xca9/*\x9b9\x1f\x01\xb4w\x12\xbd\xb5\x18\xdb=\xba\xee>\xedp\x97\xc8\x1a\x8c\x05|\x0e}|ӽ\xeb\xde4\x00:by\xfe\x85FdQc\xe8\xc1%k\x1b\x00\xa7F\xec\xe1\xe8m\x1a\x91\x9d\n|\xf0b\xbd\x9e\x82uG\xb4\x18}G\xbe\xe1\x80:\xc7\xdeG\x9fB\x0f\xa7\x8b\t\xa2\xf2\x9ar\xda\x16\xb4ME\xfbXъ\x83%\x96\x9f\x9fq\xfaH,\xc51\xd8\x14\x95\xbdɬ\xf80\xb9}\xb2*\xde\xf2j\x00X\xfb\x80=|\xca\x14\x83\xd2h\x1a\x80*O\xa1\xdc\xce\x02\xbc\x9d\x10\xf5\x01G5\xe5\x02\xe0\x03\xba\xf7w\x1f\xb6\xef6\x17f\x00\x83\xac#\x05)\"/'\x02Ġ`f\x02\u007f\x1d0\"l\x8bj\xc0\xe2#r%\xfd\b\n0\xf3\xe7\xee\xd1\x18\xa2\x0f\x18\x85f\x81\xa7\ufb3dάW\xbc^g\xea\x93\x17\x98\xdcW\xc8 \a\x9c\xd3GS\xb3\x05?\x80\x1c\x88!b\x88\xc8\xe8\xe4T\xae\xd3\xe7\aP\x0e\xfc\xee\x0f\xd4\xd2\xc1\x06c\x86\x01>\xf8dMn\xc7#F\x81\x88\xda\xef\x1d\xfd\xf3\x88\xcd \xbe\x04\xb5J\xb0V\xf6\xf4\x91\x13\x8cNY8*\x9b\xf0{P\xce\xc0\xa8\x1e b\x8e\x02ɝ\xe1\x15\x17\xee\xe0\x17\x1f\x11\xc8\r\xbe\x87\x83H\xe0~\xb5ړ\xccc\xa5\xfd8&G\xf2\xb0*\x13B\xbb$>\xf2\xca\xe0\x11\xed\x8aiߪ\xa8\x0f$\xa8%E\\\xa9@m\xa1\xee\xa6n\x1f\xcdw\xb1\x0e\"\xbf\xbe\xe0*\x0f\xb9\x8bX\"\xb9\xfd\xd9Ei\xf7g*\x90;}j\x84\xe9\xe9\x94\xc5I\xe8l\xca\xea|\xfei\xf3\x05\xe6Х\x18\xd7\xea\x17\xddO\x0f\xf9T\x82,\x18\xb9\x01\xe3T\xc4!\xfa\xb1`\xa23\xc1\x93\x93rЖ\xd0]\xcb\xcfi7\x92\xe4\xba\xff\x99\x90%ת\x83u\xd95\xb0CH\xc1(A\xd3\xc1\a\ak5\xa2]+\xc6o^\x80\xac4\xb7Yد+\xc1\xf9\x9a\xbcv\x9eT;\x1f\xb0\xba\xc4n\xd4ky\x927\x01\xf5\xc5\x00e\x14\x1a\xa8N\xf6\xe0㕮j\x9e\xf3e\xbc\xee\xc2}y\xc0a\xda\xf1\x03\xed\xaf\xad\x00ʘ\xf2\v\xa1\xec\xddͷ\xcf\b\xb6\x90\xf7\xbaDʍ:\xf8\x98\x19\x1d\xc9`l\xe7<+\x93\x14k\u0084\xd6p\xf7\x04\xf2\x86\xe65\xc9\x02\xf9\x94\xe6\x05\x8f\xbbꖙd\xa1\xe7gӆº0\xcb\xfaT{\xbc\xc5`!\xe3\xdc\xe1\x14\xf1jV\xdb\xc7\x00_\xd5;\xa2$\xf1˻\xa7<\xab\x9e\xbb\xdaA:ňN*\xe6¦\xfd\u007f:(\x1c\x14\xe3\u007fh\xbe\x1c\xe1.\xbf\x9c\xcb`i@\xfd\xa0-N\x80\xe0\x87\x85n{\x11\xe5\xfc\xa1K\xe3Sn-\xbc?*\xb2jgq\xe1\xeeW\xa7n\xde\xde,\xfeb=\x9f\x189\xafSӃ\xc44a\xd7.\xab\x96S\xf5\x95\xd6\x18\x04ͧ\xeb\u007f=\xaf^]\xfcq)G\xed\xdd4\xac\xdc\xc3o\xbf7\x13*\x9a\xed\xfc\x0f$\x1b\xff\r\x00\x00\xff\xff2\x1e\xaa\xc01\n\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec<]o#9r\xef\xfe\x15\x05\xe7a\xee\x00K\xbeE\x1e\x12\xf8m\xd6\xe3E\x84\xdd\xcc\x19;\x8e\xf3\x10\xe4\x81\xea.Ib\x81F\xaf\xa5\xbe\xb2\x15f4\xd7\xde躺\x83\xf6\a?$\xe0\xe1\xd7\xf0#\x8f\xe6\x0f\x85\xb4\xee\xe7\xce\xc7_\xa4u\xfcCU\xd4F\x14\xcdL\xfc\xcdJ\xb5\xaf\va\xe2\xd7+\x00\x9b\xe9\n\xef\xe03MQ\x89\f\xf3+\x80\xb0\x1c\x9er\x15\x10>\xfe\xe0!d\a,\x85\xc7\x05@W\xa8>>n\x9e\xff\xf1K\xef3@\x8e63\xb2rL\x14\x8f\x18H\v\x02\x9eyY`\x02\xf9\xc1\x1d\x84\x03\x83\x95A\x8b\xcaYp\a\x84LT\xae6\bz\a?\xd7[4\n\x1d\xda\x064@V\xd4֡\x01\xeb\x84C\x10\x0e\x04TZ*\aR\x81\x93%\xc2\x1f>>n@o\xff\x82\x99\xb3 T\x0e\xc2Z\x9dI\xe10\x87\xa3.\xea\x12\xfd\xd8?\xae\x1b\xa8\x95\xd1\x15\x1a'#\x9d}\xebHU\xe7\xeb`y\x1f\x88\x02\xbe\x17\xe4$N\xe8\x97\x11\xa8\x88y \x1a\xad\xc7\x1d\xa4m\x97\xcb\x12\xd2\x03\f\xd4I\xa8\x80\xfc\x1a\xbe\xa0!0`\x0f\xba.r\x92\xc2#\x1a\"X\xa6\xf7J\xfew\x03ۂ\xd3\xa3\xc1`u\x0fg\x03ؤ\t\xa9hא\x11!\xf4T\xfb+)\xe6\x91\xc5\t\x83@r+\x95\x87\xc7:\xf7\x80\xa3\f\xa2&\x1d\x96#\xb8M\x8a\x99od*Ŷ\xc0;p\xa6\xc6\t\xca\bc\xc4i\x82.Ѽ\xa7\x92\xa5\xe9\x1f\xb4H!3\xb6?\x8d\xae`\xcaxk%\xcc9F\xf0{&\xcaA\xeb\x97%B\xfc\v\xf5i\xf5\x1ed\xec%\xc1\x16\x0f\xe2(\xb5\tK\x0ffh\x8b\x80_1\xab\x1d\x8eɿp\x90\xcb\xdd\x0e\r\xc1\xa9\x0e¢\xf5\xa6o\x9a \xd3[\x99\x9a\x99f\xe6\xd9:ZF\x92\xa4\xf2ʧP\xa7\r=\xdcW\xb1\x11\xa2d4\xc8mQ\xb9<ʼ\x16\x05He\x9dP\x99_\x8fh\xf0:_\x0f\xcc1\xf9\fg\xaf\x0e#\xe6ĉ\x9ej\xd4\nA\x1b(\xc9\x1e\x9cw\x1dZ\xb0\xb6M-{+H;i/\xa2\xa6.І\xa9rֹ\xad\x0e\xb8\x99\x04\xddp\xc4\xfb\x12\x85\xd8b\x01\x16\v̜6\xe3\xe4Xb\xb2o)zm\x82\x8a#\x1a\xae\xd5ݴ\xd4va3 \x81\xd4\xf6\xebAf\ao\xe6I\x82\x18\x0e\xe4\x1a-\xefrQU\xc5ij\x91\xb0\xc4\xf90\xc9\xdcFo\xdb\u0096\x1f\xc2\x1b\xdb\xfcmKЍm[В}\xca6\xe2\x00N\xcf.\xfb\xff'a\xa3\xda\u007f\x83\xd0nΆ\xbe\xaf\xd0\x12I%\xb9\xf3\x9b\x1d`Y\xb9\xd3\rH\x17\xbf.A$g\xa5\x9d\xff\xef\x981\x97K\xfcf8\xf2]%~\x96+K\x10\x89+\xcd\xf4\u007f\x87Lac\xf1%؊d\x86\xfc\xd2\x1du\x03r\xd70$\xbf\x81\x9d,\x1c\x9a\x01g~\xd3~y\x0fb\xa4\xd8;j\xa5p\xd9\xe1\xe1+y^\xb6M8%\xd2e8\xd8\xfb\xafџ\xef\x1b\xe6\x05\xb8\xc0\x01\xaa4X\xfa\xc0\xf7\x89\xa9\xd9~a\x8f\xea\xe3\xe7O\x98ϑ\a\xd2$\xefl!\x1f\a\xc8v\xa7\x0eNy\xea2\x82\xeb\xd3\xc47>\xa5q\x03\x02^\xf0\xe4=\x16\xa1\x80\x98#h\xa2\x89H\xe7\x9c8\x9c[a!{\xc1\x13\x83\tɒ\xc5ѩ\xa2\xe0\xdb\v\x9eR\xba\r\bH8I\x1b\x92@DI\xfa\xc0\x84\xe0\xd8:\x9dx\xc0\x89\xaf\xa8\x8b\x96\x17\a\xe9\x8a$\xb6H\xfb7,\xb3a['iȌ\xfd`=\x8bh\x17\x1cd\x95\xb8P2s`\x91wKL}=\x8bB\xe6\xcdD^\xee7j\xda\x1b\xee\xb7\xcf\xdam\xd4\r<|\x956d\x1f?i\xb4\x9f\xb5\xe3/߄\x9c\x1e\xf17\x10\xd3\x0f\xe4\xed\xa5\xbc\xda&:tsh\t\xc2\xed\xdb\xc6Gx\r{\xa4\x85\x8d\xa2\xb8%Ѓ3\xa2~\xbay\xfb\xd0oem9I\xa6\xb4Z\xb1\xa9\\\x8f\xcd䉝\bR\x9b\x1eG\xceQk&\xf5\x13&\x82}\"K\xe2\xc7\xfb\x1co!2\xcc!\xaf\x99\x98\x9c\x99\x14\x0e\xf72\x83\x12\xcd~\xcept[E\xfa=\r\x85D\xad\xebۅ\x12\x96f\xdac\v\xaa;_FfE;7\xa1Wd\xf6b\u05c9\x84\xe4t\xd7\xe5\x15\xb1\x89e\xffc\x91\xba\"\xcf\xf9,I\x14\x8f\x17h\xfc\vxqn\xfb=b\xdeB\x96\x82\x93\x8c\xffCf\x8e\x05\xfa\u007f\xa1\x12\xd2$\xec\xe1\x8f|4T`ol\xc8bu\xa7\xa1\x19\xa4\x05\xe2\xefQ\x14\xe7\xa9\xee\x91\xc5i\xd2-XxC\xaewg\x1e\xcb\r\xbc\x1e\xb4\xf56u'q4\xa5\xdao\xd2\xc2\xf5\v\x9e\xaeo\xce\xf4\xc0\xf5F]{\x03\u007f\xb1\xbai\xbc\x05\xad\x8a\x13\\\xf3\xd8\xeb\xdf\xe2\x04%JbR7>\x82Ku\x95)\x96\x8c\x9e\x00\rlΝ\xc8͝\xc3:I\x0e+m]2*\x8f\xda:\x9fY칥\x97d\xb1\xc0\xcbP\xc8^\x81\xd8\xf9\x93?m\xe2\x99\x0e\xa9\xbdA\u0095\xb8f\xe75,\xb1\xb1Ɉy\xa0\x14X]\xb7;\xd8\xeb\xd3k\u007f\xd0Ó\x88\x8c\x9d\x8bE\xb8\x95\xd1\x19Z;/\"\t\xdaz!I\xd8$\b\x85\x0f`\xfc\x81\xc9|R2\xb6t\x87\x94\x88t\xa1+\xff\U000354fd\xa4\xcdO\u007f/\tߥx\x01\xefٲ\x14Ó\xc1$\x14\xef\xfdȸM\x02 \x1f\x1a\x98}\xcd[=݃\f\x82\xf4{0ӥT\x1b\x9e\x00~xw\xb3\xde(I|\x8b\xe3~\x1fǶDo>\xf0\xeeM\xf5\x884g\xee\r\xf68w\x9e\xe7&G1\x11\xa4Ү\x9bN \xb8\x95\xce?X\xd8Ic]\x17\xd1T\xa1\xa8\x17v\u007f\xdb.\x8d\x9cԃ1o\n\x9c\xfe\xecGv\x12Y\a\xfd\x1a\xcfW'\x0f3\xc7\x1a\x1f\n!\xc8\x1dH\a\xa82]+N\xbf\xd0V\xe7)<\v\xbc\x82N&Y\x9a\x82\xa0\x86\xaa.\xd3\b\xb0b\xa9\x93j6O\xd3\xed\xfe\x93\x90ŷ`\x9b\x93%\xeaz\xd6p\xb6\xadǶ'?\xb2wP^\x8a\xaf\xb2\xacK\x10%\x91>5\xec\xd9\xf9\xe2\x98\x1e\xc7\xe1UHǖ\x83\xe0\xb2\x19q\x9a6UU\xa0Kݑ[\xdci\xc3\xfb\xd9\xca\x1c\x1b\xc3\x1c\xa4@+\x10\xb0\x13\xb2\xa8M\xa2\x86\xbc\x88\xb6\x97\xc4\x1aAY\xbc_\x10\x916\xf9\x8aI\x91\x90\x88Mt\x16\xe7\xb5ue\xd2]\xc5G\x83i\xee\xd9RR:\xbag\x95\x91$K\xfa\xbd=\xb4 bB\x9d\xbe\xbbhg\xed\xbb\x8b\xb6о\xbbh\x93\xed\xbb\x8b\xb6ܾ\xbbh\xa1}w\xd1b\xfb\xee\xa2}w\xd1\xe6\xba\xcdi\xeb%\x8c|\xc5\xfdď\x8bX$\x1cOϡ8\x03?TS\xdc\xfb\xea\xfb\xd4\n\xcb\xcd\xf8\xa8\x91\xba\xdaPֿ\xe2\x1b\tc\x12\xd0\x16]\xb4\xa6\xa4)\xb9\xa4\r\x12\xc5\xdb\x17\x10/\x14a&\x95S\x8eWߦ\x14\xfc,\x95\xf9\xf4\xebL\x9b2\x9bXh\xaa\xe3$#t\x887\x1b\xc8\xed\xed\u0590\xf4\xebu\xd8ύ\x98\xfe\xcdkP\x13Jq\x16\np\xe6\vs\xe7\xe85\b=\xfa\x043\xbd\x82\xd1\xdf\r\xbd\x16\xaad\xa6kc\xc2I\x10:q\xfca\xdd\xff\xc5\xe9P)\x03\xaf\xd2\x1dF\x96\xf2z@\xc5gXj\xdf-{\x8d\xf2\x16\xae\x98\f\xe9\bڀ\x92\x05\x93sFZ{\xe4\x85?W>\x84\xbbx_·\x1fi\xb54o\xae\xa0\xe9W\xc8L\xa8\xe8K\x8f\x8c\xd2\v\x85\xd3kd\xe6\x8bZ.\xa9\x8c\x19ֽL\x02]\xae\x87I\x89\x1c\x17j_\xdeP\xf1\x92X\xed\xf8\x9b\x0f\xc6RjZ\xdeTɲX\x10\x98X\xbfүL\x99\ayA\xd5J\x12q\x96+T.\xaeK\tu \xb3\xebH\xaeF\x19\xa93\x99\x05\xf0\xb4\xaaY\xac\x06Y\xf4\x91\xe7\xf1[\xac\xf7\xb8\xa4\xcac\x91bo\xac\xe8h*6&潴\x8e\xa3_\xa71\x014\xa5zc\xa2:c\x02\xe2l\xcdFjM\xc6\x04\xec\x05\xb3;+%3?\x8e_\x84\x84E\xfbV\xfc\xb5$\xea\xad\v\xd3&G3롧\xa29\x8bb?\xe35\x98sPf\x1f/NR\xaf\xae\xd7?\xc6rݔ\x84g\xf0\xb3T\xb9\x97\x13\x12\xf4\x8e\x9f\xc0\x17\x85\xb9(\xa6qWZ\u007fo\x1c\xe8 ҰX\t\xc37ɷ'\x9f\xad\xb0kx\x10١\xdf\x11\x0e\xc2RLZ\x8e\xbaa\xd7M\x98v\x1bGї\xeb5\xc0O\xba\x89\x84\xbb\u05ec\xac,\xab\xe2\x04\xb5E\xb8\xee\x0fy[\xcc1*\x01V\x89\xca\x1et\xbc\x03\xbb\x10v|\xe9\xf7\x1e\x89\xe8\xe3\rج\xd0u\xde@\x9f`\x9eP'x|f߇\xef\x0ef\xed=\xca\xe0\xdf\xc4Hbx\xcd\xf2\xc7\xf7\x8f\xf0\xad\xd3F\xec\xf1\x17\xed/#/Q\xa2\u07fbw\x13=谘q\x8b\x05Yb\x84\b\xe1Z\xf4\x00X\x9bH\x0f\xbb\xa1M~\x10\x96c\xeamf\xff9W,,\xe6\xe9\xe9\x17\xbf\x00'K\\\u007f\xaa}6eU\tc\x91\xa8\x19\x17\xe6\am\xe9\xbf\a\xfd:\xa6\xf0tX\xf3\x8fC\xbc\rr\xb2\x9e\x936\x17a\xef\xafMG\xc1\x8b$Z\x12\xd4\xe7\xf1Q\x9d@\xaf\xc3$\xbf\xcb\xf5ع\xc4\x14\x9c\xce\xeb\x12\x14X\xfbb\xbb\xf7\xbd\xfb;\xe5\xb1Lݿw\xc2\xd5v\xf9\x06>w\x8b\xefm\x84#\x9f\xda\xf0\xc5]\x0f\xc2_t}\xd3%\xfc\x90\xa1\uef412ϧ\xfb\xf3\x11\xfc҅\xc9=j\x9c\x1bonӿ\n\xdbd\xc1GM|\vΏd\x0f\x9a\xa0a\x0exD\x05Zqқ\xaf\xc4\xfa\xd7X\x86cƒI\x1d(!\xab^W\x85\x16y\xdc\xe1\xd1f\x85\x17<\x9eX\u007f\x99#\x9a\x0fv\x06&\xbf\x18\xb0\xd3f\x8c\b\xe7\n\xd3\x1b\x96;ȅ\xc3\xd5(\xd0$\xdd7*l|>\xb4xo\x9e;\xf9\x1dćK\xf1q\x03\u007f\xbaT\xa2\xb5b\x1f/̿\x92\x02ۣB\xb6\x9b#\xeb\x0f\xf1L{\x1aѿ.\xeeS*\"s\xb5\b\x13\xc4lR\xa7ׇ1\xb3R\xe8=\xecd\xc1]\xc3\xd3\x1eA\xb3O\xa9\x1d\xa9\x1c\xeeq\xe8\xae\xe2\xd7J\x9a\x14K\xf0\xd0t$\xdap>\x8d\xb5A\xfb\x04\x0e\x16r/I\x8d\x12\xb3\xf7\xc2l\xc5\x1eW\x99.\n\xe4҅s\xbc\xbe%\xaf=\xec\xd1'nΖ\xf6S\xb7o\xf4\xa7\x82\xb0{8\xf1ś\x9b`\xa1\xc7\x1d\xd4R\xfcE\x9b\x1b(\xa5\xa2\u007f\xc8\r\xe3\xc84\x0e\xbe\xc8\x1e\xf0\xeb\x02\vx?R\x9f\xe6@\xbb\xa3\xdd0\x8aٔ\xff0~\x88\xb9\x82\xcfxn\xee\xfc\xb9$\xe6\x9c{\x19{ׇ\xbalԣ\xd1{\n\x9dF~\xbc\x8f\xbal\xe4\xb7Ga\x9c\x14Eq\xf2\x93L\xce>\xf2\xc3'$m2iR\xc6\xc9\x1a\xb0\\\xa2l\xe8\xd6\xc6iRyI\xe0\x13ŭ\xae]o\x83\xb6\x1b|D,\xe2\x9ck\xf8\xac\x1d\xc64\x9e\xec\xc3$\xbb\x8a֭p\xb7\xd3\xc6\xf9\xb0p\xb5\x02\xb9\v&j\x04.ixN]\xfbgv@\xba6}\xd2J/{\x9f\x06\x85e\xe9u\xfc\xd8\x0f\x9f.\x89,#\x0f\bo\xad\x13ň\xd6\xf8M\xd9j\xf6\x05H\xfa0\xff\xb7\x11\xe3xF\xf0M\xb7\u007fS\xe2_\x97[\x1f\x1d18O9>\xfd\xf7\x1a\xb3\x98ʣl\x11\x15\xbc\x1a\xe9\x1ci\xa9nn\x1f\x1c饢\x00\xaba'&\x1et\x98ӗ\xfc\xbbv\xa2\xd8L\xe7\x94\xfaNg\xd39.\x8b\x87\x9f/N\x13[\xb6L\x82\x89e\xf9\xea2i\xe3Xbev\x10jOBet\xbd?D\xb9\x9c\xb07S\x99\xf8\x9a\x90\x82\xaa\xa8\xf7$\xea!7\xeej\xa3:qqȖ\xe7\x1dtE\xf62\x89i\xc8\x0eƧ\xden\xc3\x13\r\xab\x9d\xd1\xe5*\xf0\x82\xf3\x067!^5R\x93SF\xd1\xd5\x04\xd0\xf6.4\x8bAU\xa1\x02a\x03>\t\xa5o\xf3l\x9d\v\x1e\x9d0.\xd5\xd5\xfb\xd2\xeb\xbc\xe0\xe51\xe4q|\xbf\x84hܗ\x00\xde\x0f\x1fݣ\xb8Y\xc5W\xe6|\xb4\xefE\xc1\x92\xf3g\x90\x03\xa8\xd1\xf3\x8a3\xb7\xad\xe7\xa4\xf5\xd1\xff\xeb\xfag\xc7\xc6\xc2<\xa4xjσ\xee\x83sT\xda\xe5-\xc4\xe0]\x8d\xd0\xe3\x0fr\xe7\x0fR2\xc2\xfa\x8f\u007f\xf3\xf3\xd1c\x92\xcf\xf2a\xd6]aO\xa4\xf1;\xe0\x13V\x063ڽc\xcbx,\x90\xfc\b\x8b\xd8\xf7\x84>\\\xe4H\xf6\x03X\xfb\xd19,\xab\xd1\x19g\"\xd8vؔ\xb2\x14\xb1\xc3\xc8B\xe2\x03\x88\x11X(-\x9a\tY/XP\xe3\xc4\\\xb6\xa0f\xd8Ԃl\x9d\x91\xd2\xda\xd5\xe3欉\x03\xdfyu\xaf\xc2(\xa9\xf6K{\xec\xdfC\xb7\x91x(@\x18\x89\x88F\x96\xd1\xc4H\x8b\x11Q' \x8a8N\xbcK6\b\x92\xde)$\x1a\xb5\x03g\x1fY\x81杽\x1df\n_\xda,\x85\xc82$q\xfd<|\xe8\xf4\xfa\x9a\xff\x88o\x99\xf2\x9f\x99V\xde\xdc\xda;\xf8\x8f\xff\xbc\x82\x90\x06{\x8e\x8f\x96\xd2\xc7\xff\v\x00\x00\xff\xffy\x8fd\x10\x14V\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4YK\x8f\xe3\xb8\x11\xbe\xfbW\x14f\x0f}\x19˳\xc9!\x81/\x81Ǔ\x00\x83\xf4l7Ɲ\xce!\t\xb04Y\xb2\xb9M\x91\nI\xd9\xeb\x04\xf9\xefA\xf1!ɒ\xdcv\x0f\x92]^\xba\xcdG\xb1\xea\xab75\x9b\xcf\xe73V\xcbg\xb4N\x1a\xbd\x04VK\xfc٣\xa6_\xaex\xf9\xbd+\xa4Y\x1c\xbe\x9f\xbdH-\x96\xb0n\x9c7\xd5Wt\xa6\xb1\x1c?a)\xb5\xf4\xd2\xe8Y\x85\x9e\t\xe6\xd9r\x06\xc0\xb46\x9eѴ\xa3\x9f\x00\xdcho\x8dRh\xe7;\xd4\xc5K\xb3\xc5m#\x95@\x1b\x88\xe7\xab\x0f\x1f\x8a\xdf\x15\x1ff\x00\xdcb8\xfe$+t\x9eU\xf5\x12t\xa3\xd4\f@\xb3\n\x97\xb0e\xfc\xa5\xa9\x9d7\x96\xedP\x19\x1e\xef*\x0e\xa8КB\x9a\x99\xab\x91\xd3\xd5;k\x9az\t\xddB\xa4\x90؊\"}\f\xc46\x91\xd8}\"\x16֕t\xfeϗ\xf7\xdcK\xe7þZ5\x96\xa9Kl\x85-no\xac\xff\xa1\xbbz\x0e[\xa7\xe2\x8aԻF1{\xe1\xf8\f\xc0qS\xe3\x12\xc2\xe9\x9aq\x143\x80\x84Y\xa06\a&D\xd0\x02S\x8fVj\x8fvmTS\xe9\xf6.\x81\x8e[Y\xfb\x80r\x94\x05\x920\x90\xa5\x01\xe7\x99o\x1c\xb8\x86\xef\x819X\x1d\x98Tl\xabp\xf1\x17\xcd\xf2\xff\x81\x1e\xc0O\xce\xe8G\xe6\xf7K(⩢\xde3\x97W\xa3\x8e\x1e{3\xfeD\x028o\xa5\xdeM\xb1tϜ\u007ffJ\x8aV\xeb \x1d\xf8=\x82b\u0383\xa7\t\xfa\x15\x11\x02\x82\b!#\x04G\xe6\xd2=\x00\x87H%`4ͩ\x1a\xddu\xc66\xb1\x02\xcf\x03*\x91\u007f\x9aI\xdc\xf7\xc8f\xc3/FF{Fw\xb5\xc3K\xc4Π\xf8\x84%k\x94\xef\x8bJZR}\xbb<\x17\xabF^\x88x\xea\xec\xc6Ogs\xf1֭1\nY\xa4\x12w\x1d\xbe\x8fV\xc8\xf7X\xb1e\xdaljԫ\xc7\xcfϿݜMÔ!\r\x9c\x82\x14\xc7z\xba٣Ex\x0e\xfe\x17\xf5\xe6\x92h-M\x00\xb3\xfd\t\xb9\xef\x94X[S\xa3\xf52;K\x1c\xbd ՛\x1d\xf0tGl\xc7] (:a\xb4\xa3\xe4/(\x92\xa4`J\xf0{\xe9\xc0bmѡ\xf6}x[\xc6J`:\xb1W\xc0\x06-\x91!_n\x94\xa0\xa0v@\xeb\xc1\"7;-\xff\xd5\xd2v\xe0M2^\x8f\xce\x0fh\x06\xff\xd4L\x91\xa96\xf8\x1e\x98\x16P\xb1\x13X\xa4[\xa0\xd1=za\x8b+\xe0\vٻԥY\xc2\xde\xfb\xda-\x17\x8b\x9d\xf498sSU\x8d\x96\xfe\xb4\bqVn\x1bo\xac[\b<\xa0Z8\xb9\x9b3\xcb\xf7\xd2#\xf7\x8d\xc5\x05\xab\xe5<\xb0\xaecЬ\xc4w6\x85sww\xc6\xeb\xc8k\xe3\bQ\xf3\x15\rPČV\x10\x8fF):\xa0i\x8a\xd0\xf9\xfa\xc7\xcd\x13䫃2\x86\xe8\aܻ\x83\xaeS\x01\x01&u\x896*\xb1\xb4\xa6\n4Q\x8b\xdaH\xed\xc3\x0f\xae$\xea!\xfc\xae\xd9Vғ\xde\xff٠\xf3\xa4\xab\x02\xd6!c\xc1\x16\xa1\xa9\x83\xdf\x17\xf0YÚU\xa8\xd6\xcc\xe1\xff]\x01\x84\xb4\x9b\x13\xb0\xb7\xa9\xa0\x9fl\x87\x9b#j\xbd\x85\x9c\v/\xe8kҋ75\xf23\xff\x11\xe8\xa4%\v\xf7\xccc\xf0\x8b\x01\xae\xc9\xc5/'\xd3<\xa6\x9d\x9b\x06\xe3\x1c\x9d\xfbb\x04\x0eW\x06,\xafڍg<\xd6h+\xe9BZ\x84\xd2\xd8a\xc6`m\x04\xee\x8f\x1c\xa9\x8a\xd1\x1a\xea\xa6\x1a32\x87\xaf\xc8ăV\xa7\vK\u007f\xb5ҏ/\xba\xa0H\x1a\x91\xc5\xcdI\xf3G\xb4҈+\xc2\u007f\x1clo!؛#\x94\xc1\xac\xb5W'\x8aA\xee\xa4\xf98\xda\xe6\xb1z\xfc\x9c#ot\xa0\xe4o\t\xab\x02V\xc9sM\t\x1f@HG\x05\x80\vD\xc7`QyF\xebK\xf0\xb6y\x93\xf8\xdc\xe8R\xee\xc6B\xf7k\x9aK\x16s\x85\xf4\x00\xb9u\xb8\x89B\x13YGm\xcdA\n\xb4s\xf2\x0fYJ\x9e8il\xcc\\\xa5D%\xdcX\xd2\v^\x16D\xb1(ȫ\x99\xba\xa2\xc3u\xbb1\x94\xc6L\xeah\xc1\x1d\x81\x10ll\x95R\xaa\xf6\xa8E[\x8d\x9cqcB\xd4r(\xe0(\xfd>\x86C5\xe5w\xf0\xaa\xef\xd1x\xc1\xd3\xd4\xf4\x80\xf7\xa7=\xd2Θ@\x11\x1cr\x8b>X\x1b*2\x1f2\xa5\x02\xe0K\xe3B@\x1dƉ\x86\x8d\x19\xf0\x9a\xf9=H\xed\xa4@`\x13\xf0\xc7b킠m\xfe\u007fHQ\xe4\x1b\xd4\xf3\x9a\xb7Gv\xde\xe2\xf0\x19\xe3+\xfe\U000d8db5(\xe4\xdf)\xf2\x9fׂ\x97\xfcxR\xa2C\xfb`\xf0\xa7Xa\xf1\x89Ty\xc6\xcc\xf3\xf8\xc4+\x95Z~\xb6\x98rf\xaa\v\x8c\xb5\xe8j\xa3\x055O\xb7\xd5i\x1d\xcb\xff\xbbjmZ\xad\xf3\xf3(7X\xcbZ\xb8\xa9U\tO4onV\xe2\xc3U\xbf\x150[G\x9dbׯ\fd\xfcEڔw\xbd>\x85\xfaa\r\x8d\x0e\x95Z\xc8\xf8\x05\xfc]\xc3'\xeam);\x89%\xf1m\xa7\f@:\xd0\xe6H\xc7{\xf4\x02\t0:\xe6k\xea֘\x16\xa9\x19\x0eKG\xa9\x14el\x8b\x959Lfl*4-\xaa\x130G\xa6s\xf8M\xf1\xa1x\xf7\xabuA\x8a9OM\r\x8a\xafx\x90\xe3W\x9e1\xba\xf7\xa3\x13\xd9\xf1[w\xa0\x1f?\xe6fyaӶ\x1f'\xc0(\xa5\xa2Zp\"Nt\x15\xc3\xf8=\xf2\xe3\xe6\xfe΅\x12\x1e\xb5\x9f*\xfb\x8eh1tL(\xa8\x8a7\xe9]\xa2q\x1e\xed\x84\x01\xb4\xda\v:\ae\xf4n\xe08q\xa4W\n\xaaТA\x19\v\x02=\xa5&\xbd\x03\xbegz\x87\xdd+T\xe2\xffuN\xc9|\x066\xd3Y\x88ԗ\xcc\xe3&\x8d>ɩ2}\xf4\x02\xdcm\x9e~\xfd\xcd\xdcg\xcd^ls\xae\xe0>ڟ\xb34\x81:\xf7\u074bp7\xbe\xbd\xbd\x1d?7߀\xc4[\xdf\xc2_y׀#sݫ\xf8\xaf\x87C\xf8@p-\x81Ӟ,-o,\xb5i\u074bSp\xb8\xa9\xb8}\xfb\xd3\xcdj\xf0\x1d\xa3\xbf6\xfe\xcaq\x83\\\x93yl4\x19sQ\x0f\xb3\x14Z\xfa3Ͷ}\x85͜\xa7l\b\xff\xfeϬK\x8c\x94}j\x8f\xe2\x87\xe1g\xacw\xd19\xf3\xb7\xa8\xf0\x93S\xc5\x10\xbf\xc3\xc1\xdf\xfe1\x8b\x17\xa3x\xce\x1f\x8fh\xf2\xbf\x01\x00\x00\xff\xffHX\xbeK\x01\x1c\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4V=s\xe36\x10\xed\xf9+v.\xc55\x11u7)\x92Q\x97\xf8\xae\xf0$\xf1x\xec\x1b7\x99\x14\x10\xb0\x127&\x01dw!\xc7\xf9\xf5\x19\x00\xa4%Q\xf4\xc5)\u008e\xfb\x85\x87\xf7v\x97lV\xabUc\"= \v\x05\xbf\x01\x13\t\xffR\xf4\xf9M\xda\xc7\x1f\xa4\xa5\xb0>|l\x1eɻ\r\\%\xd10ܡ\x84\xc4\x16?\xe1\x8e<)\x05\xdf\f\xa8\xc6\x195\x9b\x06\xc0x\x1f\xd4d\xb3\xe4W\x00\x1b\xbcr\xe8{\xe4\xd5\x1e}\xfb\x98\xb6\xb8M\xd4;\xe4R|:\xfa\xf0\xa1\xfd\xbe\xfd\xd0\x00Xƒ\xfe\x85\x06\x145C܀O}\xdf\x00x3\xe0\x06\x1c\xf6\xa8\xb85\xf61E\xc6?\x13\x8aJ{\xc0\x1e9\xb4\x14\x1a\x89h\xf3\xc1{\x0e)n\xe0\xe8\xa8\xf9#\xa8z\xa1O\xa5\xd4O\xa5\xd4]-U\xbc=\x89\xfe\xfcZ\xc4/4F\xc5>\xb1\xe9\x97\x01\x95\x00!\xbfO\xbd\xe1Ő\x06@l\x88\xb8\x81\x9b\f+\x1a\x8b\xae\x01\x18\xf9(0W\xe3\x8d\x0f\x1fk9\xdb\xe1`*~\x80\x10\xd1\xffx{\xfd\xf0\xdd\xfd\x99\x19\xc0\xa1X\xa6\xa8\x85\xd5\x05\xfc@\x02\x06F\x14\xa0a\x04\a\xc1#\x04\x86!0BE*\xedK\xd1\xc8!\"+M\xfc\xd5\xe7\xa4uN\xac3\b\xef3\xca\x1a\x05.\xf7\f\nh\x87\xd3Mэ\x17\x83\xb0\x03\xedH\x8012\n\xfa\xdaEg\x85!\a\x19\x0fa\xfb\aZm\xe1\x1e9\x97\x01\xe9B\xea]n\xb5\x03\xb2\x02\xa3\r{O\u007f\xbfԖ|\xcf|hot\x12\xf9\xf8\x90Wdoz8\x98>\xe1\xb7`\xbc\x83\xc1<\x03c>\x05\x92?\xa9WB\xa4\x85_3M\xe4wa\x03\x9dj\x94\xcdz\xbd'\x9dFƆaH\x9e\xf4y]\xba\x9f\xb6I\x03\xcb\xda\xe1\x01\xfb\xb5\xd0~e\xd8v\xa4h51\xaeM\xa4U\x81\xee\xcbش\x83\xfb\x86\xc7!\x93\xf7gX\xf597\x8c(\x93ߟ8J7\u007fE\x81\xdc\xcbU\xf6\x9aZoq$:\x9b2;w\x9f\xef\xbf\xc0tt\x11c\xce~\xe1\xfd\x98(G\t2a\xe4w\xc8U\xc4\x1d\x87\xa1\xd4D\xefb \xaf\xe5\xc5\xf6\x84~N\xbf\xa4\xed@*SKf\xadZ\xb8*{\x04\xb6\b):\xa3\xe8Z\xb8\xf6pe\x06쯌\xe0\xff.@fZV\x99طIp\xba\x02\xe7\xc1\x95\xb5\x13Ǵ\xa3^\xd1kah\xef#ڬ`&1gӎl\x19\x0f\xd8\x05\x86\xa7\x8el7\r\xed\x8cݗ\x01o\xcf\x1c\xcb\x03\x9d\x9fZ&/\xa5\xb9\xe7\xd5\xcbCю\x18g]\xb8:)\xf6&^\xd4h\x92\xff\xc8Lə\xb8\xb1\x89\x19\xbd\x8e\x95ʶXJz+\x17\xc8\x1c\xf8\xc2:\x03\xf5\xb9\x04\x95\xef\x9c!/`\xfc\xf3\x98\b\xda\x19\x85'\xe4<\x066\xa4\xbcgЁK\x17\xfc\x8d\xb4tX\xc5\xca\xc2F\x0e\x16Eڋ8R\x1c\x160}E\x9d\xfc\xe4o\xa8\xd9\xf6\xb8\x01儯(k\x98\xcd\xf3\xcc\x17;#\v\xadpF\xc1m\x8eY\xd2\x00\xebV\xc7\u007f\x17\xa1\xd0\xed\xd3py\xd2\nn\xf0i\xc1z\xedo9\xec\x19e\xde\xf2\xd9y[\xd9+\xdf\xd47\xb2\xb4ؔ\x17F\xc9\xfbΝ\xb0(\x1a\xd8\xec'^\x8f-l\xacŨ\xe8n\xe6\u007f\x1d\xefޝ\xfd>\x94W\x1b\xbc\xa3\xfa\xd3\x04\xbf\xfd\xdeԪ\xe8\x1e\xa6\xbf\x81l\xfc'\x00\x00\xff\xff\x8c\xdb\x1fܮ\t\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4WM\x8f\xdc6\f\xbd\xfbW\x10\xe9!-\x10{\x12\xf4\xd0bn\xed&\x87\xa0i\x10\xec\xa6{)z\xd0\xc8\x1c[]YREj6ۢ\xff\xbd\xa0dχdz\xbb9T7K\x14\xf5D>>\xd1U]ו\n\xe6\x16#\x19\xef֠\x82\xc1/\x8cN\xbe\xa8\xb9\xfb\x91\x1a\xe3W\xbb7՝q\xed\x1a\xae\x12\xb1\x1f\xae\x91|\x8a\x1a\xdf\xe2\xd68\xc3ƻj@V\xadb\xb5\xae\x00\x94s\x9e\x95L\x93|\x02h\xef8zk1\xd6\x1d\xba\xe6.mp\x93\x8cm1f\xe7\xd3ѻ\xd7\xcd\x0f\xcd\xeb\n@G\xcc\xdb?\x9b\x01\x89\xd5\x10\xd6\xe0\x92\xb5\x15\x80S\x03\xae\xa1\xf5\xf7\xcez\xd5F\xfc+!15;\xb4\x18}c|E\x01\xb5\x1c\xdaE\x9f\xc2\x1a\x0e\ve\xef\b\xa8\\\xe6\xed\xe8溸\xc9+\xd6\x10\xff\xb2\xb4\xfa\xc1\x8c\x16\xc1\xa6\xa8\xec9\x88\xbcH\xc6uɪx\xb6\\\x01\x90\xf6\x01\xd7\xf0Q`\x04\xa5\xb1\xad\x00ƻgX\xf5x\xbbݛ\xe2J\xf78\xa8\x82\x17\xc0\at?}z\u007f\xfb\xfd\xcd\xc94@\x8b\xa4\xa3\t\x9c#8\xc3\f\x86@\xc1\x88\x00\xd8\xefA\x81r\xa0\"\x9b\xad\xd2\f\xdb\xe8\a\xd8(}\x97\xc2\xde+\x80\xdf\xfc\x89\x9a\x81\xd8G\xd5\xe1+\xa0\xa4{P⯘\x82\xf5\x1dl\x8d\xc5f\xbf)D\x1f0\xb2\x99\xa2\\\xc6\x11\xb9\x8efg\xc0_\xca݊\x15\xb4\xc2*$\xe0\x1e\xa7\xf8`;\x86\x03\xfc\x16\xb87\x04\x11CDBWxv\xe2\x18\xc4H\xb9\xf1\x06\r\xdc`\x147@\xbdO\xb6\x152\xee02DԾs\xe6\xef\xbdo\x92\bɡV\xf1D\x87\xc30\x8e1:ea\xa7l\xc2W\xa0\\\v\x83z\x80\x889N\xc9\x1d\xf9\xcb&\xd4\xc0\xaf>\"\x18\xb7\xf5k\xe8\x99\x03\xadW\xab\xce\xf0TT\xda\x0fCr\x86\x1fV\xb9>\xcc&\xb1\x8f\xb4jq\x87vE\xa6\xabUԽaԜ\"\xaeT0u\x86\xeera5C\xfbM\x1cː^\x9e`\xe5\a\xa1\x19q4\xae;ZȜ\u007f$\x03\xc2\xfaB\x98\xb2\xb5\xdc\xe2\x10h\x99\x92\xe8\\\xbf\xbb\xf9\f\xd3\xd19\x19\xf3\xe8\x17\xe6\xec7\xd2!\x05\x120\xe3\xb6\x18K\x123\xf3\xc4'\xba6x\xe38\u007fhk\xd0\xcd\xc3Oi3\x18\xa6\x89̒\xab\x06\xae\xb2\xd2\xc0\x06!\x85V1\xb6\r\xbcwp\xa5\x06\xb4W\x8a\xf0\u007fO\x80D\x9aj\t\xec\xf3Rp,\x92s\xe3\x12\xb5\xa3\x85I\xc9.\xe4kV\xea7\x01\xb5dO\x02(;\xcd\xd6\xe8\\\x1a\xb0\xf5\x11ԡ\xf2\xc7\x006'\x9e\x97+7\x83S\xb1C\x9e\xcfΰ|\xceFr\xfc}\xafN\x85\xe6[l\xbaF\xb4\x82F E=\xbek\xce<^\xc6\x00\x8b\xec]D2\x91X\xc2 q\x15)\x10\x91:\xc6t~\xb4\ftiX>\xa0\x86\x9f3\xe6\x0f\xbe{t\xfd\xca;\x16\xba?jt\xebm\x1a\xf0Ʃ@\xbd\u007f\xc2\xf6=\xe3\xf0<\xcb\xe9A\xde?R\xe7\x86\xd7(R\x8e\x97/1\x1a\\#%{\xe1\xb8\v\xb4\x9eF~\xbe\x9eΑ<\x80S\x8edK\xd1t\x04i\v\xa2CF:\xc8˽\xe1~\xd1#\xc0}ot\x9f7\xe6\x04\x8br\x11ym\xb2\x0e|=|\xa9\v\x13q\x81du&\xdf´\x80?\x9b\xbeP͗\x0e\xa8\xc7\n{\x96\"\xb0\xe2D_\xa1\t\xd9~\n\xb5N1\xa2\xe3\xd1K~#\xe7\x1b\x9e+\nS%\xfdv\xfd\xe1\tex{\xb0\xcc]\xa02\xae\xa0\t\x11k2\x9d\xbc\xec\xb2&ڐk\xf6<\x18e\x9cv\x1a\xa7\x81Z\xcc(~\t&f\x05|\x02\u2efda\x110t\xe5q\x9a\xf7R\xd9!R~\xf8\xb5\x9a\xb7\x1c26\b-Zdla\xf3P\x94\xf8\x81\x18\x87s\xdc[\x1f\a\xc5k\x90G\xabf\xb3@#\xe9w\xd5\xc6\xe2\x1a8\xa6K,[\xbcx\xe8\x15-\x94\xe1ɝ?\x89\xcd\x121\xf6\xc5\xf8(3\xe0\xa2^\xd6\xf0\x11\xef\x17f?E\xaf\x91\b\xcf\xcb\xe8\xe2M\x16\x8b\xe0l\x92\xa4\xb3h\x8f\xa246\xac\xc73i\xb3\xef\x94&\xc4c)\xc1?\xffV\x87\xaaRZc`l?\xce\u007f\x14^\xbc8\xe9\xfc\xf3\xa7\xf6\xae5\xe5\x1f\a~\xff\xa3*\ac{;5\xf42\xf9_\x00\x00\x00\xff\xff\xcbT\xc3P]\r\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4Y_o\x1b\xb9\x11\u007fק\x18\xf8\x1e\xdc\x03\xbc\xd2ݵh\v\xbd]\xec^\xe1\xf6\xce1\"7/A\x1eF\xcb\xd9]\xd6\\\x92%\xb9RԢ߽\x18\x92\xab?\xab\x95d\x1bH\xba/\x89\xc9\xe1p\xfe\xcfo\xa8IQ\x14\x13\xb4\xf2#9/\x8d\x9e\x03ZI_\x02i\xfe\xcbO\x9f\xff\xec\xa7\xd2\xccV?N\x9e\xa5\x16s\xb8\xed|0\xed\a\xf2\xa6s%\xddQ%\xb5\f\xd2\xe8IK\x01\x05\x06\x9cO\x00Pk\x13\x90\x97=\xff\tP\x1a\x1d\x9cQ\x8a\\Q\x93\x9e>wKZvR\tr\x91y\u007f\xf5\xea\x87韦?L\x00JG\xf1\xf8\x93l\xc9\al\xed\x1ct\xa7\xd4\x04@cKs\xb0F\xac\x8c\xeaZZb\xf9\xdcY?]\x91\"g\xa6\xd2L\xbc\xa5\x92/\xad\x9d\xe9\xec\x1cv\x1b\xe9l\x16()\xf3h\xc4\xc7\xc8\xe6]d\x13w\x94\xf4\xe1\xefc\xbb\xbfJ\x1f\"\x85U\x9dCu,D\xdc\xf4RםBw\xb4=\x01\xf0\xa5\xb14\x87\a\x16\xc3bIb\x02\x90u\x8fb\x15Y\xbbՏ\x89U\xd9P\x8bI^\x00cI\xff\xfcx\xff\xf1\xf7\x8b\x83e\x00\xeb\x8c%\x17d\xafZ\xfa\xf6<\xba\xb7\n ȗN\xda\x10\xed}\xcd\f\x13\x15\bv%y\b\r\xf5B\x91\xc82\x80\xa9 4҃#\xebȓN\xce=`\fL\x84\x1a\xcc\xf2\x9fT\x86),\xc81\x1b\xf0\x8d\xe9\x94\xe0\bX\x91\v\xe0\xa84\xb5\x96\xff\xde\xf2\xf6\x10L\xbcTa\xa0l\xe1\xdd'u \xa7Q\xc1\nUG7\x80Z@\x8b\x1bpķ@\xa7\xf7\xf8E\x12?\x85ߌ#\x90\xba2shB\xb0~>\x9b\xd52\xf4\x91\\\x9a\xb6\xed\xb4\f\x9bY\fJ\xb9\xec\x82q~&hEj\xe6e]\xa0+\x1b\x19\xa8\f\x9d\xa3\x19ZYD\xd1u\x8c\xe6i+\xbes9\xf6\xfd\xf5\x81\xacaþ\xf5\xc1I]\xefm\xc4@;\xe3\x01\x0e5\x90\x1e0\x1fMZ\xec\f\xcdKl\x9d\x0f\u007fY[S\xe1\x87`F$[&\x17\x90\x80a@\xc0٠\x803U}T\xe2\x9f\x1f\xef\xfbJ\xde\x1b1\xcb\x1e\x8e\xef\xbd`\x1f\xfe*IJ\xbc\x9ft;!S\xe69Y\xd7\xe4\"p\x19\xfbbS\xe1R\xfd=\x18\xc7\x16\xd0f\x8fEd\xcc\xdeK\x85\x92đП~\xfa|R\xe2C{\x81Ԃ\xbe\xc0O u\xb2\x8d5\xe2\xfb)<\xc5\xe8\xd8\xe8\x80_\xf8\xa6\xb21\x9eNY\xd6h\xb5a\x9d\x1b\\\x11x\xd3\x12\xacI\xa9\"\xe1 \x01kܰ\x15z\xc7q\xbc!Xt\xe1l\xb4\xf6\xe8\xe7\xe9\xfd\xdd\xfby\x92\x8c\x03\xaa\x8e\x95\x98\xbbf%\x19\xcd0\x8cI\xbd8F\xe3Q3\xef?ߥ\xf0\t\x06\xca\x06uMI_\x82\xaa\xe3\xee8\xbd~K\x1e\x1fC\x92\xfe\x1b\x81&\xc3\xc2\xf1\u007fk\xee/T.\"\xe8\x17(\xf7\xb0\x17\xe5g\x95\xe3Y\xc5i\n\x14\xf5\x13\xa6\xf4\xacZI6\xf8\x99Y\x91[IZ\xcf\xd6\xc6=K]\x17\x1c\x9aE\x8a\x01?\x8b\xe3\xc6\xec\xbb\xf8ϛu\x89\x83\xc2K\x15\x8a\xc4\xdfB+\xbe\xc7\xcfޤT\x8fa_\xdeǮ\x17\x19Y\r\xcfrZ\xac\x1bY6\xfdp\x92k\xec\x89d\x92\x8c\x84E*ͨ7_=\x94٠\x9dc\x896E\x1e\x80\vԂ\xff\xef\xa5\x0f\xbc\xfe&\vv\xf2E\xe9\xfb\x8f\xfb\xbbo\x13\xe0\x9d|S\xae\x9e\x00\xe0)F\xac\xb9\x17l\xcaJ\x92\xbb\x00\xcc>\x1c\x10\xf7\xd0q\x04\xb1ni^\x85\f\x03\xd6#P\f\x85\x88\xcf\x1e\xa8\x1e\xcf\x02\xb6\xb3\x168P\xe3\tk\x0f\xe8\b\x10Z\xb4\xec\xb9g\xda\x14\xa9\xc5[\x94ܟ\xb9\x05g̳$@k\x95\x1cmŹ\x91g\x10\x9a\xf1>\x0f\xdaX\xfbS\xba\x8f\xfa!q\xb8`\xff4\xe0\x8cA\xf6,@\xc27[\xd8\x1e\f,\xc7R\xf4\f(>iE\x9eK\x19\xad\x1d\x8aX\x8c\x0fP\x03\x1a\x1e(\x06Kֈ\xc1\xcaa$\x0e6\x93~/\x9a*\x03\x86οb\xae\x8c\xf4\xbdMS\x15\t\x99K\x84\xd0o\x9d,K\xc3\xe8\xf4\xf0i\xed\xbc{o\x8fO\xc4G\x1c'\x92pA\xb6\x1c\xb39\xca\xd6\xe8\xfb;\xc6FC\xd8c\x97Nƺ\xcd\xdcHD\xe8\xc8ȶB\xa9H@\xff\xb67<3\xc2u\x9f˒*.r\x9dU\x06E?\x90e\xf1\xb6\xf0\x8c\xe7\xf5\xf8:r\xed\xcf\xf0\xec<\x898ɏ\x18\xe1\x18\xb2UƵ\x18\xe6 0P1\xcaTwJ\xe1R\xd1\x1c\x82뎷\xcf\x14\x8b\x96\xbc\xc7\xfaR*\xfe\x96\xa8Ҝ\x9a\x8f\x00.M\x17\xb6\x83\xeaAQ\xb8\xf69\xa6^7+\x8f\x8e\x80\x87\xe1\x8c\f\xd1}\x86\xaaJ\xc53\xfb\x85`\xf7 \x1c\xa5Z\xd2x\xab{KM\x00\xb0\r\xfaK\xa6zd\x9a\xb1\x04\xdbV\xaf\xb3\x19\xc6\x1f\xe9\xae=\xbe\xa5\x80\aZ\x8f\xac\xde\xebGgjG\xfe8p\x8a>\xbeF\xaay\x01\xbf\xc4lx\x95\xfe\xf9\xa2K&\xc8d\xd0\x18\xd5'\xb3\t\xa8@w\xed\x92\x1c\xdba\xb9\t\xe4\x0f\xcb\xf9ثD\x9cfvf\xdc;\xdf\xfb/q\xca\x03Z\x89:\xbe\x1erv\x05\x03Bz\xabp3¸W$\"\x16N..\x01\xbbx\xee\x93ڒ\x8b[\xaf}M\x892\xdd\x19}\x02_\xf7\xf9,u\xf8\xe3\x1f\xce\xe0\x1b\xa9\x03Ճ\xe6\x90\xf7ٜ\xef\xf8\x96\xafsÙ\xd6\xed5Zߘp\u007fw!\n\x16[\xc2>\x1bv@)־\xf8\xb6\x99\x89r(\x8c\xb9j[[^\x95\xaa>\xa0\v/mE\x8b\x03\xe2\v](r\x1e\xefA\v\xb2\xe88\xd3\xe3K\xf8\xed\xf0\xb7\xa6\x1b\xf02>\xef1\xdeJ\x00,\rߞ\x9b\x13\x03K\xe3h\xa4d\xc2q[9h\"\x87\xe2\u007f\xcb\xfe1\x1a'G\x8bQr\xb1\xc7;?\x11\xe7\x95\x1d\x86\xc1\x92\xa7\x03\x12\x0f\xc3\xdfӮ\xd2\xebM\xff\x03Y\xfc\xb34:Ae?\x87O\x9f'\x90\x9f\x8d?\xf6\xbf{\xf1\xe2\xff\x02\x00\x00\xff\xffTTw\xa4\x84\x1c\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4YKs\xe3\xb8\x11\xbe\xebWty\x0f\xceV\r\xa9\xddI*I\xe9\xb6kgSJv=\xae\x913\x97\xa99@DS\xec\x98\x04\x18\xa0)YI忧\x1a \xf4\xa4\x1ev\xd5Lx\xb1\x85G\xe3\xeb\xaf\x1f\xe8&GY\x96\x8dTK\x9f\xd0y\xb2f\x02\xaa%|a4\xf2\xcb\xe7\xcf\u007f\xf69\xd9\xf1\xf2\xc7\xd13\x19=\x81\xbbγm>\xa2\xb7\x9d+\xf0\x1eK2\xc4dͨAVZ\xb1\x9a\x8c\x00\x941\x96\x95\f{\xf9\tPX\xc3\xce\xd65\xbal\x81&\u007f\xee\xe68\xef\xa8\xd6\xe8\x82\xf0t\xf4\xf2\x87\xfcO\xf9\x0f#\x80\xc2a\xd8\xfeD\rzVM;\x01\xd3\xd5\xf5\b\xc0\xa8\x06'\xd0Z\xbd\xb4uנC\xcf֡ϗX\xa3\xb39ّo\xb1\x90S\x17\xcev\xed\x04\xb6\x13qs\x8f(j\xf3h\xf5\xa7 \xe7c\x94\x13\xa6j\xf2\xfc\xf7\xc1\xe9_\xc9sX\xd2֝S\xf5\x00\x8e0\xeb\xc9,\xbaZ\xb9\xe3\xf9\x11\x80/l\x8b\x13x\x10(\xad*P\x8f\x00z\x02\x02\xb4\xacWq\xf9c\x94UTب\x88\x19\xc0\xb6h~z\x9c~\xfa\xfdlo\x18\xa0u\xb6EǔԋώYwF\x014\xfa\xc2Qˁ\xf4[\x11\x18W\x81\x16{\xa2\a\xae0\x81B\xddc\x00[\x02W\xe4\xc1a\xebУ\x89\x16\xde\x13\f\xb2H\x19\xb0\xf3\u007fb\xc19\xccЉ\x18\xf0\x95\xedj-n\xb0D\xc7ర\vC\xff\xde\xc8\xf6\xc06\x1cZ+ƞ\xe3\xedC\x86\xd1\x19U\xc3R\xd5\x1d\xbe\x03e44j\r\x0e\xe5\x14\xe8̎\xbc\xb0\xc4\xe7\xf0\x9bu\bdJ;\x81\x8a\xb9\xf5\x93\xf1xA\x9cܹ\xb0M\xd3\x19\xe2\xf58x&\xcd;\xb6Ώ5.\xb1\x1e{Zd\xca\x15\x151\x16\xdc9\x1c\xab\x96\xb2\x00\xdd\x04\x97\xce\x1b\xfd\x9d\xeb\x03\xc0\xdf\xeea\xe5\xb5\xd8ֳ#\xb3ؙ\b\xcev\xc6\x02\xe2m@\x1eT\xbf5j\xb1%Z\x86\x84\x9d\x8f\u007f\x99=A::\x18\xe3\x90\xfd\xc0\xfbv\xa3ߚ@\b#S\xa2\x8bF,\x9dm\x82L4\xba\xb5d8\xfc(jBsH\xbf\xef\xe6\r\xb1\xd8\xfd_\x1dz\x16[\xe5p\x17b\x1c\xe6\b]\xab\x15\xa3\xceaj\xe0N5X\xdf)\x8f_\xdd\x00´τ\xd8\xebL\xb0\x9b\x9e\x0e\x17G\xd6v&R\n9a\xafô0k\xb1\x10\xf3\t\x83\xb2\x95J*Bl@i\x1d\xa8\xa3\xf5\xf9\x9e\xe8\xe1Еg\xae\x8a箝\xb1uj\x81\xbf\xda(\xf3p\xd1\x01\xb6\x9f\x87\xf6$p\x92Yb\x18c/\x1c|\\y$\x14\xa0N\x9bW\x15:\f{$\x8bQ!\xeee=\xb1uk\x11\x1cT\xd2\xf9\x91\x84\x13\x86\b*[}A\x8dG\xdb\a\x84\xc3\x12\x1d\x1aq\xf7\x98!Z\x1b\xf2\b+2),b\x8a\x05\xb6\x03Z\xcc#\xeaa\x88\xa7\xa9\x873\xd9s\x10\xf0O\x8fӔ1\x13\xc3=t>>\xf7\x02=\U00094135~T\\]q\xf6\xed\xb4\x8c\x87\x85\xdc\xc1\x16\x14\xb4\x84\x05\xee%c \xe3\x19\x95\x06[\x0eJ\x94[\x1b$\xc0\x1c\xf6;\xde\xc5Lѧ\xa4m\n\x17\xeaAI\x8e\"\r\u007f\x9b}x\x18\xffu\x88\xf9\x8d\x16\xa0\x8a\x02\xbd\bR\x8c\r\x1a~\a\xbe+*P^\xd4 \x87z&3y\xa3\f\x95\xe89\xef\xcf@\xe7?\xbf\xff2\xcc\x1e\xc0/\xd6\x01\xbe\xa8\xa6\xad\xf1\x1dPd|\x93\xfe\x92ϐ\x8ftl$\u008a\xb8\xa2\xc3KkÀxW\xaf\xf6*\xa8\xcb\xea\x19\xc1\xf6\xeav\b5=\xe3\x04n$\xcaw`\xfeG\x02\xeb\xbf7'\xa4\xfe.\x06Ѝ,\xba\x89\xe06\xf7\xddnDnAr\xa5\x18\xd8\xd1b\x81.\x14\bCOHޒ\x12\xbf\a\xeb\x84\x01cwD\x04\xc1b\xbd\x98\x8fP\x1f\x81\xfe\xfc\xfe\xcbI\xc4\xfb|\x01\x19\x8d/\xf0\x1e\xc8DnZ\xab\xbf\xcf\xe1)x\xc7ڰz\x91\x93\x8a\xcaz<Ŭ5\xf5Zt\xae\xd4\x12\xc1\xdb\x06a\x85u\x9d\xc5zC\xc3J\xad\x85\x85d8\xf17\x05\xadr|\xd6[S\x95\xf1\xf4\xe1\xfe\xc3$\"\x13\x87Z\x84|'\xb7SIR5H\xb9\x10\xef\xbc\xe0\x8dG\x97fz|\x17݇-\x14\x952\v\x8c\xfa\"\x94\x9d\xdcB\xf9\xed[\xe2\xf8\xf8\xeaO\xcf@\tp\x988\xfeo\x97\xe8\x95ʅJ\xf5\n\xe5\x1ev\xbc\xfc\xacr\xd2\x188\x83\x8cA?m\v/\xaa\x15ز\x1f\xdb%\xba%\xe1j\xbc\xb2\xee\x99\xcc\"\x13\xd7̢\x0f\xf8q(\xed\xc7߅?o\xd6%\x14\xe4\xd7*\x14\x16\u007f\v\xad\xe4\x1c?~\x93R\xa9V\xbc\xfe\x1e\xbb\x9d\xf5\x05\xcc\xe1^\t\x8bUEE\x95\x9a\x80>Ǟ\b&\x92\x8aS\xc7Ԭ\xcc\xfa\xab\xbb\xb2\x10\xda9A\xb4\xce\xfan3SF\xcb\xff\x9e<\xcb\xf8\x9b\x18\xec\xe8\xaa\xf0\xfd\xc7\xf4\xfe\xdb8xGo\x8a\xd5\x13\x85n\xf4\x91\xd6N\xb5PY\x12\xba\vu\xd9ǽũ\xae\x1c\xa8\v7k^U\x18z\xa3Z_Y\x9e\xde_\xc01\xdb,L\x18\xb6\x06\xe8\xcb\xc1$K\x1c\xf7l\x15x\x06O\x14u\x01K\xac\xed\x87j\xec\x1eI\xac9\u0088Ե\x01\xcfp\xb0\xbe\x16\xa1\xb4dR@\xed#̆;\x87\x835\xad\xd5\a#\xfb\x9ep0\xb95\xcd\xc1DT\U000aad8a\x15w\xfe5\x8dUؐ\x98\x8d\xf1ͽ\x98Pܾ\xb9\xb5*\xac\x14\x8e\xfb\xaf\x98\xce[\xf9\xeexGx\x8f\xe1tD\xc7\xd4`\xe8W\x02\x0eX)\x9f\x0e\x19\xb2(\xecȋ[CN\x15q\xa8CY'Ug\xa9\xa8F\r\x9b\x97\\\xf0$\x1dfh\xe8o\x87\xaa\x98$\xa8\xf3\xa8C\xef9\x00\xfax_i]\xa3x\x02\xd2\xc6g\"\xe2h\x85\xe9\xeaZ\xcdk\x9c\x00\xbb\xeex\xfaL\x005\xe8\xbdZ\\\x8a\xa0\xdf\xe2\xaa\xd8\xf1\xf5[@\xcdmǛ\x96\xaf\x0f\xa5\x9e\x8a[\xdf{\xc1\xeb\xda\xceJ\xf9KP\x1ee͐\xc7m\x82\xfa\xbc\xcbɃ\xa6k\x8e\x8f\xc9\xe0\x01W\x03\xa3S\xf3\xe8\xec¡?\xb6L\x96\f8\xd0\x04d\xf0K\xf0\x8eW\x11\xd0\x1ft\x89\x83~\x19T\xb6N\xdemY\xd5`\xbaf\x8eN\x88\x98\xaf\x19}b$\xa5\x86\xa1\x1e:\xd4\xde[&\xb7\x12R\xb6\x8b\xa2\xfan\xa2P&\xbcR\x12\xffe\v\x9a|[\xab\xf5\x80ܤI\xb8^\xc5}%\x8e\xb6\x1e\x93\xa2P\xc2?̽\xb6\xf7\x0f\xa0\xee\xad9Q\r\xa6\x90!\xc3\u007f\xfcÙۘ\f\xe3\xe2 \x95\xf6\xf3B\xe8\xcfr\xca\xd79\xe1̅\xefY9\xbe6\xed\xcd\xf6\x16_\xcaxA\xf4p\xbe\xdbM]ljj\xff\x98o\x99\xa3\x06\x89:\x1a\f\xc8\xf5\x8e\xec\xfe\xbdY?\xb2\xbd\xd9T!\xc5\x1c\xea\x87\xc3O\r77{_\x0e\xc2\xcf\xc2\x1aM\xf13\t|\xfe2\x82\xfe]ڧ\xf49@\x06\xff\x17\x00\x00\xff\xffñ\x1b\xae\xa0\x19\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4W\xc1n\xe36\x10\xbd\xfb+\x06\xdb\xc3^*y\x17=\xb4ЭM[ h\x12,\x9cE.E\x0f\x145\xb2\xa7\xa1H\x96\x1c:u\xbf\xbe\x18J\x8aeY\x897\v\xacn&g\x1e\xdf̛\x19ҫ\xa2(V\xca\xd3\x03\x86H\xceV\xa0<ῌV~\xc5\xf2\xf1\xa7X\x92[\xef?\xae\x1e\xc96\x15\\\xa5Ȯ\xdb`t)h\xfc\x15[\xb2\xc4\xe4\xec\xaaCV\x8dbU\xad\x00\x94\xb5\x8e\x95,G\xf9\t\xa0\x9d\xe5\xe0\x8c\xc1Plі\x8f\xa9\xc6:\x91i0d\xf0\xf1\xe8\xfd\x87\xf2\xc7\xf2\xc3\n@\a\xcc\ue7e9\xc3Ȫ\xf3\x15\xd8d\xcc\n\xc0\xaa\x0e+\b\x18\x99t@\xef\"\xb1\v\x84\xb1ܣ\xc1\xe0Jr\xab\xe8Q˱\xdb\xe0\x92\xaf\xe0\xb8\xd1{\x0f\x94\xfap6\x19h3\x02\x1d\xf2\x96\xa1\xc8\u007f,n\xdfP\xe4l\xe2M\n\xca,\x11\xc9ۑ\xec6\x19\x15\xce\f䀨\x9d\xc7\n\ue10bW\x1a\x9b\x15\xc0\x90\x82̭\x18\x82\xdc\u007f\xec\xb1\xf4\x0e;Փ\x06p\x1e\xedϟ\xae\x1f~\xb8?Y\x06\xf0\xc1y\fLc|\xfd7\x11v\xb2\n\xd0`ԁ<紿\x17\xc0\xde\n\x1aQ\x14#\xf0\x0eGR\xd8\f\x1c\xc0\xb5\xc0;\x8a\x10\xd0\a\x8ch{\x8dO\x80A\x8c\x94\x05W\xff\x8d\x9aK\xb8\xc7 0\x10w.\x99F\na\x8f\x81!\xa0v[K\xff=cG`\x97\x0f5\x8aqH\xf2\xf1#\xcb\x18\xac2\xb0W&\xe1\xf7\xa0l\x03\x9d:@@9\x05\x92\x9d\xe0e\x93X\u00ad\v\bd[W\xc1\x8e\xd9\xc7j\xbd\xde\x12\x8f\x05\xad]\xd7%K|X\xe7ڤ:\xb1\vq\xdd\xe0\x1e\xcd:ҶPA\xef\x88Qs\n\xb8V\x9e\x8aL\xdd\xe6\xa2.\xbb\xe6\xbb0\xb4@|\u007f\u0095\x0f\xa2m\xe4@v;\xd9\xc8\xd5\xf6\x8a\x02Rn@\x11\xd4\xe0\xdaGqL\xb4,Iv6\xbf\xdd\u007f\x86\xf1\xe8,\xc6<\xfb9\xefG\xc7x\x94@\x12F\xb6\xc5Ћ\xd8\x06\xd7eL\xb4\x8dwd9\xffІ\xd0\xce\xd3\x1fS\xdd\x11\x8b\xee\xff$\x8c,Z\x95p\x95\xbb\x1cj\x84\xe4\x1b\xc5ؔpm\xe1Juh\xaeT\xc4o.\x80d:\x16\x92\xd8/\x93`:\xa0\xe6\xc6}\xd6&\x1b\xe3\fyA\xaf\xf9\\\xb8\xf7\xa8E>ɠ\xb8RK:\xf7\x06\xb4.\x80:\xb3/O\xa0\x97[W\xbeZ\xe9\xc7\xe4\xef\xd9\x05\xb5\xc5\x1b\xd7c\u038df\xdc~Y\xf2\x19\xc9\xc9d\xe9\xdb\x18\x97\rϰ\x01x\xa7xҿ\xac\xc8>\x8f\x81\xc5x^\x11!\v\xa1\xa4\x9d\xad\xb2\x1a\u007f\xcf\x15e\xf5\xe1BL\xb7\v.\x12\xd2\xce=\x81k\x19\xed\x14t\xe0\xba\x10I\x8d\x10\x92}\x13\xd9~~_7Rx-a\xb8@t33\x1f\xf3\xde&c\x06\xacB\xbb\xce+\xa6\xda\xe0\xf2\x91\xf2I\xd9P\x8fr\xe8{\xff\xeb\xf3\xbdw&u\xf8|\xdd\\\x88\xe0\xe1\xd4zZ8\xfd\xc2@EB\x81pzq\x9e~C\xadD\xf0\xae\x19H\f\x05\x1d%\xbe7\xc4 \x92S\xc0\xd9\x04-\x96\xdbcf\xb3Tm3\x93\xb9Ƴ\xedY\xfe\xbeh|\xb0\xe2\x14\xdf2@\xb2Øl\x9dB@\xcb\x03L\xbeQ\xbfz\x84\x18\x15y\xd2>\xf2\xa2\xbaP\x017\xe7\x1e#1\x01\x03\x96\x85i\xbf=\xa9\xf9-\x94E[\xea\xb4օNq\x05ra\x14\x02tf!\xef-\xacnP5\xe7}\\\xc0\x9d\xe3\xe5\xad\x17#\\슳\xc5(\xef\x92f\xa2s\xec\x1byX9\xf6\x90\xd2\x1a=cs7\u007f\xbd\xbf{w\xf2\x18\xcf?\xb5\xb3\r\xf5\u007f=\xe0ϿV=*6\x0f\xe3\x03[\x16\xff\x0f\x00\x00\xff\xff\x83\xf9\xd9\xe0\xf4\f\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec\xbd}s#\xb7\x910\xfe\xbf?\x05Jq\xfd$]Dʛ\\\xf2\xbb\xdbJ=)eWvT\xf6jU+e\xfd\xa4\x1c\x9f\x0f\x9ci\x928\x81\xc0\x04\xc0P\xe2\x9d\xef\xbb?\x85\x060/\xe4\x90\"0Ծ\u0603\xab\xbax\xa9\x99\x1e\xa0\xd1\xe8wtӂ\xbd\a\xa5\x99\x14/\t-\x18<\x1a\x10\xf6_z|\xffoz\xcc\xe4\xf9\xf2\xc5\x17\xf7L\xe4/ɫR\x1b\xb9x\aZ\x96*\x83\xd70e\x82\x19&\xc5\x17\v04\xa7\x86\xbe\xfc\x82\x10*\x844\xd4\xfe\xac\xed?\tɤ0Jr\x0ej4\x031\xbe/'0)\x19\xcfA!\xf0\xf0\xe9\xe5W\xe3\xff\u007f\xfc\xd5\x17\x84d\n\xf0\xf5;\xb6\x00m\xe8\xa2xID\xc9\xf9\x17\x84\b\xba\x80\x97D\x816R\x81\x1e/\x81\x83\x92c&\xbf\xd0\x05d\xf6c3%\xcb\xe2%\xa9\xff\xe0\xde\xf1\x13q\x8bx\xe7^\xc7_8\xd3\xe6\xdb\xe6\xaf\xdf1m\xf0/\x05/\x15\xe5\xf5\xc7\xf0G\xcdĬ\xe4TU?\u007fA\x88\xced\x01/ɵ\xfdLA3ȿ į\t?;\xf2\xb3^\xbep \xb29,\xa8\x9b\x0f!\xb2\x00qqs\xf5\xfe\xf7\xb7\xad\x9f\t\xc9Ag\x8a\x15\x061\xe3\xe7F\x98&\x94\xbcǵ\xd9\t\xe0&\x103\xa7\x86((\x14h\x10F\x133\aB\x8b\x82\xb3\f\x91XA$DN\xab\xb74\x99*\xb9\xa8\xa1Mhv_\x16\xc4HB\x89\xa1j\x06\x86|[N@\t0\xa0I\xc6Km@\x8d+X\x85\x92\x05(\xc3\x02b\xddh\xd0Q\xe3\u05f5\xb5\x1c\xdb座Hn\t\bܔ=\xca \xf7\x18\xb2\xb35s\xa6륭/\xc7/\x89\n\"'\xff\x05\x99\x19\x93[P\x16\f\xd1sY\xf2\xdc\xd2\xdd\x12\x94EN&g\x82\xfdw\x05[ۅڏrj\xc0\xefw=\x980\xa0\x04\xe5dIy\tg\x84\x8a\x9c,\xe8\x8a(\xb0_!\xa5h\xc0\xc3G\xf4\x98\xbc\xc1\xed\x11S\xf9\x92̍)\xf4\xcb\xf3\xf3\x193\xe1\xfcdr\xb1(\x053\xabs<\nlR\x1a\xa9\xf4y\x0eK\xe0\xe7\x9a\xcdFTesf 3\xa5\x82sZ\xb0\x11N]\xe0\x19\x1a/\xf2\xdfT\xdbvܚ\xabYY\xca\xd3F11k\xfc\x01\xc9|\xc7\x0eX\x82w\xb4\xe4^u\xab\xa8\x11m\u007f\xb2\xd8ywy{פ3\xa6ױ\x8fxo\x10_\xbd\x05\x16aLLA\xb9MDj\xb30A\xe4\x85d\xc2\xe0?2\xce@\xac\xa3_\x97\x93\x053v\xdf\xffY\x82\xb6\x04-\xc7\xe4\x152\x152\x01R\x1695\x90\x8fɕ \xaf\xe8\x02\xf8+\xaa\xe1\xd97\xc0bZ\x8f,b\xf7ۂ&?\\\u007f\xd8a\xad\xf1\x87\xc0\xbc\xb6\xec\x97?\xfd\xb7\x05d\xad\x13c_cS\u007f\xcc\xc9T\xaa\x16s\xb0\xaf\x8c[@\xbb\x0f\xad\x1d\xee\xf4[\x0e\xb6\xfe\x97\xb5\xa9\xfc\xa5z\xd0ҏ\x9dD)\xd8?K@\x16\xe7N,l\xb0\x94\r\x90$\xcc\x0f\xc9b\xbc\xf1\xf7-8\xb5\x03\x1e3^\xe6\x90W\xdcvc-k3\xbe\xdcx\x01\xc5\x11e\xc2ҿe\xffvڢ\xfe\xabe\xa7\x1d3\xa6\n\x88\xa5@&\x1c<\xc2\x04.\xb6\x13\xd3v0\x03\x8b\x8e\xc9\xed\\\x1dA9G'\x1c^\x12\xa3J\u0602\x19\xaa\x14]mAL\x90\xcd\xfb\xe2\xa5z\xde3\x04\xce2h\n\n\x87\x1a'd\xa8ڜ\x11\xf9\xa4\xb12\x97\xf2\xfe)L\xfc\xd5>S\xf30\x92\xa1\x8eC&0\xa7K&\x95_\xbb\x17)\x13 \xf0\bYiP\xcco\xc0-\x91\x9dJE\n\xa9\xcdv,l?\x89\xc4\x1d\x8em[\xb8\x13\x85\x1b+\xf3\x8c#l\xb1]h\x8b\x89H\x01v\xae\v{\x12\xebg\x95,ݳ\xebܾ\x81\xf0n\x8c\x90\tՐ\x13\xe9i\xa0\xe4\xa0\xfd\xb7rdO\xf5);\xdb\n\xbaZ\xbc\x93\xbb\x9cN\x80\x13\r\x1c2#\xd5&&\xf7\xc1\xa7\x1b\xfbp\x8e-x\xec\xe0!\x9e\xf7zN\\/l\aHb\x95\x8e\x879\xcb\xe6N$Z\xdaD8$\x97\xa0\xf1\x18Y\xb5m\xb5m\x91䩽\xf7\x1f\xd9u\x90\xea\xf1đZ\x87\xd7u\xb8\xea\xb1\a\xf3\xa9\xc7\x13l\xa8\x8d\xd9Z_\xed\x94#\xf5\xf8e\"6\xf0\xd5\x04\xa2\xbd\xdax\xf5\xb0D\x8b&\x86U}\xaf\xa6\x04\x16\x85Y\x9d\x11f¯OA\xa4\x9c7\xbe\xff\x19oL<\xc5_\xad\xbfyP\x8a߹+OA\xb4\xbbR}\xfe3\xdc\x14\x14\x16\xb7^V\xec\xbd!\xdf5\xdf:#lZmH~F\xa6\x8c\x1bPk;\xd3\xeb\xbc\x1c\x02\x19\xfb\xc8;;\x16\xd4d\xf3\xcbG\xab\xd9\xe8\xda\x1d\xb3'^\xd6_v\nbИۂ\xf9\t\xb8\x04\x8d9\xa6`\xe1\x8c\xc4;\xc4f\xfd\vj\xd7\x17ׯ!߅\x1e\xb2\x1f\xe5m,\xe4bm\xb2\xcdO{\xadw\xdfexէ\xb2 \x9c\xf9\u007fF(\xb9\x87\x95\xd3X\xa8 vs\xa8\xfd\xd0\x16[b\x139\xe8\x87@\"\xbb\x87\x15\x82\xf1\x8e\x85'\xdfޗ\x14ܸ\x87\xd5>\x8f\xad!\xd0\xceɛ{\x0e\x93\xf6\aD\x04\x9a\xa1\xfb#\x8f\xa0\x93(\xf0\xa2\xa7\x17G\xf6g$a\x04\xdc',\xb3ڶ\x863\r7\xf6X\xbb-\xb2\xa7`Ί=\x17\x8a\xbe4\rxZ\x82\x9b\xe8=\xe5,\xaf>\xe4\xe8\xfeJl׆\xdb\xe3Z\x9a+qF.\x1f\x99\xb6S\x139y-A_K\x83\xbf<\v:\xdd\xc4\x13\x90\xe9^\xc4\xe3%\x1c۶xh\xfa\x9b\xf6 n7\xae\x9c[\xa1\xda\x1e\xa6ɕ\xb0\x86\x8b\xc7\az\x0f\xdd\xe7vˇ\xf6X\x94\x1a\x1dJB\x8a\x11\x8a\xcaqח\x1c\xb2\xf7\x04)UkG6\xa7V}\xd4}pO\xb0wV\x92\xb8\xf7\x9d?\x94\xd3\f\xf2`m\xa2\x17\x8f\x1a\x98\xb1\x8c,@\xcdv\t\x8e\xe6(,\u007f\xdfo\n{r]7\")l?\xd1\x1e\x86g\xdd\xf9ӓ\x19ٓ\xbb\xc7Sa\xb3\x9f|t\x8b\xf3n\xfb\xa3O\xaf\bE,\xea\x1fOb\x97\xe69FZ(\xbf\x89\xe0\xf8\x11{\xb1)\xfb\xddĜ\x84\\\xd0\u009e\xdf\xff\xb1b\x0e\t\xfa\u007fIA\x99\xda\xe3\f_`ЄC\xeb]\xef&j~\xc6~\x81ib\xf7wI\xf9\xa6[\xb8cq\xd2\xf2\x16\xe0N\x90\xcb\xe9\x86\xc6rF\x1e\xe6R;\x99:e\xc0\xbb\\6\xed\xc149\xba\x87\xd5\xd1\xd9\x06\x1f8\xba\x12GN\xc0G\xb3\x9bJ[\x90\x82\xaf\xc8\x11\xbe{\xd4G\tړ\x12\xf7zLt:}\xeb\xd1\"\x8b\xa6\xe3\xb7\xf6\xf8z5w\u05ec\xf7\xa2\xc3Bj\xf3\xd7n\x87ݖ\xf9܄7ںi\x87\xdf\xebI\x9d\xdd\xfb\xb0*\xa6j5\xb9\xa9\x01\xe5\x9dx\x8e\xd1\x06\v\xa0\xa7m\xf4\x94\x93\xaer\xd0\xd1\xcaQn\x11\xfc\x04U\xb8\x00\xc0>S\x8c\xd1\x1a-^\"\xf5\xed\xcbdž\x8fўP\xfb\xef\xe6B\x0e\xad\xd5fr\xb1\xa0\xeb!\xaf\xbd\xa6\xfaʽ\x19h\xda\x03r\xbb\xaff%\x9e\xcb\xfdս@C\x18\xecz`f\xce\x04\xa1\xe1\xf8\x83\xf2\x04EI!\x9f\xe6Dn̩&\x13\x00\x11\xd0\xf7$kp\xe3y\xe5\xf5\x82\x89+\xfc\x00yqp\xf9Njt%mg@u\xb5\xa1\xd5\x0f(q\xf6U\x8ddN\x1e栠E\x15\x9b\x0eo\xab1\xee\tRH\xd3\xf4+X\xb8\x85̏5\x992\xa5Ms\xa2\xfb\x12\\\xa9\xf7%\x87\xc8\x1d\xb6\xab\xbbc\v\x90\xa5I\u0603\xcb\xfa\xedV\xb4rA\x1f٢\\\x10\xba\x90\xe5\x1e\xc2\xdd\r+_آ\n)\xfa\x1dx\xa0\xcc \xbb\xb3p\xd1\xc3b\xa4ݥ\x82\x83\xd9w\x8b'0\xb5\xec(\x93B\xb3\x1cT\by\xbb\x9de\xd2\x1e\xdc)e\xbc\xec\n\xdft\x8dX3U\\*\x95d\xa5\xbeuo6\xbc\x86s\xf9\xd0F\xd0\xde(\x98\xd3%\x106%\xcc\x10\x10\x99\xdd\x17P\x8ee\xe3'<2\x105{\x93\xe5~\f\xde\x0e\x10\xe5b?\x04\x8c\xf0d3\xb1\xd3)\xd6|\xfck\xca\xf8sl\x9b\xa5\xbc\xf4\xa3\xf1}\xfd\xf6\a9\x1a\x15S\xd9_\x84M\x80\xbc\x03\x9a\xaf\xc2\xf9\xa0\xc6XS\x15i@\x12U\x8a&G|\x86\x93\x11c\xdf\xf9Y\x1c\xd2pc\x82\xed\xb1\xb1k\xfe|f\x9aڎ\x05\xf1\xacڎ\xfd@%\xe8R\\3W-\x00VT\x06\xc5\x19\xe7^QM\x84\xe63\x01k\xa0B\xee\x9c^V|z=\xda%\xf2l\t\x83w\xae.FuYs\xf3zC\xb3\x91\xfc\x16}\x04\xbc\x83w%K\xf2@\x85\tD_)s\x85ܓ\xeacw\xd5\r\xaaf\x11Ood\xd6\x05\x955\xa4\xb7\x810j\x85\xe9V\xfbN\xda\rk\x9a\xe62\xbb\xb7\xeaȂ\xce\xe0\xf8X\x93Wo^[R\xb1Z\x87\x15\x19\x11\x12\xc1\r\xe6\"\xb1\x85\x92K\x96[\xd5\xe9=U\x8cN\xb85\x82\xa7\xa0@d\xa0ɗ'\xef/\xde\xfdt}\xf1\xe6\xf24\n\xb85\x9dᱠ\xc2\xd2`\xa9\x834\xafv\xdf.\x00Ē)),\x82\xe2\xb0q5%\x94,\xc3l\xb3*\x13͚Z|鵹(\x88Պ\x83#\x84\x89\xa24\xc1;\xfa\xc08'\x938\x88\xa5\xc8\xe6T\xcc,^_\xcb\xd2\xce\xf3\xcb/\x11+\n\xf22\xf3\a3\n\xa2?L_\x9e\xf9p\x16\xe5\\>h\x94-\xa03Zx\x1cG\xc1ll/\xd1+a\xe8\xe3K\xc2\xc60&G_6\xfet\x14\x05\x13\xb1U(i\x97\xe9\xe2\x11\x0e\x8b\x9c\x19P\x94\x93\xa3&不\xbf\xb4넼I\xa0\xf85\x01KP\xce\x0ep$\xb7\xaf\x03\xdf\r\x053\xaar\x0eZ[\x9e\xfb0\a3\a\xe5\xddJ\x9e\xc8 \xc6\xeb\xec\x86T\xf6|ufJֹ\x91Q\x10C\x1e\xe5}\x95\b7T\xdf\xebs&\xacH\x1d\xe5\xd4\xd0Q\x83\xe9\x9e;i8\xf2\xf2y\x14,\xe9Qu\x1c\xcf\u007f\xa3J!\x98\x98\x8dh\xf5\x14\x13#:\xd2s\xe0\xfc8b\x96Q\xe2\u008dh{\xb7\xf9ZL\x80!\xd21\xe1F\x9b\xa3_V\f\xdc}yL\xae\xa5ٕ\x81\xb6}T\"\fq<\xee\xe4\xf1\x97\xd7w\xef\xfe~\xf3\xf6\xea\xfa.\x96\xb57\xc5\xc2vV\x9f\xc6$[b\xa1\x83\xd5\xc7\xed\xfe.\xb1\xd0f\xf5Qp\xb7\x88\x85\rV\x1f\x87\xd8\x0e\xb1\xb0\xc9\xea\xe3X\xf0\xa6X\xd8\xc2\xea\xa3\xc0\xae\x8b\x85\xad\xac>\nj[,lc\xf5Q \xbb\xc5B\a\xab\x8f\x17B\x9bb\xa1\xcd\xea\xe3 n\x17\vk\xac>\nl\xb7X\x18X}\xc7kq\xac\x1e\xc42\x99\xcd\u007f\xe7ͯ\x06+\xaa\xf6<\x8e\x0e\x8dČ\x03&\xda|\xaeK+x^̷]\x82b\xf9\x9e\xb6\xd3*Ds\xb1Q\x90I}\x1cB\xc66\x8a\xb5ʢ\x8dc1)V\x9a\x1bOEκ\xc7f<\xcdߚH\xc7\ai\xe0dL\xde\xf8\f\x03J^\xfdt\xf5\xfa\xf2\xfa\xee\xea\xeb\xab\xcbwqH!\xe9g\x87\x84\xa4\x91\x9e\xa89\xee0\x0f\x13\xf0\xb2[s\x88\x16\xc8n\x14\n\x96L\x96\x9a\xaf\xaa\xf4\xf6\xfeG\u05cd\xf5\x93\xebS\xcaVD\x83Z\xb2,e\xb6\x9dS\xeb\xa3\xea\xb8\xf1\xa4\u0093\xbe\xfa'Ԟ\x04\xc0\xdbmb\xaf\xfc\xa4\x90\xd6!-c\x0f\xf2\x19\xecc7\x9e\xb0\x92\x13 \x1eV\x81j\xccr\xa7\x1a\x95\x00t\xb7\x8dM\xf6N\\l\x0eT\xbf^Ô\x96\xdcyێ\x8e\xc61\xba\x8c\x1b}Y\xec\xd7J\xee\x19@i\x8e\x16\x9b\xbdu\x97\xbcB\xc4\xe00B\xe8\xd8'ƶ\xd4\x0e\x1di\xaf\xba\xc1|\xeed\xb0)\xa3\xf2\xe6\xea\x91.\xe5\x89\vIO\xd9\xec\r-\xbe\x85\xd5;\x98\xa6\x80XG;\xe6\xcc\xfa\xf4\xd2XӠ\x1e\xa8\xf5\xb8\xa9\xa5pžx!1\x19\xc5]\xa3\x85\x93;\x9f\xfd\x8c:\xacEOڒH\xbf\x83\x15F\x9av\x17F[\x95i\xa8y\xc9\x10+\u007f\x88\xd9\xd7pˤȠ0\xfa\\.\xad\xee\x00\x0f\xe7\x0fR\xdd[K쁙\xf9\xc8\xc5\xc3\xf49^\xc39\xff\r\xfeO\x8f\xd9ݽ}\xfd\xf6%\xb9\xc8s\"\x91Ֆ\x1a\xa6%wiw{g\xfav\x8d\xba\xa8\xc0\x19^l?#%\xcb\xff\x1c\xcfl\xc38\x00m\xc8\xc2eb\x1e\x88>n1\x92\xbf\nR\xaa\a\xae,\v\xaf8\x02\x91\n\xc3o\xfb\xa4\xc1n\x1f!a\xd9+\xba=\xd1>\x91\x92\x03]/\xe2\xb0\xdf\xd8?4\xdc5\xf6K\a\xee\x1a\x11\xe1㮁'\xe00R\xe3\xb8\x16\x1b\xfb\xa5\xb3v\x0fop\x162\u007fItY\x14R\x19]\x15,\x18[F\x10\x17ϨG\v\b\xde\xed;#\xffY\xfd\x88wG\xf4\x0f\xc7\xc7\u007f\xfa\xf6\xf2\xef\xff\xe7\xf8\xf8\xc7\xffL\xfdN\r\xb3Qk\xe6\x10\x80u\x01\xd9X\xc8\x1c,\xcb>s\xff\xf4\x96\xd7E\x86\t2\xd7=У\r5\xa5\x1eϥ6W7gៅ̯nz\x82D\x18:A\x05%\aQ\x02\xb6\x15~\x89\x19-J\x0f\x05bz\vM_K\xc6\xd2\xfb\xd7\xf6\xc8\xdcP3\xdf?Ůk<(f\f\b\xb4UA-4\x91\xd33\xcb\x1d\xd1\x14\xe8û%9Z\xbe\x88\x8cP\xb6\x00\xf4\x17lӀ\xa2\x03m#b۳\x9b>\x1c\x8b\x04צe\u007f\xc1KPeS\xf6\x00zqs\x15\n\x0f}D\xc4\xf7\x95lն}\f\xf9\x16\x12ο~\x169\x17\xa0\xf7\x13uuN\xb1\xbb\x83\xb1\xdfM\xde\xed\x833\xacPDE^W):q?\x8e\xb3\xa2Le\xe6\x1e\xc2\x02\x16R\xad\xce\xc2?\xa1\x98\xc3\x02\x14\xe5#m\xa4\xa2\xb3d\xf1\x13\xa6\x8aS\xac\xff\xe5>\x97\xca\xf9\x1b(\u061ci\\\x12O\x03\xaa\x02\x92\x95\xcaZ;|\x15t\x14\xc8?\x9a|\xab觻DҾ\xa3M\xe4u\xb2z?[\xb3\xe6\x1f\xe8\xc6YJ^.@\x9fUVJ\x0f\xc0\xe8\xd2\x14K\xb2\xa4J\u007fT\x8b+gK\xa6\xf7M\x97\xee\x1aT\xac\xde&\xb2&\x82<\xd6-\x82\t\x03\xb3\x1e&\xda\xe8\x10\xc8\xe84\x1f}\xf9\x88\x1e\x9b-KS\x94\x98<\xbc\xa0\xa6\x8aJ=\x162\xcds\x17F\xa3\xa0O\xdet\x98\xbeHqc\xbbQPc@\x89\x97\xe4?N\xfe\xf1۟G\xa7\u007f>9\xf9\xe1\xabѿ\xff\xf8ۓ\u007f\x8c\xf1?\xfe\xe5\xf4ϧ?\x87\u007f\xfc\xf6\xf4\xf4\xe4\xe4\x87o\xdf|sws\xf9#;\xfd\xf9\aQ.\xeeݿ~>\xf9\x01.\u007f\xdc\x13\xc8\xe9響L\x9e\xf2\xe3\xa8\xf6Ќ\x980#\xa9F\x8e\b\x9e,\xf6\xb0k\x04\xe4\x1e\x8a'\xbd\v\x9aH_\xc1K\x9a\x1a\xdbGd\x1d}U\xab^h\xe8\xa9Yi\xc8\x14\x98O\xcf\xe7\xec\xe6\x15\xd4pw\x8b\xa92\xf8\u007f1n\xe8\xfe\xa6\xa7CSm\xb7`\xb1C\x82\x01\xfa\x1e`1\xb4\xbf\xc4:\x12\xfe\v\xf7\x90\x10\x11\tcp\x95\x0f\xae\xf2m\xe3\x97\xee*\xbfu\xe7\xa7\xf6\x93\xa3\xb7\xbb\xdf\xd9\x1c\xfc\xe4I\xd2.\xf9\xe5\xb4պ\x9a\xdcQ\xaf$\xce01\x9706\xb4ߙOX\x17\x92#\x85,JN\xf7\xbe\x1d[\xcdc3sh3\xb90\n\xa6\x17\xafua\xd0:/ݥ\"D\x1f\xc1\xcd\\7r\xc19a\xc2\tI\xfc\xd8\x03\xe3qv0\xa659\xaf\x03\xa1._qi\xd1\xf00\a\x91t\xb7\xd2\r\xa6\x896T\x19&fc\xf2\xbd\x85\xe5\xb41\x9f\x8b\xc2\x04Y\x94ܰ\"2!\xa9\xb2\xb0\xaa\xda$\x84j-3F\x8d\xc7p\xb4@\xe5T\x9b\xb0%\x98\x96c\xe8=f\\f\x90\x83\xc8\x00kY\x95\x91r0\xec\xf9de1z)\x96nn\x94\xe4\xa5K)\x86h\xee\xd3=\xb7\x8f\x9d\xeej\x8f\xafO\xad\xa9\xb3^#\x99\"\xfa\xb8\x1d\f9\xadK\x89U\xf1\xdd8x\xe9*v\x95\xfd\x92d\x86l\xe8\xd6u|\xbaҌS\xa2\xaaJ.>p\x16P\xba\x9a\xbbUŭ\x15\xd5D}\xe1SSo\x9fE\xb5=\xa4Z\xdbS\xa5\xed\xa7\xce\xeeRe{X<\xf5\x89:D\xb2F?\x05\xb4\x87\x12X(\x98\xb2Ǟ\x1c\xe6BT{DX\x0e°)K\xb2\x13\xacΤ\xa0\x00\x81i\xc2@\xb3\xb9+\x83)\xdaI6)4\xfd\td\xe8;\xcf\xc1a\x18\xfa횟c\xe0\xe6\x037\xdf1\x06n\xbe{\xf8\xe3\xf4\x19\xb3\xf2\x0fh)\xe3\xcd\xe5\xf4\xebկ\x1b\xf7\x9f\x91#<\xc7]\xf9\xea\xbcօU\xce\xf1\x8bq\xc7\x12\x8b\xc0\xe2ѳVd%\xe4\\1}\xf9@\xe6l\x16\xeb\x11\xe3\xb0\x04\xee\xf5{\xb2\xa0\x82\xce\\)M#C\xa8.\xf6v\x84eI\x8a\xe5\xad\xc2k\xb8T+8-\x9b\xe2\x92\xc6\xd1r\xdd;N\x13\xce\ue07c\x86\x82˕\xaf\x98)rrk\xa8\xb1l\xe9\x16L\\\x02\\\x12\xf3\xc0\xd5ܔ\x9c\xdfHβ\xa8\x98C\x9b\xf4\xae\x90抒sR \xa81y+ 6,s\xc1\x1f\xe8J\x9f\x91kX\x82:#W\xd3kin\x9c\xb9Y\xdfO\x89;\xbc\xd2\x03%lJ^\xba\xfeh\xc4\xd0\x19\xba.\xaa\xfauqD\xa1Z\x13s\xc2\xe4\x81\xe9\xbevz\xb4\xc0\xdc8\x80\xbf\xc1\xafZ\xd1\xe9\xfe\xfd\xec\xe4\xc3\xd9\x14\xb2U\xc6\xd3y\xd6E\x86ɮu\xf9\xf5\xfa\xdcƹ\x8cV\xda\xc0\"\x94\rC\a\n\xc32\x93\x85\x14\x1a\\A\xbd\x14\x0fW\xb5B\xe70\xd3=\xf78U\xc9+\xa46\xb7\x86\xaa=˴գ}Jo\x02\x18K\xfe\x19\xe5\x1cr\xc2\x16\v\xc8\x195\xc0\xe3\xfd\xe4\xa1\x02h\xb3ܣkw\xe9\x8aI&\xe9\rs*r\x0e\n\xeb\x15z\x1f`\v\xbe\x01\xb5`\x82\xc6\x16\f!Uz\x17\xba,!'4ˤ\xca}-\xb8Pً\xaax\x8d\xbf\xe2x\xa8+5$\xcfz\xbe^4\xe4\t\x97ٽ&\xa50\x8c\xd7\xe5!CmHߨ1\x1aj\x12\x8b\xa9\xfesT\x9d\x89\x11\xf6#;\xffM\xfd'\xfc!V\xf9\xedc\xf9\xecW\xcfws\xacՠ\x04$\rL\xa6\x94\xf1b+\f\xcc\v\x96V}\xb1DU\xd7W\xad\x04M\x9a\x91\x81\x05\x88\xdb]u(\xb2M\xac\x8bF\xef#\xab\x90\xb8\xd17G!\xa1\x16Ps\xec([\x9c\x1ck\f\x85m9\x13Ь_̰&jz\b\xb3y\x82\x1d?\xf2\x16jz\x10\x98)\xec1\xb2jԶts\xef\x93̯\xa44\xe4\xe4\xf8\xfc\xf8t#\xa8u\x9c\x0eu\xca88\xe9\xea\x8a,e\xd5f%\x83\xd4lQ\xf0\x15\xee\xcfq\x8e\x1d\x9d\xfcuXU\xa6ň\t25\xbbˡ \xd4\x19ђ\x18EC\x97\x81\xf4\xb9Zh\x16\xb8Q\xa5\xd7UN\x8e\u007f>>#`\xb2\xd4|`B\x1e\xa486HFcr'I\xa9\xeb\x89'\xc3\\ɒ\bpu\x00\xe0\xb1\xe0,c\x86\xafP\xcc'Ô\xa5q\xc5\x17\xb1A\"\x16ں|d\xc6\xdf\xd3I\a;%_\xe1iw\xaa\x02\xa1\xd6\x18Z\xc2\xf9\x1c(7\xf3\xf4\xfb}\x96.\x85\x14\xa3\xff\x06%\xb1\x8c\x97\xf0\x10S\xbd;\t\xb1\xb3\xe68@\xc6I\x8a\x1ba\xfd\xed\xc4$\x06\xab |\x03\xd1*'\xd9\xe8Hzww\xf3\r\x98\xb6\bKB\x87\x9dQ\xc8\xcfG\x976\xa8\xa9T\x1d-w\x9f\x1e}\xe5\xdf\\\xea$̐\xcd~\xadڸ\xee\x13\xceH\x11i>g7\x8cl\xa7%\xfb\x8cFru\x93\x9e\x88\xf5wYZlM脯\xaa*\xb2\x1a\f9\xb2SOO{f\x02\xf7\xf3\xaf@s\xac\xdb+\xb4\x01\x9a\xa8\"\x1d\xe0\xa85\xe6r\x18\xa5\xc6\xf5ݝ;\x90=v\xb4Y\x02\xcb\xd3\xfe\x18\xcfT:\x9bt\x15^\x14\x14\x8e\xfd\xfa9~$&\xb9\xc1+\xdc.\xf8\xdf'\xbdr\x13ih\u007f\xec\x96\xe8k;'\x16\xef\b\x83\t\x9c&\x1e\x8a\x1e\xb3\xeb\x9f%Lzg\xab\x92\xae\b\x98\xc3U/\x98\xfe\xeee|Z\xda\xfa8\xc8\xfd\x92\xe4\x12^\xcd\xf1\x9ch\xc2\xe9}|<\xf5K\xb5$i\x89\x88\xed\xd7\xfba\xa2\xe7\r\x05\xd2[\xdf\xc2\xcb<\xc9\u05cd7/\x1b\x1bIh\x96\xa5\x15\x8ar\xc3w/G\x86\xa5A-c\x13\x1c\xebћ\xc4\n\x19\xef\xbf\f\xa3ׅ\xb7\xc3\\w;\xc8e\xb7\x8ez\x89\x8a\x88r1\xe9\xc1I\xaa\x02\x18\xca\xd4\x04\xe37\xbe\x873\xa5*\xb6y\x8d\xd3\v!\xdc>\xfa\x1e\xaa0T̀\xbc\xb03\xfd\xe3\x1f\xfe\xf0\xfb?\x8c\x11\r\xc9PC`\x99\nruq}\xf1\xd3\xed\xfbWX\xc4-\x95ʟ\xe5f\x1b\x96mH\x96?\xed\xc8<\x82\xb2\xd8+5\x16:\xeb\xb3\xc3\xd6\xd6\xf0\xfeo\xe7\\֩q\xb6\xe60\x12\xd9\xcdG\xe23}\x84\xd8\b\x0fч\xb6\xb3MV\xdc\xca\xec\xfe\x00\x96\xf6\xf1ݫ\x1b\a\xaa6\xb6\x93v\x81\x8a\xe0bfb)\xf9\xd2\xf5\v\xbc{u\x83\bJ\xdbY\xfb6\xc6\a\xd0շ\xb2s\f7\xe1]jN\x12T\xb6(|\xc7LJ\x14Pδa\x19~\xab\nS$\xda\xf7\xf2>%\x8b\xe7\x93\xf1+\x1c\xbf\r\xe9@\xe8bH>\xcdk\xae\x89\x96\x8b\xa1\x0f\x8bh\xb8&R\xaft\r\x1a\xc9\xc15\x12'\xea\xa5\xea\xa7\xc7\x0f\x1aɧ\xad\x91|n22\xf9\xd5B\xc1\xad\x91EϬ\t\a\xe4@9\x13\xa1\x13ݶ\xa4\x06\x92'l\xa9\xeb\x1d}qsUy\xc7e+\x11\x01\x93W\xa2\xa1\xea2\x9b\x87،\x00\xad\xcf1=\xa2,\x9c\xe7+4\x94\x8c\x8fX\x15\n\xb0\v\x9f\x14gUE\x02D\a\b\xf7#\x98,\xfe\xb4\xa0O\xc6\xe7\x8e\xf8xbخ\xbei\x18\x99\xa2z\x0eة\x02\x1e\x99ѡ\xdb5\xd5R\xb8\x10\xae\xdf>&\xe3\x03\x98L\x93\x82j\xed\x02w\xa6^\x84\xfbȍ̏\x13\xa2\xb7\x8d\t\x91\x99\xa2\x19\x90\x02\x14\x939\xc1\xaa\u007f\xb9|\x88\x9f\xe7\x04fL\xe8@\xbfv\xa2\xe1`X]\t\x92\"\xc2u\xe7\xd9w\xad.Rخ\xbc4\x99L\xe0\xc3\xfeu\x8f\xc5\xf5\x04\xa2諓8M{|J\xca\xf9\xaa>\xa8ᦧ9\xfc&mf\x12\xa5\"\xa1^\xf7z&Q|4\xb0\x95yd\x8fB\x9d\x95ԇ\xfc[\xd4ɴ=UټG\x9b/\x12\x9f\xbe<\xa46=9\x86Ԧ\xbdǐ\xda4\xa46\r\xa9MCjӐڴ\x17\x88!\xb5\xa9\x9aѐڴc\f\xa9MCj\xd3\xd3cHm\x1aR\x9b\xea1\xa46\xed5\x86Ԧ=Ɛ\xda4\xa46m\x1dC qHm\xfa5\x06\x12\x87Ԧ\xb8ׇԦ\x881\xa46\r\xa9Ma\f\xa9M\x91c\xd0H\x86Ԧ_\xa3F\xf2\xb9\xc9\xc8^\xd5\xc5\"_\vy<7JNz\x94\x19\xbb\xc1X=\xcb|\x1a\x90\x9c&\xd7\xd6q\xd3\x19\x93W\xad\xf4\f߄\xdfUi\x89\x82\xe8\x13}\xea\xf4\xa4\xbe\xf5z\xa2k2\x85\xa2`\xfa\xbc\x90\xee\xff\xd59\x05\x8dd\x02\xe7_\x8b\x11\x0e\xa9\xc27%\x8b\xe0\xa9\f\x82$^\xb7;{\x003\x01\xa2a\x1e2s\xa0\x8fv\xd3#c`G\xb6@\x00\x9b\xc4\\\xbb3\x05\xd6\"\xfeɩ >K`3\xda\xdf'\xe1\x02S\xe16\"\xfd\x89\x10\xab\xec\x80mQ\xfe4\x95\\\x1f>\xc2\xff\f\xd1\xfd\xc3G\xf6wD\xf5\xc9J\x96I0\xb7D\xf4}d>\x916;\xa3\xf9!*\x9f\x06\xb3;\x92ߊȧ\x12S\xaf(~\x8f\xe0TO\xe5:ݓ\x9c\xac)\xf9d㻹\x02=\x97<\x9a\u05f6\xf8\xec\x1b&آ\\X6\xa1-{d\xcb*\x9b9\x9eFB\x9e\x93\xd3:\\\x18\xce\x02f9`\x13K\xcaxJ\xa9:,\xad7\xa7\xe8\x9f\xd0e\x96\x01\xe4VN\xbe\xaeC\xe0\xd10\u007f?\xaeV\xee\xbaj0M^\xc4R\x9ek\xa8\x88\xf6\xdd\xef\u007f\x97\xb4\xfb)\x96ab\xc2\xc6\xd3\xc9\x1a\b9\x1a\x93\xfd\x135\xfa\xa8\x1b\xa9\x8e\x94\xe7I\xceؑ\x98A\xfe\x9e(\x1av$e\x10\x96&f\x0f\x94\x90ыs\xf6L\xc4ؑ\x84\xe1q\x94\xa8\x80l&`\xac'R\xa4\xa1<=\xf9\xa2\x87l{\xae\xa4\x8b\xed\t\x17\xa9$Iz'[\xf4O\xb48`\xb7\xc3:s\xa0ww\xfc^.\xba\x03x\x0e{&U<\x17Z\x0e\x91B\xf0\x11\xbb\xcf&\xefj\x9f䉞\x89\x13}\x92&R\x13&v$K\xf4\xf14\xf7L\x94\xe8E>\xa9\xe1\x88\xe4PD\xff0D\xef\x10Ď\x84\x88>=a;C\x0f\xa9\xfd\v\xc3h\x87\x1d\xd6\xc2\a\x89ja3\xe4p\xd0\xd0\xc1\xc1\xc3\x06\xe9I\f\xbb\x13\x18\x1a\x89\b\xa98\xdcL^蓄Ѓ\xa2S\x99\u007fRP%\x99i3\xc1\f\xa3\xfc5p\xba\xba\x85L\x8a\x13\xa2u\x1bsN}\xe7L\xc8Å\xda\x10\r\x89g\xad\xa8>\x12\x8aq\n\xbbzӾ=\xf9q\xe3\x16䣹\fܕ\xd2C\x10\xc1_\xe5\x03\x91S\x03\x82\x9c0\x11\xe8 ޏZ;\vj\u007fQu\xac\xed__|\x15Ϲ\xdcd>_\xc7\x0e\xba\xb6\xb4~>\xbf\x9e\xff\xc0\xe1\x1d{\x1e\xf0\xb4\x8c\xf7ѷ\x9c{\xceA\xd8\xe6\xefћW\xb7\xe1{\x81\xf3\x0e\xdc\x04\xbdԾlC\x02\xccϔ\xa8\x92\xd3ΞL9#\t\x9d\xc7v\xa5\x9bթc\xd1`\xb7\xa4\x9a\xd5ic\xf1\x13ݖf\x96\x942\xf6\xd1=\x9ckib\xe9\xe6\xe7\x96\x141\xaf\x9e%*\xf1\xc9\xe9a\x83\x1d\x169v\xa4\x81\rv\xd8'd\x87}\x1e\x16F\xa3\xd6\xc97\x8afps053\xb0+\x92\x97\x8az\x91\x11\x14\xbc${\xc32\x19\x01\x90;NU\x15\xae\xc1\x8a+Ӓ'\x14\xaf*\v)\xda՟\\NE\xb3\x88K4P\x9f\xedұj\xaf(\xa5\x9c\xd0BI\xa7\xf6\x11U\na\xa5\xae?K\x16)\xd6V\xd2i\x12\xb2Y\xb3G\xb3\x99\xdd.\xabba\x15\x1c\x96 _\x1e\xe6 \x82\x96\xe9'lg7\x95*c\x13\xbe\"s\xcaS\xc2/\x0f\xcc\xcc\t%\xf7\x8cs?\xcd1\xb9\x05C̜\xe9Dg*\x97b\x86\x9bA݄᱀̪\x1d\x19\a*\xca\"m\xfdVY]\xc9R\x85\xf5\xfb\xb6qa\x96)I\x1b\x82\xf1\xb3\xb0\xd5\xc7z\xf7\x81M@\xacKP,5\x84:M\x0fL\xc3Y\x1f̆6\xa3\xee\x1c\xb8u\x17J.Y\x0e9\x99\xac\xd2\xe8_樵\x8e\xc9{\x84\x17\xf8\xbe\x90b$`F\xadm\x14\u007fR\x9d\x10wg\xde\xcd\xd3գ\x109˨I\xb0\xb14\x16֫\xcb\xe9\x91%\xa3\x88\x85\x06\xe5F\x03=\x11\x92HT\x8aK\xc1\xcc\nc\xa3\xf3Ґ\\>\x88\xd31\xb6\x9eMaR\x94L\xc0P\u007f\xafյ\x12D\x81\xa5\t\b:\xe1)\xca\tf\xe2\xdeu\x12(\x99\x025eBw\xbf\x195\xd0\xe9\x0fp\xf4p\xd8\xe3\xc0\xb4\x8f\x80NI)4Dߟi؇\u007f\xfc\xd7\x0fg\x1f\xb2\x05\xc8\xd2|R\x0e\u00879\xcb\xe6M\u007f\x03[\x80&\xb2\xecsm\xcdH\xf2\xc2O\xab\x9b\"\x9e\xb9}\xe4/Ϋ\x98\xa45Ɔ\xd8;\xe2F\xeb\xd5\xfc\xaa\xc4\xe9\xa8uS\xcb\xc3^_\xdf\xfe\xf4\xdd\xc5_.\xbf\x1b\x93K\x9a͛eH\x05\xa1VnD\xc1D\xb92\xa7K \x94\x94\x82\xfd\xb3t\x9d\xc7\xc9I\xf5\x9dӐ\x83\x1f\x057-_?\xc9R\xb4\x82\"\x8a\t\xb46\xe8;\xa6\xb1\xd1+B\xf1\xe9\xacR\x03\x99*\xb9\x88cJ-\xeb\x91\\Z0\xce[\x84\x96\xe6\x1c\x14\x90\x19[F\nY\v\xd57G\xa6yH*\xc6#lO\x8f\xd5b\xe9D\x96\x91&\xd0\x1c\x88\x00cOw\x15\xe1\x92B\xb7jږ\x1at\\~\xf9\xa4\xc4T\xeaB\xb1\x05U\x8c\xaf\x9a\x93\xb4\xea\xeb\xb5\f~\xb8U\xac\xa8m\xa2\xf0\xf5\xdb\xcb[r\xfd\xf6\x8e\x14\n\xcbz\xba\x9c\xe1h\v\xd2n/\x99\x80\xdd \xb7\xe1\xf9\x98\\\x88\x95\xfb\x90\xe3\xe5\x91Z\x06g\xda\x00Z*ޕ\x10\x9a\xd6\x1f}5\xc6\xff;\xb2;\xa8bCDUzy\xb6q\xc9\xc6y.\xd8$\xf2\x1e).\xbdA\x03=\xef\xd8$\xa4z\xad]\x9a\xf0뺱\xa8WP\xb8\xbe\xec\xb1ҒV$\x8d[\x88\xccО?\x9e\xec\xd3Iw\x80f\xcd%\xc5{\xdd:\x1d^\xd3\xcaa\xe5\xe85\xc1\xdf#Eì\xba\xba\t\xe4\xe84j\x94\x04\t@-\x1dZ\x9d\x84\xe5n\x82.A\xe2\x8c|E\xfeD\x1eɟ\x12 \xfe\xf1\x0f\u007f\xf8\xfd\x1f\xe3\xfdY}\xf4\x89>ʨ\xf3v_\xdd\xf4\xdc\xe7\xef-\x1b\xb3\x90\xec\xce\x18I&,鎋\xb3\xec\r(+&<\xc5\xc4㲇\xc7\xd6.\xe1\x93${\x97\x85q5\xad\x95/g\xf4'@\xac\x9c\xb0[\b?\xc5\x17K\xfe\xe4\t\xdfN\xf1\xafR\x9bk\xcfΘn\xcc8\x89\"\xfc\xe1&\vj\xb2y\x9b\xdfZ\x13\"\xe9\xd8\xd7\xe5\x97I.1\x96\xe5\xae\x03\xcdYB\xbe\xf0\xc7;\xbai\xe9\xb3-Jݤ\xa8>\xactͭ\x8f~\n\xaf\x97\xbb\x82\xe5\x89<\xa1\x90\xb97\x18\xec\x92\U000c640c\xb5\x18\xdc\xd8b7\xf8(EZ\xf1\x97\xfab\xbe\xe5\x85\x19\x15\xee&\xf1\x14\x94\u009bf)(]a\xc6$\xcb \x81,{p\xc1BI#3\xc9Sh\v\xb5Ɨ\xe4\xeeU|\x19\xc5v機\x03\xfah]\xb4\xfaM2a\xfe\xed\xf5͙\x9d\xd2\x19\x91\x8aܾ\xba\xbb\xe9w\x95\x89\x90\xa3\xbbW7G\x1fpOҢS\xa3\xb6.\x17\xf9n\xa0\x828\xeb\xacOE\x81\xd8D\xe7V\b\xd0Z0\xa3\x05-F\xf7\xb0\x8a\xd2yӱ\x94\x84\xa3\xcdI\xbb\xc5/\xe8\xfe7\xc1\x14М}B\xc5\x14<\x97\xaa\xe7\xd5]Ua!\x97\x91N#\xb4\xf6\x02t\x10y!\x990\xba\xab\xd4B\x14\xd8M\x93\xf1\x93IY\x1cJ-t\x8c\xa1\xd4\xc2\xd61\x94Z\x18J-\f\xa5\x16\x86R\v\xddc(\xb50\x94Z\xf8\xac\x92\xa7\x87R\v\x8d1\x94ZH\x18C\xa9\x85mc(\xb5\xb0\xd7\x18J-l\x8e\xa1\xd4B\xe7\x18J-t\x8c\xa1\xd4¾c(\xb5P\x8d_\xce\x15\x9f\xa1\xd4§z\xc5g(\xb5\xb0\xcf\xf8<.B\r\xa5\x16\x86R\v\x01/C\xa9\x85\xa81\x94ZX\x1bC\xa9\x85ϕ\xa8\x86R\vC\xa9\x85\xa1\xd4B\xf7w\u007f\xedv\xd8Pj\xe1S\xb5\xc3>\x0f\vc(\xb50\x94Z\x18J-\f\xa5\x16\x86R\vC\xa9\x85\xa1\xd4\xc2Pja(\xb5\xf0+\xf2*&i\x8d\n\xb4,U\x16g\a\xb7\x89\xec\x95\\\x14\xa5\x01\xf2.\x80\xaa\x94娅\xa3,a\xbay\xa3\xff\xc3v\"̤\x98\xb2\x99W\xf4\xce\x17T\xd0\x19\x8c*\xfc\x8c\xea\xfbw\xe7\x1f\"7\x9e\xb3\x05\x8b+\xb2`G]\xb1ই\x87#Ѡ\xeekN\xf74\xa6\vj\f(\xf1\x92\xfc\xc7\xc9?~\xfb\xf3\xe8\xf4\xcf''?|5\xfa\xf7\x1f\u007f{\xf2\x8f1\xfeǿ\x9c\xfe\xf9\xf4\xe7\xf0\x8fߞ\x9e\x9e\x9c\xfc\xf0\xed\x9bo\xeen.\u007fd\xa7?\xff \xcaŽ\xfb\xd7\xcf'?\xc0\xe5\x8f{\x029=\xfd\xf3\x97\xd1S=\xb0q\xda>\x8f\xdf!\xe5\xd4)Eȷ\x17\xf4\xd12\xd8xRX\xc8R\x18w\xcb\xc6\x1d\xf3\xeaD\xb84\xac\xd8CI>\x95\x83I\xfa\x18\xda>\x1fm8\x9f\x11c8\x9f\xee|\xbe\xf3\xb4\xd3>\xa1\xd1s\\x\x95i\xc7\t\x8d\x86\x19\x047\x1a\xba\xd5<\x99&r\xc1\x8c5\xa7S\xae\x197\n\xa9\xe0\x95\x94\xa6\x8b\xda\xf1\xaax-o\xeanS0ݼ\xa0Ѹ\x14.\x83훢\x96RQ\xc7)\x90\xe7\x8cr\x982\x01\xb9SO\u007f}\xfc.\xe95\rY\xa9\x98Y\xbd\x92\xc2\xc0c\x94c\xbf}^nۀ\x88ی\xf8C\x13&Dd\xe1\xae\x1d\xadU\bÛ\u007fq*+\x10U\n\xf4g\xb9:\x17`\x9cs\a\xcdp\xbcٳ6\xf9(\xf0\xc1\xf5\xe2\xa8)\xb6\xe4\xaeO)\xd7ь<]\x1a[E\xe6\x8d5\xdf{\xedb\x05\x85\xe4 \xa4\xf1\x1e\xbf\xa4\xd3 \xa7\b\x8d8\x9fB\xe3\xb2@K\xe2%)\x99\xb50f:\xe0\xfc\xa6\x9a9F\x98\xe2S>4\xac\x9bq^\x18W^\x86\xa4\xa0\xbf\x02\x9acM\x99\x82\x9a\xb9\xcbS]P}\x0f\xb9\xfb!QǮB\xb2v\xc6\xd5\xd2\xefV\x05$\xc7SQ\xb7vٿ\x18\xe7\x8d\xf7\xc6\xf6(\xefE\U000f70af\xdeIi\xbe\xaeʘ\xf4\"\xe4ｵԎ\x03E#e\x8ee\xbb\xed\xfcF\xb8\x89X\xb6\xa5Yi\xc5S_\x8a\x95\xf0\x81\x19\x84*Ņ\xfeF\xc92Z\x15\xd8Pֿ\xb9z\x8d\xfc\xb2\xf4\x192¨\x15\x96\xa6Ja\x12\xed3W\xd9c\u007f\xf39MI\xd96\x15{\b\xe1z\xf2\x86\xae\b\xe5Zz\xc31!\x18\xdc\xe5!!\xdeU\x93r3z\"\xcd|ݧ\x83\xeca\xf3;\xf1\x05\x8c\xea\x04\x9bʓi\x97\xd0Dž\x84`\xe9=hR(\xc8 \a\x91ES\xef\xc7I\x83@ʿ\x96²\x97^\xb4\u007f\x15\xf2\u007f\x9c˸\x9fU\x8f\x99Jަ\xa7\x98\xaf\x84̥Ԡ\\r\x98*!m\xe3\xbf-'\xc0\xc18G\t\x16\xb9\xa5\xc6y\xfe\u0602\xce\xe2O\x135\x95(4\x92\x80Х\x02\xef47$\x97\tf\x80\xaf#e\x97\xfe\xb7\xab\xd7\xe4+rb\xd7~\x8a\xe4?\xa5\x8c\xa7T}\xc1\xdb\x1fk܄M\xc3\x14-J\xe3u\x02\x81ƾr\xac\xfa\x8c\bIt\x99\xcd\x03NS\xbcC\xc1y\xe5oH\xe1\x15\xb6\x815}\x02\xac\xa9\xa7`\xfd\x9b\x06\xd5[\xae\xfe\xed\x03\xc8\xd5>n0˛ڻ\x86\f\x85,\xc0М\x1a\x9a\x12|+E\xa3:\xe2\xdaQH\xa1\xdd\xddG\x01I;\x1a\xe6\xaf\xec(|\x1c)\xad\xe1;&\xcaGw;\xa0\xbfC\xf9\xf6\x12\xc1\x11\x1fJJ\x91(\x13 \xb4(8s\xe5\xfb\xd6Z\xc4\\\xb5H7m\xef7MM\x14\x0f\x94siՌ\x84\xf0\xb8\xa2\"\x97\x8b\x8d\xc5[C\x14h\x82U\xdcXp\xc7\xe1\xdcv\xd8\xe2ew}8\u007fm\x87\xad\x8f\xeb\x9e\xc3\x12\x12\xaa\x94\xaf7Q\xb2P\xacA\x1a\xa8\x06\xc1&y?9\x9d\x00w\xaa\xa1;9z\xf3\xe4${E\x13\x9d\xaaJ\xf2\xfe%/\xdeI\x0e.7> ɂ\xfd\xc5\xe0\b_\xee\x8b#\xf4>\xb5p\x94\xecE\xff\x14qT&hxd\x1dGVMl\xe3Ȃ\xfd\x85\xe0(9\x04\xa1!\xcb䢸Qr\xca\xe2\x0f\xeb\x86\xe8\xf7\xe0\xea\xe4\x9cx\xd1_j\xe8\xca\"G=\x12\x81\xc7k\xe4~2T5.=Q\xe3d\x9e\xbf\xc5\x15\r\xf4\xffk\xa8\x10ȵ\xcf\xd6\xf4\n\xff\xd5\xf8\xd96\xf3\x85\n\x99\a@\x1fT\xbaɌr\xecH\x94F\x17d\x9d6\xd6\x01\xf6\xb8\xcfE\\c;\x0f'\xe4\xf4aK\x16\xfc%\xc13@Bs?\x99C\xa3v\xbc\xbb\x80w\xe7\xee\xcbX\xd8I\x80õ8\xab\xa7\x84\xe4\xab<Į\xec\x17Ӧ+}\xa9\xec7UO%\x8bp\x10yj%\xa8\x82\x9a\xf9\x19Q\xc0\xf1\xe2^`h\xf7Ρu\x9c\xb6O\x8d\x05\a\xce\x106\x0e\xf5l&E\xda%v\\5\x86\x05\x82F\x947\xe5{\a.\x98Ιew\x86\x89YB\nF\xedQ\xa1\x9c\xb72\xf4\x0e\xe1R\t\x9c\xa0ꓺ\xe9:H\xf6\xd9;bn\xfb\x12{\xfb\x0f\xb6\xb87jwE\xbc3e\x87{ù+\xa2A~\x1c\xf7\xc6l\xa1\xe9+e\xbfk\x18\xe5\xb7E|\x87\x1f\xb2N\xcb\u07fc\xb9\xbdh\x83L\xe3\xec\x98\u05eb\x9czla\x12\x9a/\x98\xd6L\n\xf2\x00\x93\xb9\x94\xf7IpOB\xea\xfc\x8c\x99y9\x19gr\xd1Ȣ\x1fi6\xd3\xe7\xfed\x8f,vҚ\x9c0\xc1í\a\xe7!\x14F\x87\x88\x81]L\x9a\x96Ua\x15\t\xd0w*\xf4\t\xae\x9bh\xbfN-R\x857\x16>\xb8J\xb5I\x8a\u05c9\x05ş \xc7d\xbc\xf8\xea2\x8djO\x8e0\xeb}I\x13\xb7v/]\xe8\xe7÷\x11p\xa6Z\x06\xba\u007f\x1b\x81\xbfְH\x0e\xae8D\xa2\xddǦ\xad\x86\u07b5B\xe2\"ډ\xb6\xe41\xd6n\xf3S\xe5\xa2\x1f\x91\x1a\xb0o?\x8f%ی\f\xf16\x92\xb3\xe9\x14\xc2\r\xe7H\xb1WPE\x17\xd6p\xd0ħ\xfeN`\xc6\xdc5\xd3J\xb5\x8a\x8cPTE\xc2Μ\xba\xc7\fY\xb0\xd9\xdcyi\b\xc5R\x94\xf1\xe5&\x8d$\\Ҝ \x17\x97\x8a=4\x9f\xde{\fͧ\x87\xe6\xd3C\xf3\xe9\xd814\x9f\x1e\x9aO\xef=\x86\xe6\xd3C\xf3\xe9\xa1\xf9\xb4\x1bC\xf3\xe9\x8414\x9f\xde6\x86\xe6\xd3{\x8d\xa1\xf9\xf4\xe6\x18\x9aOw\x8e\xa1\xf9t\xc7\x18\x9aO\xef;\x86\xe6\xd3\xd5\xf8\xe54=\x1b\x9aO\u007f\xaaMφ\xe6\xd3\xfb\x8cϣ5\xdc\xd0|zh>\x1d\xf024\x9f\x8e\x1aC\xf3\xe9\xb514\x9f\xfe\\\x89jh>=4\x9f\x1e\x9aOw\u007f\xf7\xd7n\x87\rͧ?U;\xec\xf3\xb00\x86\xe6\xd3C\xf3\xe9\xa1\xf9\xf4\xd0|zh>=4\x9f\x1e\x9aO\x0fͧ\x87\xe6ӿ\"\xafb\xe2M\x91\x9cE\xb5lۣ[@J\xed\x93P\xbb\xd32\xb2r:\x05\x85\xc2\x17g\x17\x94\xa3\xb4K`\xa13T%\xba}F,6\nT@sW\xf3\"\xce/\xd89\xadP\x84\x14ۗ\xb9{\xa9\x91\xe1\x01r\xf9\xf6\xeb\xda\xe9\x9a\xd2\xea \xedv \xae\xe7\xad\xc8\xd2\xef\nՄ\xd0Q\x9d5\x0e\xb7.\x01?\xe3R\xfb{\xb2\x88\xeclN\x85\x00\xee\x95n\x16\x87\xd99\xd5d\x02 \x88,@8\xb5\x85\x12\xcdČ\x03\xa1\xc6\xd0l>\xb6+\x88\xf3\x9ey\"\xf0]\xeb\xea\x99j\xa3\x80.\x1c1(X\xc4\xf6\x19\xb4S$4SRk\xb2(\xb9aE5I\xa2\x01KeD&-]M\xeb\r\xc6$\xf1\xfa\x02\xeaY\xb5\x8a\xe89\xba2h\x8d3o\xa82g\xd8\x0evQ\x98\x95\xbbK\x15'\xf8\xb0k\xa7҆d\x9c\x810~ծ>#\xce\xf3\x8c\xc4\xe6\xc6\xe3\xf5]\xb7\vڣV\xe4\xe8\x15)\x8cv7}\xd2&꧘3\xed\xbdo\xfa\x8cP\x13\x04e4\xd1\aZB\xb2\x0f\n\x9c\x9b\xb5\xff)q\x9auI\u007f]_5\xab\x99\xe1\x94\xd38\xc500\xa5\xb3V-\x87\xda>\xc4$wd\xabQ`\xb1\xec\x90\xc3\x02\x1e\x1c\x01K\xcb? \x03\xb6D\u007f\x90\xe5\x8cQ\x10\u05f9\xe8\xb33ц\xee\xfa\x06\xb4\xa63\xb8\x89L\xb1\xd9\xe6 \xc6,\x9b\x9a\xb8\"\r.\xacFfdC\x87\xab/\xa0\xb4-\xd0(\xb0\v\xb7\xc6\xca\xe6|P\xcc\x18@\"\xc6\xceU\x98y\x18\x99\a\xbe1\xb9\xe6\xf5\x987\xe1\x83\xeeC\xb1Tk\xf5)\x91\xbbK\x1d\x13 \x13\xc5`J\xa6LP\xee\xefa\xc4]8\xc2~\x16T[ҤZ\x83B\x9f\x8bw;\x05\xdc\xc4\x11\xec\xf7\x1e\x91F\x95\"\xa3\x8d.\x97X\xfd\x8eM\xc9\f\xefzD\x1a\x13s*ȿ~\xf5\xef\u007f$\x93\x95Ղ\xd186\xd2P^m \a1\x8b\xac\xed\xef\xc5S\xbb\x0eYE\t\x9c-X\xac[\xc8\x1a\x03\xbf\xbb\x9f\xb4\x03\x8d\xe79,\xcf\x1b\xf49\xe2r\x16\x87\xd3W\xe1~eug2F\x91Or\xedw\xb0\x01\xc9Y\xb6Jf\x04\xa1y\x0e\x99\xcb\a\xe7\xca\xebyb\xeb+\x8e\x85,J\xeeR2\xbe\x0e\x95%\xa3@\x96\x1a6\xabau\xf2\xc1Xj\bS[\xef\x10\xee\xaeL\xf9\xa5\xc4)-\xbe\xf0\x9c\x0f\x8dW=s\xd0O\xfc5\xe5|B\xb3\xfb;\xf9\x9d\x9c\xe9\xb7\xe2R\xa9\xc86\xfbH\xfd\x01\x1f\x9cZ-f^\x8a{ly[\xd7\x1a\x96q\xd2V\x96\xa6(M\xb8\xe4\xddt\xef\x86͌\xae\aY)h\xc13\\\xcf\x0e\x1e\xed\xb9E\xf7l\x1c;\xf0\xc5w\x1cw\xe1rV\xcd[\af\x10{#\xe8w_\xfd\xeb\xbf9\x96E\xa4\"\xff\xf6\x15^\x19\xd5gN\x88\xa1n`\x15\xd9\x05\xe5<6\x98\xd5d0\x96\xe8\xc7\x1dL\xe2\xd9y\x84Ig\a\xcfbr\xdf\xdd\xfd\x1d\xedmf4\xf0陫=\x12<\x88Q@\x8fQ\x89;\xf6R\x16\x8b\xdc|\x04\x83v)y\xb9\x80װdY\\\x84\xbf\x85\xea\x16\x94\x10\t\xe2L\x1b\"\xe3\xaa@L\xb8\xcc\xeeI\xee\x015\xeef\xac\xf7\xb1\x8e\xc1L\xc2-\x94\xad\xab\xab\uf7d0\xd8fD\vZ\x14U-\aE\x1fZ\x8bE^\x12}\x01\x85\xa6\x06\xaaӳ:\xdctc\x15\xf6\xf0n\x03\xab5\xa0@0E\xac\xf4s\xc3\u07fc\xdehH\x15:\xe8%\xe5\x1a\xf8=qz\x9a\xdd9\xe4\xcc\xf1\xe1\xf5\x1eI\x0fiwzZ8\x16U\xae\xc0\x82\x1ao\xd3$\xe6\xcf \xd5\x16\xa04\xd3V\x81y\x8fg\xe2\x15\xa7l\x91~\xc3?\xa5!A\x8f\x16\xb0)y\t\xa3\x06\x9dF\xbe\x18\x8d\xe8^u\x8f\xe2\xee\xb68\x96\x86-}ӹ\xfe\x8d\xcc= dծ\r\xb35e\xa3\xc9a[\xa1\x87^\nG_\xb6\xff\xbe\xc6Q\x93\xeb\xbbu\xc6\x1fg<@\x0e\xa6g\xf6\x1f\x83}\xe3\xe4\x0f\xc0\xbd\x91o\xfbe\xf4\xad;\xd7t\xd8x\x82j\x98^\xdeG2v\xb9\xb1\t\xe0-\x05\xf9\xe9\x91\xe3\x97\xc7\x1f\x94\x87;t+Y\xd0\x19Z#=\xb1\xbe\x0e\xae_\xa1Yk&#Īg\f\u0085\xbc\xaam\x9e\x04\xd4]\xa8\xaf\xe5p0\x9f\xb0\xf2X\x02\xc4\a\xba\"T\xc9R\xe4.\xf6P\a\xa5ެ\xa1\xe3Z\x8a\x94)\xfb路4Uմ\xc54\x01&ȋ\xf1\x8b\xaf>7\xc1\x8f+Y\x13\xfc\x89\x85\x9f\x1b|\xeb\x83b!\xb4l\uf2497\xde\xc5ZwXO*;\xe9b@\b\xe4A1\xe3\xa9\xf9\x81i '\xb1^\xf30\xa4jֲ\x0e\xfc{\xd8\xe9\xfbj\xa1\xe3\x1eVU\xd8\x1d\xf1b\u007f\b\x01\xd0\x1a\x15\xbe\x8d\xfbnٿ3ʹ\x97\xbe\x11\xb0\xb6\xf7\xf4+4+\xb0\xc4\xe7HŮ\xe1X\xfbV\xcdR\xe89+\x9eJ\x8e\xa1\x98\xb2-\xa7\x01\xfbU\xf3^\a\xde\xd1ߕ8#\xd7\xd2\xd8\xff\xb9|d\xfa\x89\v9v/_K\xd0\xd7\xd2\xe0ӽ\x91㦶7j\xdc\xe3H\xd2\xc2\xf1H\xbc\xa4\x84ߨ\x96y\xf5\xf4\xfd\xf7\n\xc5L\x93+a\x19\x95\xc7AuYQ{\xf0\xcd;\x86\xc8\xd5vk\xa2\xee\xdb-\xf8\x0e\xad\xf6\x1bM\xcc5?\xb5\x1b\xe5\xadi\xb8)8\x8f\xb6\xfb\v&h\x17\x9cf\x90\xfb>\x13v㍢\x06flw\xfb\x81\x05\xa8\x19&\x1ad\xf3]\xab\xda#t\xb8\xa7\xe2}\b\x15y;\xab\x19Uh\u007f\x0e\x15\xda\xcb\x10\x14\x9f[\xb0\x11:\x89Q~\xf3$G{\x12c\x9b\xb2\xc8}\xda\vsZX\xca\xff\x1f˞\x91\x88\xfe\x97\x14\x94)=&\x17\xfe\x86ʖ\xef6\xdf\xf0\xbaN\x13\xb8\x85\xcb4\xb1\xbb\xb0\xa4\x1c\\\xa1b*\b\xec,\xbf\"\xa7\x1b\xd2\xf2\x8c<̥F\xc9P\a\x91\x8e\xeeaut\xd6:![ ڇ\xaf\xc4\xd1Y\x15/k\x1d\xcaJNa\b\xe3\b\xffv4\xde\x10\xb0[`?!vwRɎ?VZ\xf7\x1b\x97ڴ\xb9\xf3\xfb\xd2\xc7N\xdaب\xc0\xd8\xfcf\x8b8\x9a\xcaqˬ\xe8\xfa$U30]&\x88ט1\x95aL.\xc4j\x03.^\x8c\xebT\xb9\xbd\x11W\xd1Y\xd1j_\x84--1C\xa2\x01\xca'.\xe9nC\xd8>\xb8\xb9k;6\x05\x05\xa8Zµ\xcc\xe1F\xaa\xae\xfc\x8ev\xbcf\xfd\xf9\x0e\x8b\xb6\x81\x14\xc9s̳\xc7G;拺\xb1\u05cb\x0fi|\xfa\xef\u07fc\u007fj=\xef\xaa\aw/\x84bo\r\xb7_\x1d\xeb\xb0ﻻ6\x82\x16z.\r9\t\x97\xda3.\xcb\xdc\xdf\xecW\x1d\xf1\x9e\x1e\xab\xd4\xd9\x1c\xf2\x92Cw\xd3\xc1\x8d2\x95\xe1Ѡ\xfb\x95\x82\xfd\xb3l\xb7\xe8\r\x1e*\xfft\xd7A\xa8qR\x99\xd6\r\x8f\xaaeG\u007f\xc1\xfd\f_\xf2V\xa4\x87\xbc%\x15\xbe\tҝ\x05\xa9\r\xdeS\x12\xa6Qt-\x98\x9d\x99o\xcf\xe1\x1f\xef\xbcf\x17ְ\xed8t\xb0\x8fn\xe1:\xf2_݈\x88o9V.\x97\xbe\r\xa3\x8b\xe6n]\xce}F\vS\xaa\xd0\xfe\xbaT\xd8\x0f\xacnaB\x03\xe6<\x8aZ`\xb7\x1b\x06\xde/Ȥ\xb8c\vІ.\x8a'(\xe4\xd5\xe6\x1bv\x03\xa4\xcauU\xe9\xa4\xe9\"\xa8\x9bnuYŴn\xf5\x96\x8f\x1b\xb0\x1d\x18\xd4\xc9,h\xc8\t,A\x10\u007f\xc9\a#\xef\xce\x01\xd1\x01\xf4\x0e\x8d\x13\xb5\xc4X@\x80\x83\xf9\xbeS\xa9\bvի\xa6\xbeI\x11\xe1\xeaxN\r\x8c:o\x12\xeeu\x12;\xa5\x8e\xeb\x91\xfb\x04\x82/]#]\x94@\x19\xe6\x89\xd9\xed\xe5ܽ\x1dn\x1e\xf8\xab~\x0f\xa0\x80\xcc@X\x14wr\x1c\xafʺVO\x16\xb1\xfe\x04W\x0e\x9c;\xd7\x1d\xab\xa4<\xb4\xf0\xc5\n8A\xaalu\x99\xe1#\x9d\xb7\xacv]\xa0\xf77>\xde\x01\xd5]yK-D|\xdd|\xd6\xdb*\x0e\a\xb8\xf4\x8c\xba>y\xaec)S\xd0I\xfa~J\x12\xbf\x1cq\xc6\t)\xe6T?\xc5.o\xec3UK\xaeơ\xac8\xe5\xbb-s\x02Q.6\x81\x8f\xc85\xb3%&4ۣ\xd1Uy\x05\xed\v?\xa8\xc6\xc4SqW\x8f\xe7G\xb9\xb4\xee\xe7\xde\xe3O\xd2:~U\xe6\x95\x11yg>~j\xa5z\xacra\xda\xe7g\x006\xd1%^\xc1g\x9a\xaa\x14\t\xa6g\x005a<\xf5\xaaF}\xf7\xde\xc3H2,\x84\xc7\t@\x97\xa8>|\xb9}\xf8\xfd]\xef1@\x8a61\xb2t̞\x80\x1eH\v\x02\x1e\x98@0\xb5(\xc0e\u0081\xc1ҠE\xe5\xa8Gip\x150L\x1b\x90\x00\xda@\x89F\xeaT&\xf0\xa3H\x9e\xaa\xd2\x0f\xb6\x99\xae\xf2\x146\b\xa6R\xebf@it\x89\xc6\xc9\xc0B\xdf:*\xd3y:\xc0\xf8\x1d\x11\xe5{AJ\xba\x82\x16\\\x86\x811\x98\xd6|\x00\xbd\x05\x97I\xdb\xe2\xcf\xe2\xef\x01\x06\xea$\x14\xe8\xcd\xdf1qk\xb8CC`\x02։V;4āD?*\xf9\xcf\x06\xb6\x05\xa7y\xd2\\8\xac\xe5\xda6\xa9\x1c\x1a%r؉\xbc\xc2\v\x10*\x85B\xec\xc1 \xcd\x02\x95\xea\xc0\xe3.v\r\u007f\xd6\x06A\xaa\xad\xbe\x82̹\xd2^]^>J\x17\x96J\xa2\x8b\xa2R\xd2\xed/Y\xeb\xe5\xa6r\xda\xd8\xcb\x14w\x98_Z\xf9\xb8\x12&ɤ\xc3\xc4U\x06/E)W\x8c\xba\xe2\xe5\xb2.\xd2\xff\v\x12\xb5\xefz\xb8\xba=\xe9\x97uF\xaa\xc7\xce\vV\xe8#\x12 \xcd\xf6\n\xe3\x87z*ZF\xd3#\xe2\xceכ\xbb\xfb\xae2I;\xe4>\xf3\xbd\xa3a\xad\b\x88aRm\xd1x!n\x8d.\x18&\xaa\xb4\xd4R9\xfe\x91\xe4\x12Ր\xfd\xb6\xda\x14ґ\xdc\xffQ\xa1u$\xab5\\\xb3\xfd =\xac\xcaT8L\xd7p\xab\xe0Z\x14\x98_\v\x8b\xdf]\x00\xc4i\xbb\"\xc6Ɖ\xa0k\xfa\x86\x9d=\xd7:/\x82\x99\x9a\x90WX\xe3w%&\xbd%C\xe3\xe4V&\xbc0`\xabMk\x02lך\x856\xbej\xa16=\xd4}\xf8\xfc\xa8\xb5\xb96Z\x01\xbe\x90\xec\xdb\xd5L\xba\U000dc862\x15f*Ex\x1e\xc0\x84\xdaĬ\x0f\xdeLp\x93_aQ\xd2r\x9dA\xf1\xbe\xeeF(\x12\x8f\xd2\xc6\x1dy\x83\x82\x8dyӵU\x83\x03\xa3\xc2\xd3eH\xfc\xdaɴ\xb6\x1a\a\xdc<\xceQ\x8f\xd8VT\xb9{\xd0yU\xa0\xbd\xd7_\xd1:\x99\x8c\xf5\x1c\x10\xf1qt`\x907Z\xe2\xb0\xcb\xd0\xd0\xe2\xe4\x17l\xefF\xe1\x02\xaf\x19\x8b)\x9b<\xf1\x84 `\xe39@\xb63ϡ\xd4)\xec\xfcL\xb0\xd9\a\xa4\x0fe\xd3\xcag\xa3u\x8eb\x8ck\xf8\x92\xe4U\x8ai\xe3\xf2F\xf92\xa0\xf6\xe6`\x10\a\aB*\xd22rń\xaaj\xdeN\xd0\xc9\xfeJ\x18\x042\x14Ry\x98 Y\x05k\x92lj\x92\x0e\x8b\t<\x8fj\xa4o\x14\x84\x88M\x8eW\xe0L5\xa6\xeb\x01\x860F\xec\x8f\xf0,\x04PKX\u058c\xa9\xcdy.\x13$f5F\x9b\xb9Ƭ\x99\xa0\xef\u007f\x90a\x99\xd6O1L\xfa\x13\xf5k\x9d\x13$\x1c\xa7\xc2\x063\xb1\x93\xda\xd8a\x84\x83/\x98T\x0e\xa7֑p\x90\xca\xed\x16\r\xc1*3a\xd1\x06\x93r\x8cY\xc7M\x045s\\\xf0\at\xb5B'\xe117\xa6HaS<\t\x15\x18q\xb2\xd8U\tR\xa5r'\xd3J\xe4 \x95uB%\x9e>\xd1\xe07N\x1f\xcc)\xc4\x01\xfe\xde\x00\a*HJ=Ϧ\x15R8Zh3\xae\x1c\xa1\x1d\x82\x99f\xc3F\x90\x05\xd4S\xee\xa8m\x86\"\xfa\x1a\x95\x94]jkw.ZI\xf9\xa00\x17\x1b\xcc\xc1b\x8e\x89\xd3f\x9a=1J\xe0[\xac\xfd\x9c\xe0\xec\x88%m}\x06)\xea\xac\x11m\x9b\xd3\xf0\x9c\xc9$\xf3\xf1\x1bi\x19ÂT\xa3e\x8b!\xca2\xdf\x1f#\x1ab4\xa3\x9el\xceh\xb4-\xc2|\f\xe1N\x19\x92\xb6E\xda\xe0\xb6\xcdX\xe3>\xd7\x1b\xb5ycz\x0fM\xf5M\xca~{0\xfc\xf5\x95\x9d\xd8-i\u007fw\xbb\x05,J\xb7\xbf\x00\xe9\xc2\xd3\x18\xa8\x14`\xb5x\xfc\xc6\x04w\xdaj\xb9\x1d\x8e~\xf5\xd5\xf2*Rk\xd0\xf8\x8d\b\x8d\x9d\xd5]\xed\xab\x16\t\xecSw\xe4\x05\xc8m#\xb0\xf4\x02\xb62wȱ\xd4\x1c\xa2\x9d@gVr\xafɠX\xdfK\xad\x10.\xc9n\x9a-mĈ\x01\xaf\x86\x00|\\\x1e\xf60,\x83\b\x90\xd0\x04\x15\x9c\x05\x91\x06\v\x9f]\xb9\xe7\xf5\xd1>\xe1\b\xf0\xc3珘α\f\xe25\xf5\x80\xa8\x0f\x83H\xa7\x8b\x02\x13\x18\x05\xb2C\x14\x87i\xcd\x1e\xcf\xe7\xd0.@\xc0\x13\xee}d5\xba\xb9\x1ck$Zр4\xc8\t=6#O\xb8gPu\x86.\n\xde\x12U\xf1\xed\t\xf7\xb1]\aL%\xfc\xea\x1c\x85\xe7.=`*b\x96R\xdb\x1a\xa6\xd6k\a\x9c\x8e#\x16\x96\x19\xa5\xd0\x02\xc7O$\xbb\x11X/-\xfd\x84\xfbw\u058b\x8fVM&\xcb\x05\x1c \x83\r\x16y\x85\x85|\xec\x83\xc8e\xdaL\xc6\xebd\x01\xc4[u\x01\x9f\xb5\xa3?7/\xd2\x12\x8a*\x85\x8f\x1a\xedg\xed\xf8\xc9we\xb1'\xe2D\x06\xfb\xc1\xbc,\x95w\vėE\xf3\xb78\xb0\v%\x15m\xc4&-\xdc*ڟy\xfe,\x11S\x86\x019\x8fVQY\xce\xe8*\xadV\xec\xa6\xc3l\v\x80v\xf1\xaaE\xa5MOR\x17\v!\x8e\xa2X\xa3wO\xdeʿ9ȅ\x1fk\x06\xcb\\$\x98BZq\xbe\x9d\x13\xef\xc2\xe1\xa3L\xa0@\xf3\x88P\x92߈W\xaa\x05\x96ܷ\x13\xb40>\xb4\b\xadv\vi\x1cb+Z\xf5\x91=\x83\x98\xa3\xbaOdُw\x8f\xa3\x92\xdd;\xc7CQ\xdc\x17iʩg\x91\u007fY\xe8Y\x16\xca\xeb0\x06\xf1H\xfa\xf0\xa3\x10\x9c\xec\xfd\x17\xb9WV\xef\u007f\xc7yC!\x8d]\xc3\a>\xdc̱;>d\t;SE\x81$L\xa4\x05ғ\x9d\xc8)| \xe3\xad\x00s\x1fL\xe8\xedA\x04\x15gb\x9e3m\xbd\xcf\xdfJ\xcc\xf9t\xeb\xfc\t\xf7\xe7\x17\a\xd6\xeb\xfcV\x9d\xc7\xc1$\x9b\u007f`\xb4\x9a\xa8E\xab|\x0f\xe7\xfc\xee\x9c\x03\xb3%K\xe4\x84\xe0m\x81VGw\xe5\xe3\xe7%[\x01\xdak\x87\xa8\x85\x067\x87\xb4\x14\xc2\xcfQ\x11\xadӥ\xb6n\x11Z_\xb4u>\x01\xd8\v\xb7G2\x841\xbb\xbf:k\bb\xebЀuڄ\x03Q2\xbb\x83\x049I\xde\xce˞D\xddd#=`\xdad\x9e\xb7\x16\xc2\xdb\xf4s\u007fRJ\xff\xcf\xc3L8Xbإ\xd1\tZ;\xafJ\x91\x9ec&a\xdb$k\x85\u07fcm\xa3LsL*9\xb4e\xa18\xb1\xf6\x84\x8d\xcd\xcdK'\xefLf\x88~Ǩ\xf2)8\x02\x17\xbd\x14\x85\x18\x1e\xceG\xa3{\xedG\x87\x05X\x03\xf3\x1b&\xf3X\xb1QY\x167\xd7*\xf9k\v<\n\xa9ny\"x\xff݂\x15\b\xa6\x1cO\xdd\xca\\\x87\xf1\xad@\x9a\a\xb1\xfbW\bG͚\xcfj\f\xf6${x\x92\x11/)\xa0`Zi\xd7M\xd6\xd43\xbd\xb3\xb0\x95ƺ\x16\xe1\x05P\xa5\xe5\xe3\xe4\xef\xbb\xc7T7Ɯ\xbc\xc5\xfcŏ\xee\xa4\x153\xfd\\\x17F,\xd9X\a\xe6gb\x87 \xb7 \x1d\xa0Jt\xa58\xe1E悦Y\x00\xd1\v\xd1;\x93H\x9f\xd9\x19\xac\xaa\"\x9e!+\xd6N\xa9f\xb3c\xdd!?\t\x19\x97\x9d\x82\xd3\xc4\xead\x81\xba\x9au\xedm\xebWv\xf8ѽ\n\x98B\xbcȢ*@\x14$\x96%\xfb\xc6-cӔ\xcbxY?\v\xe9\xf8Џ`\x93x\x96Y\xd3D\x17e\x8e\x0ea\x83[m\xd8\x1eX\x99b\x13>\xd4\xf2\x1f\xad7\x99j\x02\xb6B\xe6\x95Y`\xa3\x17Kf龭6O\xaf\xbf\x19\x8bGd\xc5̌L\xba/\b\x9a\xe7\xfdGi\x96\x85\xcc_\f\xbe~hZ\x1aIZ\xaa\xe7\xa2\xd3Y\x98\x1c\xbd\xf6\xa3\xd3Zy\x85\xdaO\x85\xa7\xb3P\x19\x93\xb7\xf0\xb4io\xe1\xe9[x\xfa\x16\x9e\x0e\xda[x\xfa\x16\x9e\xbe\x85\xa7\xe3\xed-<\xed\xb4\xb7\xf04\xda\u007f\xc4`\xe8\xbf::\xd2!\n\xab\xc8\x12\x8c9\xb4g\xe6\xaa+\x8d\xae\xf3\xca:4K*\xa4o\xc7G\x8e\xd4\xd0'\xbeˊ\xbf֚Қ\xb6t\xa5uzM\xc94-ɰ\x98\xfc\xb7\x18\x11Qxt\x19\xf4t\xb5}l\x01\xdd\\\xd9\\\xbfv\xbc)W\xf3\xffM0\xc4\xe90}-=\xff\x8dO\xb7\xe6\xaa_\xfb\xc6\xfb\x80\x80\U0006fcae<\xb2\xacm\xa6\x98\xedx!\xfe\x94\x87\x0f\xbc\x1c\x1c.\xf4\x99iz\x85߿j^FT\x9bMטէ\x96\xe8\xc4\xee\xfd\xba\xff\xc6\xe9\xba\xe2l\x82\xb2g\xe92\xff\xfd\x11m]\xd5c\xb7\xac=\xe8i\xfd\xedߐ\xc7\x13\x10\xb5\x01%s/\x80\x00\xa1\xc7~\xf8\xa5\xf4[\xe4\x93\xd7\xf9\xfcF-\xbe.mi5ZS?4\xefU\xbe\xa1\x06m\xd9\a\x02\xb3\xf5f1HCL\x95\xd9x\xfd\xd8\f\xd4%\xb5e\xb1{\xf0\x88:\xb2\xf8\xea\xb18\xf6\x00\u007f\xb1\x1b[3\x16\x1d\xb5\xc5և}\x9f\xaa\xb0\xc8Z\xb0N\x85\xd7,\xc8\x13+\xc0\xa2\x19\x16W\xed\x15]\xe3թܚ\xe7֑ʮ\xf1z\xadY\x90c\xf5\\1UZQ\xb8F\xd7f5\x15W\xf3\x99\xc4o\xaa\xc8z\xfd\xda\xef\u05cc\xf3\x8f\xd7WEUUE\xed\x05\xe6q\x8e\xaa\x9bZZ-\x15\xc5ե\x95QM\xd5ӑ\x89\xa3\xea\xa1\x0ek\x9d\x8e\x912[\x055]\xe1t\f\xecX\xedSD]\xd3\x11\x90݊\xa7\xc5a\xc0\xac6\xcdt\x18\xff\xaa>\xb4y_\x9b\xff74\xf0[\x89\xd6&E3\xbb+Y\x82\xfa,\xda\xfd|\xe5`\xfe\xc1\xe7:\xe1cr\xea\xd5\xdd\xf1LEQ\xba\xf9|$\x81\x9f\xa5J\xbd&\xd1b\xe9\xc44|C\x05\x17\x985a\xd6t\xc5m\x1b\xd1\x0ev[\x16KAF=\x85\xcd\xdeg\x85\xec\x1anD\x925\x1d' \xf2̙\xb0\xb4\xb3/\x84\x83\xf3f\x1b{\x19Fғ\xf35\xc0O\xba\xc9 t>\x0f\x9d\x80keQ\xe6{\xa8,\xc2y\x1fз\xed\xc2&u\xc7*Q\xdaL\x87\x9b\x04\"6bw\xfd\x11#\xf9\x92p\x8f@\x92\xeb*mf8\"n\xa1\xf6\xf0\xe5\x81#9\xfez:i\xbf2\xaf#\xb5\xb0\xaf\x1a|\x84>\x01r\xea\xf2\x88E,\x9bΪX\xa7\x8dx\xc4O\xda߫\x11ó\xfe\x88\xde\xd5*\xb5]\r9ֺhrR\x93=mC\x80\xed\xd1K\xbd\xda\xda$\x14a;ergֹsy\x04q\xf7\xf7\x9f\xb2\xb3Y\xee\b\xf1XNLo'a\tku\"\xd9\x16q\x8a\x82OX\xbe\xdf\x1d\v\xc7\"\xb2#&\xa3\xb2\xf8˳B\xf35,T{\xab\xbc\xa4fn]\xf9\xcb\xc1\xc0 \xe01\xf3A\xf6o\xd0}\xcc署A\xd6\xdf\xfd\xe5\xcd83.\xdcCsȺ\x99\xf5?\xbd\xf6\xc7#\xea\xd5\xf8\xd5/\xab\xe66\x9a\xb3\b\xceZ'\\ec.\xf8᎐\x88\xd2U\xa6v\xafIe\xf8\xc2\t\x02\x82\xfe>\x86Ӯ\xf8ɅuQ\xb2\xfc\xd4tl3\f\xd6\xf9\xf3\xa8`\xa0\xe0YX\xbe;ǟF\x8d\xba\xe6沠\xc9\xdbs\xbcw\xbd\x82T8\\\x11\xfc\xd3\xc49\xba\x0e\xf8\x82\x8e\x19J\xbfP\x9f\xe6t\xbcf4\x0f\f\x17{\xdcM\xa1>~ܹ\x82\xcf\xf8<\xf2\xf4F\x11\x11\x87\x9bQ\u007f\xa6\x89)g(Ʈ7;J\xe2\xae\x19\xc5\a\xca#֢o\xe6\x06\xdd\a\x99j\x91\xe7\x1d\x88\xfe\xf0xL\xac\xff/\xb7>m\x94\x10M\xbf;\xe81i\xb8\x8e\x1a\xad)\x835\xba\xa4\x0e\x1eZ4;L;JR\xfb\xf0\xfaI\xbb\x00E\x92`\xe9\xeaÏ\xee\xed\u007f\xe7\xe7\xfc#\\\xee\xc7?\x13\xad|Lm\xaf\xe0\xaf\u007f;\x83\xda\xd7>\x84\x1b\xfc\xe8\xe1\u007f\x02\x00\x00\xff\xff_q\xf0n+Q\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4V͎\xe44\x10\xbe\xe7)J\xcba/$\xbd+\x0e\xa0\xdc\xd0\xc0a\x05\x8cFӫ\xb9 \x0en\xa7\xd2mƱCU\xb9\x87\x06\xf1\xee\xc8vҝN\xd2̀\x84o\xb1\xeb端~REY\x96\x85\xea\xcd\x13\x12\x1b\xefjP\xbd\xc1\xdf\x05]\xfc\xe2\xea\xf9\x1b\xae\x8c\xdf\x1c?\x16\xcf\xc655\xdc\x05\x16\xdf=\"\xfb@\x1a\xbf\xc3\xd68#ƻ\xa2CQ\x8d\x12U\x17\x00\xca9/*^s\xfc\x04\xd0\xde\tyk\x91\xca=\xba\xea9\xecp\x17\x8cm\x90\x92\xf1\xd1\xf5\xf1C\xf5u\xf5\xa1\x00ЄI\xfd\xb3\xe9\x90Eu}\r.X[\x008\xd5a\r\x8c\x14\x95DI`\xc2\xdf\x02\xb2puD\x8b\xe4+\xe3\v\xeeQG\xc7{\xf2\xa1\xaf\xe1\xf2\x90\xf5\aP9\xa0m2\xb5M\xa6\x1e\xb3\xa9\xf4j\r\xcb\x0f\xb7$~4\x83To\x03)\xbb\x0e(\t\xf0\xc1\x93\xdc_\x9c\x96\xc0L\xf9Ÿ}\xb0\x8aV\x95\v\x00־\xc7\x1a\x92n\xaf46\x05\xc0\xc0T\xb2U\x0e\\\x1c?fs\xfa\x80\x9d\xcaN\x00|\x8f\xeeۇOO_m\xaf\xae\x01\x1adM\xa6\x97\xc4\xf7Jd`\x18\x14\f(@<(\xad\x91\x19t B'\x90Q\x82q\xad\xa7.\xe5\xe8l\x1a@\xed|\x10\x90\x03\xc2S\xa2|\x88\xac:\x8b\xf4\xe4{$1#\x1b\x83ڥ\xfa&\xb73\xac\xefc8Y\n\x9aXv\xc8\xc9\xd3@\t6\x03\x03\xe0[\x90\x83a \xec\t\x19\x9d\xccQ&~ZP\x0e\xfc\xeeW\xd4R\r9\xb8S\x1d\xda;\xc5\xf8\xbf' 2\xcde$\xf6m)\x98Nѹpfm\xf20\x8e\xb9\x1b\xf9Z\xe9\xeem\x8f:f0\x92\x18\xb5Mktj\x0fh=\x81ZS\xa9ބ$i\xfcK,\xc3$\xc9hf\xf3%\xf6\xe7\xebh\xd6\xc7Iz9(\xc6\xf9\xe5\f\xd3C\x94\x99\xfb\xb7\xa6E}\xd2\x16\xb3\x89N\xd64ׯύڀ\xfc\xbf\xd9\x1b\xb7\bw\x1eY\x96J\xff\xb0騞\f\xe8\xc1\x10Pp.\xf6\xedbB& \xf3I\xbe\x901\x82\xdd\n\x9aU<\x9f\\\xeb\xd3&\xa0\xa2c%\xb9\x9fpH\xf6\xe0'\xe3Z1x;\xd7\xf9,\x87כ\b\xcd'\xfdI\xff\x9br\x1c7\x86p\xd5w\x99P\xad>D\x8fk\x8c\xaf\xf7׀2X\xabv\x16k\x10\nK\xed\xac\xab\x88\xd4i^5c\xa9]\xf6\xa9W\nh\xa1\x10\xfb\xe4\xe5\x80\xeeV7\xc0\x8b\x9aO\xf9+ϰ;\xddR\xbd;/\x87˖ʥ[C\x9cݥ\x98\x15\xce\xdeD\xcaj\xf6rI\xafn\x1e\vB\xb6S\xd9qf\\\xb5Ƹ\x88,c\xb8\ta5ً\xcbd\xbe\x99\x84\xc7\xe2I\xed\xa7\x01s\u061d\xff\xf4c \xc3H\x86?\xff*.\xd39.s\xbd`s?߂߽\xbbZgӧ\xf6\xae1y\x89\x87\x9f\u007f)\xb2cl\x9e\xc6\x1d4^\xfe\x1d\x00\x00\xff\xff;,8\xce>\f\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4VM\x8f\xdb6\x10\xbd\xebW\f\xd2C.\x95\x9c\xa0\x87\x16\xba\x05n\x0fA\xd3`\x11\xa7\xbe\x14=\xd0\xe4Ȟ.E\xb2\x9c\xa1\xdb\xed\xaf/HQ돕\xb7Y\xa0э\xc3\xe1\x9b7o>\xec\xa6m\xdbF\x05\xdabd\xf2\xae\a\x15\b\xff\x16t\xf9\xc4\xdd\xfd\x0fܑ_\x1d\xdf6\xf7\xe4L\x0f\xeb\xc4\xe2\xc7O\xc8>E\x8d?\xe2@\x8e\x84\xbckF\x14e\x94\xa8\xbe\x01P\xceyQ\xd9\xcc\xf9\b\xa0\xbd\x93\xe8\xad\xc5\xd8\xee\xd1u\xf7i\x87\xbbD\xd6`,\xe0s\xe8\xe3\x9b\xee\xfb\xeeM\x03\xa0#\x96\xe7\x9fiD\x165\x86\x1e\\\xb2\xb6\x01pj\xc4\x1e\x8eަ\x11٩\xc0\a/\xd6\xeb)XwD\x8b\xd1w\xe4\x1b\x0e\xa8s\xec}\xf4)\xf4p\xba\x98 *\xaf)\xa7mA\xdbT\xb4\x0f\x15\xad8Xb\xf9\xf9\x19\xa7\x0f\xc4R\x1c\x83MQٛ̊\x0f\x93\xdb'\xab\xe2-\xaf\x06\x80\xb5\x0f\xd8\xc3\xc7L1(\x8d\xa6\x01\xa8\xf2\x14\xca\xed,\xc0\xdb\tQ\x1fpTS.\x00>\xa0{w\xf7~\xfb\xdd\xe6\xc2\f`\x90u\xa4 E\xe4\xe5D\x80\x18\x14\xccL\xe0\xaf\x03F\x84mQ\rX|D\xae\xa4\x1fA\x01f\xfe\xdc=\x1aC\xf4\x01\xa3\xd0,\xf0\xf4\x9d\xb5י\xf5\x8a\xd7\xebL}\xf2\x02\x93\xfb\n\x19\xe4\x80s\xfahj\xb6\xe0\a\x90\x031D\f\x11\x19\x9d\x9c\xcau\xfa\xfc\x00ʁ\xdf\xfd\x81Z:\xd8`\xcc0\xc0\a\x9f\xac\xc9\xedx\xc4(\x10Q\xfb\xbd\xa3\u007f\x1e\xb1\x19ė\xa0V\t\xd6ʞ>r\x82\xd1)\vGe\x13~\v\xca\x19\x18\xd5\x03D\xccQ \xb93\xbc\xe2\xc2\x1d\xfc\xe2#\x02\xb9\xc1\xf7p\x10\tܯV{\x92y\xac\xb4\x1f\xc7\xe4H\x1eVeBh\x97\xc4G^\x19<\xa2]1\xed[\x15\xf5\x81\x04\xb5\xa4\x88+\x15\xa8-\xd4\xdd\xd4\xed\xa3\xf9&\xd6A\xe4\xd7\x17\\\xe5!w\x11K$\xb7?\xbb(\xed\xfeL\x05r\xa7O\x8d0=\x9d\xb28\t\x9dMY\x9dO?m>\xc3\x1c\xba\x14\xe3Z\xfd\xa2\xfb\xe9!\x9fJ\x90\x05#7`\x9c\x8a8D?\x16Lt&xrR\x0e\xda\x12\xbak\xf99\xedF\x92\\\xf7?\x13\xb2\xe4Zu\xb0.\xbb\x06v\b)\x18%h:x\xef`\xadF\xb4k\xc5\xf8\xd5\v\x90\x95\xe66\v\xfbe%8_\x93\xd7Γj\xe7\x03V\x97؍z-O\xf2&\xa0\xbe\x18\xa0\x8cB\x03\xd5\xc9\x1e|\xbc\xd2U\xcds\xbe\x8c\xd7]\xb8/\x0f8L;~\xa0\xfd\xb5\x15@\x19S~!\x94\xbd\xbb\xf9\xf6\x19\xc1\x16\xf2^\x97H\xb9Q\a\x1f3\xa3#\x19\x8c\xed\x9cge\x92bM\x98\xd0\x1a\xee\x9e@\xdeм&Y \x9fҼ\xe0qW\xdd2\x93,\xf4\xfcl\xdaPX\x17fY\x9fj\x8f\xb7\x18,d\x9c;\x9c\"^\xcdj\xfb\x18\xe0\x8bzG\x94$~y\xf7\x94g\xd5sW;H\xa7\x18\xd1I\xc5\\ش\xffO\a\x85\x83b\xfc\x0f͗#\xdc\xe5\x97s\x19,\r\xa8\x1f\xb4\xc5\t\x10\xfc\xb0\xd0m/\xa2\x9c?ti|ʭ\x85wGEV\xed,.\xdc\xfd\xea\xd4\xcdۛ\xc5_\xac\xe7\x13#\xe7ujz\x90\x98&\xec\xdae\xd5r\xaa\xbe\xd2\x1a\x83\xa0\xf9x\xfd\xaf\xe7ի\x8b?.娽\x9b\x86\x95{\xf8\xed\xf7fBE\xb3\x9d\xff\x81d\xe3\xbf\x01\x00\x00\xff\xff\xbf\xca\xff\xa71\n\x00\x00"), } var CRDs = crds() diff --git a/config/crd/v1beta1/bases/velero.io_backups.yaml b/config/crd/v1beta1/bases/velero.io_backups.yaml deleted file mode 100644 index 860731487..000000000 --- a/config/crd/v1beta1/bases/velero.io_backups.yaml +++ /dev/null @@ -1,439 +0,0 @@ - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.3.0 - creationTimestamp: null - name: backups.velero.io -spec: - group: velero.io - names: - kind: Backup - listKind: BackupList - plural: backups - singular: backup - preserveUnknownFields: false - scope: Namespaced - validation: - openAPIV3Schema: - description: Backup is a Velero resource that represents the capture of Kubernetes - cluster state at a point in time (API objects and associated volume state). - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BackupSpec defines the specification for a Velero backup. - properties: - defaultVolumesToRestic: - description: DefaultVolumesToRestic specifies whether restic should - be used to take a backup of all pod volumes by default. - type: boolean - excludedNamespaces: - description: ExcludedNamespaces contains a list of namespaces that are - not included in the backup. - items: - type: string - nullable: true - type: array - excludedResources: - description: ExcludedResources is a slice of resource names that are - not included in the backup. - items: - type: string - nullable: true - type: array - hooks: - description: Hooks represent custom behaviors that should be executed - at different phases of the backup. - properties: - resources: - description: Resources are hooks that should be executed when backing - up individual instances of a resource. - items: - description: BackupResourceHookSpec defines one or more BackupResourceHooks - that should be executed based on the rules defined for namespaces, - resources, and label selector. - properties: - excludedNamespaces: - description: ExcludedNamespaces specifies the namespaces to - which this hook spec does not apply. - items: - type: string - nullable: true - type: array - excludedResources: - description: ExcludedResources specifies the resources to - which this hook spec does not apply. - items: - type: string - nullable: true - type: array - includedNamespaces: - description: IncludedNamespaces specifies the namespaces to - which this hook spec applies. If empty, it applies to all - namespaces. - items: - type: string - nullable: true - type: array - includedResources: - description: IncludedResources specifies the resources to - which this hook spec applies. If empty, it applies to all - resources. - items: - type: string - nullable: true - type: array - labelSelector: - description: LabelSelector, if specified, filters the resources - to which this hook spec applies. - nullable: true - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector - that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: operator represents a key's relationship - to a set of values. Valid operators are In, NotIn, - Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. - If the operator is In or NotIn, the values array - must be non-empty. If the operator is Exists or - DoesNotExist, the values array must be empty. - This array is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. - A single {key,value} in the matchLabels map is equivalent - to an element of matchExpressions, whose key field is - "key", the operator is "In", and the values array contains - only "value". The requirements are ANDed. - type: object - type: object - name: - description: Name is the name of this hook. - type: string - post: - description: PostHooks is a list of BackupResourceHooks to - execute after storing the item in the backup. These are - executed after all "additional items" from item actions - are processed. - items: - description: BackupResourceHook defines a hook for a resource. - properties: - exec: - description: Exec defines an exec hook. - properties: - command: - description: Command is the command and arguments - to execute. - items: - type: string - minItems: 1 - type: array - container: - description: Container is the container in the pod - where the command should be executed. If not specified, - the pod's first container is used. - type: string - onError: - description: OnError specifies how Velero should - behave if it encounters an error executing this - hook. - enum: - - Continue - - Fail - type: string - timeout: - description: Timeout defines the maximum amount - of time Velero should wait for the hook to complete - before considering the execution a failure. - type: string - required: - - command - type: object - required: - - exec - type: object - type: array - pre: - description: PreHooks is a list of BackupResourceHooks to - execute prior to storing the item in the backup. These are - executed before any "additional items" from item actions - are processed. - items: - description: BackupResourceHook defines a hook for a resource. - properties: - exec: - description: Exec defines an exec hook. - properties: - command: - description: Command is the command and arguments - to execute. - items: - type: string - minItems: 1 - type: array - container: - description: Container is the container in the pod - where the command should be executed. If not specified, - the pod's first container is used. - type: string - onError: - description: OnError specifies how Velero should - behave if it encounters an error executing this - hook. - enum: - - Continue - - Fail - type: string - timeout: - description: Timeout defines the maximum amount - of time Velero should wait for the hook to complete - before considering the execution a failure. - type: string - required: - - command - type: object - required: - - exec - type: object - type: array - required: - - name - type: object - nullable: true - type: array - type: object - includeClusterResources: - description: IncludeClusterResources specifies whether cluster-scoped - resources should be included for consideration in the backup. - nullable: true - type: boolean - includedNamespaces: - description: IncludedNamespaces is a slice of namespace names to include - objects from. If empty, all namespaces are included. - items: - type: string - nullable: true - type: array - includedResources: - description: IncludedResources is a slice of resource names to include - in the backup. If empty, all resources are included. - items: - type: string - nullable: true - type: array - labelSelector: - description: LabelSelector is a metav1.LabelSelector to filter with - when adding individual objects to the backup. If empty or nil, all - objects are included. Optional. - nullable: true - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains - values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: operator represents a key's relationship to a - set of values. Valid operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator - is In or NotIn, the values array must be non-empty. If the - operator is Exists or DoesNotExist, the values array must - be empty. This array is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator is - "In", and the values array contains only "value". The requirements - are ANDed. - type: object - type: object - metadata: - properties: - labels: - additionalProperties: - type: string - type: object - type: object - orderedResources: - additionalProperties: - type: string - description: OrderedResources specifies the backup order of resources - of specific Kind. The map key is the Kind name and value is a list - of resource names separated by commas. Each resource name has format - "namespace/resourcename". For cluster resources, simply use "resourcename". - nullable: true - type: object - snapshotVolumes: - description: SnapshotVolumes specifies whether to take cloud snapshots - of any PV's referenced in the set of objects included in the Backup. - nullable: true - type: boolean - storageLocation: - description: StorageLocation is a string containing the name of a BackupStorageLocation - where the backup should be stored. - type: string - ttl: - description: TTL is a time.Duration-parseable string describing how - long the Backup should be retained for. - type: string - volumeSnapshotLocations: - description: VolumeSnapshotLocations is a list containing names of VolumeSnapshotLocations - associated with this backup. - items: - type: string - type: array - type: object - status: - description: BackupStatus captures the current status of a Velero backup. - properties: - completionTimestamp: - description: CompletionTimestamp records the time a backup was completed. - Completion time is recorded even on failed backups. Completion time - is recorded before uploading the backup object. The server's time - is used for CompletionTimestamps - format: date-time - nullable: true - type: string - errors: - description: Errors is a count of all error messages that were generated - during execution of the backup. The actual errors are in the backup's - log file in object storage. - type: integer - expiration: - description: Expiration is when this Backup is eligible for garbage-collection. - format: date-time - nullable: true - type: string - formatVersion: - description: FormatVersion is the backup format version, including major, - minor, and patch version. - type: string - phase: - description: Phase is the current state of the Backup. - enum: - - New - - FailedValidation - - InProgress - - Completed - - PartiallyFailed - - Failed - - Deleting - type: string - progress: - description: Progress contains information about the backup's execution - progress. Note that this information is best-effort only -- if Velero - fails to update it during a backup for any reason, it may be inaccurate/stale. - nullable: true - properties: - itemsBackedUp: - description: ItemsBackedUp is the number of items that have actually - been written to the backup tarball so far. - type: integer - totalItems: - description: TotalItems is the total number of items to be backed - up. This number may change throughout the execution of the backup - due to plugins that return additional related items to back up, - the velero.io/exclude-from-backup label, and various other filters - that happen as items are processed. - type: integer - type: object - startTimestamp: - description: StartTimestamp records the time a backup was started. Separate - from CreationTimestamp, since that value changes on restores. The - server's time is used for StartTimestamps - format: date-time - nullable: true - type: string - validationErrors: - description: ValidationErrors is a slice of all validation errors (if - applicable). - items: - type: string - nullable: true - type: array - version: - description: 'Version is the backup format major version. Deprecated: - Please see FormatVersion' - type: integer - volumeSnapshotsAttempted: - description: VolumeSnapshotsAttempted is the total number of attempted - volume snapshots for this backup. - type: integer - volumeSnapshotsCompleted: - description: VolumeSnapshotsCompleted is the total number of successfully - completed volume snapshots for this backup. - type: integer - warnings: - description: Warnings is a count of all warning messages that were generated - during execution of the backup. The actual warnings are in the backup's - log file in object storage. - type: integer - type: object - type: object - version: v1 - versions: - - name: v1 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/crd/v1beta1/bases/velero.io_backupstoragelocations.yaml b/config/crd/v1beta1/bases/velero.io_backupstoragelocations.yaml deleted file mode 100644 index 86bc003e0..000000000 --- a/config/crd/v1beta1/bases/velero.io_backupstoragelocations.yaml +++ /dev/null @@ -1,179 +0,0 @@ - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.3.0 - creationTimestamp: null - name: backupstoragelocations.velero.io -spec: - additionalPrinterColumns: - - JSONPath: .status.phase - description: Backup Storage Location status such as Available/Unavailable - name: Phase - type: string - - JSONPath: .status.lastValidationTime - description: LastValidationTime is the last time the backup store location was - validated - name: Last Validated - type: date - - JSONPath: .metadata.creationTimestamp - name: Age - type: date - - JSONPath: .spec.default - description: Default backup storage location - name: Default - type: boolean - group: velero.io - names: - kind: BackupStorageLocation - listKind: BackupStorageLocationList - plural: backupstoragelocations - shortNames: - - bsl - singular: backupstoragelocation - preserveUnknownFields: false - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - description: BackupStorageLocation is a location where Velero stores backup - objects - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BackupStorageLocationSpec defines the desired state of a Velero - BackupStorageLocation - properties: - accessMode: - description: AccessMode defines the permissions for the backup storage - location. - enum: - - ReadOnly - - ReadWrite - type: string - backupSyncPeriod: - description: BackupSyncPeriod defines how frequently to sync backup - API objects from object storage. A value of 0 disables sync. - nullable: true - type: string - config: - additionalProperties: - type: string - description: Config is for provider-specific configuration fields. - type: object - credential: - description: Credential contains the credential information intended - to be used with this location - properties: - key: - description: The key of the secret to select from. Must be a valid - secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - key - type: object - default: - description: Default indicates this location is the default backup storage - location. - type: boolean - objectStorage: - description: ObjectStorageLocation specifies the settings necessary - to connect to a provider's object storage. - properties: - bucket: - description: Bucket is the bucket to use for object storage. - type: string - caCert: - description: CACert defines a CA bundle to use when verifying TLS - connections to the provider. - format: byte - type: string - prefix: - description: Prefix is the path inside a bucket to use for Velero - storage. Optional. - type: string - required: - - bucket - type: object - provider: - description: Provider is the provider of the backup storage. - type: string - validationFrequency: - description: ValidationFrequency defines how frequently to validate - the corresponding object storage. A value of 0 disables validation. - nullable: true - type: string - required: - - objectStorage - - provider - type: object - status: - description: BackupStorageLocationStatus defines the observed state of BackupStorageLocation - properties: - accessMode: - description: "AccessMode is an unused field. \n Deprecated: there is - now an AccessMode field on the Spec and this field will be removed - entirely as of v2.0." - enum: - - ReadOnly - - ReadWrite - type: string - lastSyncedRevision: - description: "LastSyncedRevision is the value of the `metadata/revision` - file in the backup storage location the last time the BSL's contents - were synced into the cluster. \n Deprecated: this field is no longer - updated or used for detecting changes to the location's contents and - will be removed entirely in v2.0." - type: string - lastSyncedTime: - description: LastSyncedTime is the last time the contents of the location - were synced into the cluster. - format: date-time - nullable: true - type: string - lastValidationTime: - description: LastValidationTime is the last time the backup store location - was validated the cluster. - format: date-time - nullable: true - type: string - phase: - description: Phase is the current state of the BackupStorageLocation. - enum: - - Available - - Unavailable - type: string - type: object - type: object - version: v1 - versions: - - name: v1 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/crd/v1beta1/bases/velero.io_deletebackuprequests.yaml b/config/crd/v1beta1/bases/velero.io_deletebackuprequests.yaml deleted file mode 100644 index 05ecbdfc2..000000000 --- a/config/crd/v1beta1/bases/velero.io_deletebackuprequests.yaml +++ /dev/null @@ -1,73 +0,0 @@ - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.3.0 - creationTimestamp: null - name: deletebackuprequests.velero.io -spec: - group: velero.io - names: - kind: DeleteBackupRequest - listKind: DeleteBackupRequestList - plural: deletebackuprequests - singular: deletebackuprequest - preserveUnknownFields: false - scope: Namespaced - validation: - openAPIV3Schema: - description: DeleteBackupRequest is a request to delete one or more backups. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: DeleteBackupRequestSpec is the specification for which backups - to delete. - properties: - backupName: - type: string - required: - - backupName - type: object - status: - description: DeleteBackupRequestStatus is the current status of a DeleteBackupRequest. - properties: - errors: - description: Errors contains any errors that were encountered during - the deletion process. - items: - type: string - nullable: true - type: array - phase: - description: Phase is the current state of the DeleteBackupRequest. - enum: - - New - - InProgress - - Processed - type: string - type: object - type: object - version: v1 - versions: - - name: v1 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/crd/v1beta1/bases/velero.io_downloadrequests.yaml b/config/crd/v1beta1/bases/velero.io_downloadrequests.yaml deleted file mode 100644 index e7b4832d7..000000000 --- a/config/crd/v1beta1/bases/velero.io_downloadrequests.yaml +++ /dev/null @@ -1,96 +0,0 @@ - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.3.0 - creationTimestamp: null - name: downloadrequests.velero.io -spec: - group: velero.io - names: - kind: DownloadRequest - listKind: DownloadRequestList - plural: downloadrequests - singular: downloadrequest - preserveUnknownFields: false - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - description: DownloadRequest is a request to download an artifact from backup - object storage, such as a backup log file. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: DownloadRequestSpec is the specification for a download request. - properties: - target: - description: Target is what to download (e.g. logs for a backup). - properties: - kind: - description: Kind is the type of file to download. - enum: - - BackupLog - - BackupContents - - BackupVolumeSnapshots - - BackupResourceList - - RestoreLog - - RestoreResults - type: string - name: - description: Name is the name of the kubernetes resource with which - the file is associated. - type: string - required: - - kind - - name - type: object - required: - - target - type: object - status: - description: DownloadRequestStatus is the current status of a DownloadRequest. - properties: - downloadURL: - description: DownloadURL contains the pre-signed URL for the target - file. - type: string - expiration: - description: Expiration is when this DownloadRequest expires and can - be deleted by the system. - format: date-time - nullable: true - type: string - phase: - description: Phase is the current state of the DownloadRequest. - enum: - - New - - Processed - type: string - type: object - type: object - version: v1 - versions: - - name: v1 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/crd/v1beta1/bases/velero.io_podvolumebackups.yaml b/config/crd/v1beta1/bases/velero.io_podvolumebackups.yaml deleted file mode 100644 index 094dd3e16..000000000 --- a/config/crd/v1beta1/bases/velero.io_podvolumebackups.yaml +++ /dev/null @@ -1,162 +0,0 @@ - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.3.0 - creationTimestamp: null - name: podvolumebackups.velero.io -spec: - group: velero.io - names: - kind: PodVolumeBackup - listKind: PodVolumeBackupList - plural: podvolumebackups - singular: podvolumebackup - preserveUnknownFields: false - scope: Namespaced - validation: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: PodVolumeBackupSpec is the specification for a PodVolumeBackup. - properties: - backupStorageLocation: - description: BackupStorageLocation is the name of the backup storage - location where the restic repository is stored. - type: string - node: - description: Node is the name of the node that the Pod is running on. - type: string - pod: - description: Pod is a reference to the pod containing the volume to - be backed up. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: 'If referring to a piece of an object instead of an - entire object, this string should contain a valid JSON/Go field - access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen only - to have some well-defined way of referencing a part of an object. - TODO: this design is not final and this field is subject to change - in the future.' - type: string - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' - type: string - resourceVersion: - description: 'Specific resourceVersion to which this reference is - made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' - type: string - uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' - type: string - type: object - repoIdentifier: - description: RepoIdentifier is the restic repository identifier. - type: string - tags: - additionalProperties: - type: string - description: Tags are a map of key-value pairs that should be applied - to the volume backup as tags. - type: object - volume: - description: Volume is the name of the volume within the Pod to be backed - up. - type: string - required: - - backupStorageLocation - - node - - pod - - repoIdentifier - - volume - type: object - status: - description: PodVolumeBackupStatus is the current status of a PodVolumeBackup. - properties: - completionTimestamp: - description: CompletionTimestamp records the time a backup was completed. - Completion time is recorded even on failed backups. Completion time - is recorded before uploading the backup object. The server's time - is used for CompletionTimestamps - format: date-time - nullable: true - type: string - message: - description: Message is a message about the pod volume backup's status. - type: string - path: - description: Path is the full path within the controller pod being backed - up. - type: string - phase: - description: Phase is the current state of the PodVolumeBackup. - enum: - - New - - InProgress - - Completed - - Failed - type: string - progress: - description: Progress holds the total number of bytes of the volume - and the current number of backed up bytes. This can be used to display - progress information about the backup operation. - properties: - bytesDone: - format: int64 - type: integer - totalBytes: - format: int64 - type: integer - type: object - snapshotID: - description: SnapshotID is the identifier for the snapshot of the pod - volume. - type: string - startTimestamp: - description: StartTimestamp records the time a backup was started. Separate - from CreationTimestamp, since that value changes on restores. The - server's time is used for StartTimestamps - format: date-time - nullable: true - type: string - type: object - type: object - version: v1 - versions: - - name: v1 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/crd/v1beta1/bases/velero.io_podvolumerestores.yaml b/config/crd/v1beta1/bases/velero.io_podvolumerestores.yaml deleted file mode 100644 index 43a738e5f..000000000 --- a/config/crd/v1beta1/bases/velero.io_podvolumerestores.yaml +++ /dev/null @@ -1,145 +0,0 @@ - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.3.0 - creationTimestamp: null - name: podvolumerestores.velero.io -spec: - group: velero.io - names: - kind: PodVolumeRestore - listKind: PodVolumeRestoreList - plural: podvolumerestores - singular: podvolumerestore - preserveUnknownFields: false - scope: Namespaced - validation: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: PodVolumeRestoreSpec is the specification for a PodVolumeRestore. - properties: - backupStorageLocation: - description: BackupStorageLocation is the name of the backup storage - location where the restic repository is stored. - type: string - pod: - description: Pod is a reference to the pod containing the volume to - be restored. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: 'If referring to a piece of an object instead of an - entire object, this string should contain a valid JSON/Go field - access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen only - to have some well-defined way of referencing a part of an object. - TODO: this design is not final and this field is subject to change - in the future.' - type: string - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' - type: string - resourceVersion: - description: 'Specific resourceVersion to which this reference is - made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' - type: string - uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' - type: string - type: object - repoIdentifier: - description: RepoIdentifier is the restic repository identifier. - type: string - snapshotID: - description: SnapshotID is the ID of the volume snapshot to be restored. - type: string - volume: - description: Volume is the name of the volume within the Pod to be restored. - type: string - required: - - backupStorageLocation - - pod - - repoIdentifier - - snapshotID - - volume - type: object - status: - description: PodVolumeRestoreStatus is the current status of a PodVolumeRestore. - properties: - completionTimestamp: - description: CompletionTimestamp records the time a restore was completed. - Completion time is recorded even on failed restores. The server's - time is used for CompletionTimestamps - format: date-time - nullable: true - type: string - message: - description: Message is a message about the pod volume restore's status. - type: string - phase: - description: Phase is the current state of the PodVolumeRestore. - enum: - - New - - InProgress - - Completed - - Failed - type: string - progress: - description: Progress holds the total number of bytes of the snapshot - and the current number of restored bytes. This can be used to display - progress information about the restore operation. - properties: - bytesDone: - format: int64 - type: integer - totalBytes: - format: int64 - type: integer - type: object - startTimestamp: - description: StartTimestamp records the time a restore was started. - The server's time is used for StartTimestamps - format: date-time - nullable: true - type: string - type: object - type: object - version: v1 - versions: - - name: v1 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/crd/v1beta1/bases/velero.io_resticrepositories.yaml b/config/crd/v1beta1/bases/velero.io_resticrepositories.yaml deleted file mode 100644 index 727e66bdf..000000000 --- a/config/crd/v1beta1/bases/velero.io_resticrepositories.yaml +++ /dev/null @@ -1,89 +0,0 @@ - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.3.0 - creationTimestamp: null - name: resticrepositories.velero.io -spec: - group: velero.io - names: - kind: ResticRepository - listKind: ResticRepositoryList - plural: resticrepositories - singular: resticrepository - preserveUnknownFields: false - scope: Namespaced - validation: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: ResticRepositorySpec is the specification for a ResticRepository. - properties: - backupStorageLocation: - description: BackupStorageLocation is the name of the BackupStorageLocation - that should contain this repository. - type: string - maintenanceFrequency: - description: MaintenanceFrequency is how often maintenance should be - run. - type: string - resticIdentifier: - description: ResticIdentifier is the full restic-compatible string for - identifying this repository. - type: string - volumeNamespace: - description: VolumeNamespace is the namespace this restic repository - contains pod volume backups for. - type: string - required: - - backupStorageLocation - - maintenanceFrequency - - resticIdentifier - - volumeNamespace - type: object - status: - description: ResticRepositoryStatus is the current status of a ResticRepository. - properties: - lastMaintenanceTime: - description: LastMaintenanceTime is the last time maintenance was run. - format: date-time - nullable: true - type: string - message: - description: Message is a message about the current status of the ResticRepository. - type: string - phase: - description: Phase is the current state of the ResticRepository. - enum: - - New - - Ready - - NotReady - type: string - type: object - type: object - version: v1 - versions: - - name: v1 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/crd/v1beta1/bases/velero.io_restores.yaml b/config/crd/v1beta1/bases/velero.io_restores.yaml deleted file mode 100644 index b900f8b13..000000000 --- a/config/crd/v1beta1/bases/velero.io_restores.yaml +++ /dev/null @@ -1,1670 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.3.0 - creationTimestamp: null - name: restores.velero.io -spec: - group: velero.io - names: - kind: Restore - listKind: RestoreList - plural: restores - singular: restore - preserveUnknownFields: false - scope: Namespaced - validation: - openAPIV3Schema: - description: Restore is a Velero resource that represents the application of - resources from a Velero backup to a target Kubernetes cluster. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: RestoreSpec defines the specification for a Velero restore. - properties: - backupName: - description: BackupName is the unique name of the Velero backup to restore - from. - type: string - excludedNamespaces: - description: ExcludedNamespaces contains a list of namespaces that are - not included in the restore. - items: - type: string - nullable: true - type: array - excludedResources: - description: ExcludedResources is a slice of resource names that are - not included in the restore. - items: - type: string - nullable: true - type: array - hooks: - description: Hooks represent custom behaviors that should be executed - during or post restore. - properties: - resources: - items: - description: RestoreResourceHookSpec defines one or more RestoreResrouceHooks - that should be executed based on the rules defined for namespaces, - resources, and label selector. - properties: - excludedNamespaces: - description: ExcludedNamespaces specifies the namespaces to - which this hook spec does not apply. - items: - type: string - nullable: true - type: array - excludedResources: - description: ExcludedResources specifies the resources to - which this hook spec does not apply. - items: - type: string - nullable: true - type: array - includedNamespaces: - description: IncludedNamespaces specifies the namespaces to - which this hook spec applies. If empty, it applies to all - namespaces. - items: - type: string - nullable: true - type: array - includedResources: - description: IncludedResources specifies the resources to - which this hook spec applies. If empty, it applies to all - resources. - items: - type: string - nullable: true - type: array - labelSelector: - description: LabelSelector, if specified, filters the resources - to which this hook spec applies. - nullable: true - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector - that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: operator represents a key's relationship - to a set of values. Valid operators are In, NotIn, - Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. - If the operator is In or NotIn, the values array - must be non-empty. If the operator is Exists or - DoesNotExist, the values array must be empty. - This array is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. - A single {key,value} in the matchLabels map is equivalent - to an element of matchExpressions, whose key field is - "key", the operator is "In", and the values array contains - only "value". The requirements are ANDed. - type: object - type: object - name: - description: Name is the name of this hook. - type: string - postHooks: - description: PostHooks is a list of RestoreResourceHooks to - execute during and after restoring a resource. - items: - description: RestoreResourceHook defines a restore hook - for a resource. - properties: - exec: - description: Exec defines an exec restore hook. - properties: - command: - description: Command is the command and arguments - to execute from within a container after a pod - has been restored. - items: - type: string - minItems: 1 - type: array - container: - description: Container is the container in the pod - where the command should be executed. If not specified, - the pod's first container is used. - type: string - execTimeout: - description: ExecTimeout defines the maximum amount - of time Velero should wait for the hook to complete - before considering the execution a failure. - type: string - onError: - description: OnError specifies how Velero should - behave if it encounters an error executing this - hook. - enum: - - Continue - - Fail - type: string - waitTimeout: - description: WaitTimeout defines the maximum amount - of time Velero should wait for the container to - be Ready before attempting to run the command. - type: string - required: - - command - type: object - init: - description: Init defines an init restore hook. - properties: - initContainers: - description: InitContainers is list of init containers - to be added to a pod during its restore. - items: - description: A single application container that - you want to run within a pod. - properties: - args: - description: 'Arguments to the entrypoint. - The docker image''s CMD is used if this - is not provided. Variable references $(VAR_NAME) - are expanded using the container''s environment. - If a variable cannot be resolved, the reference - in the input string will be unchanged. The - $(VAR_NAME) syntax can be escaped with a - double $$, ie: $$(VAR_NAME). Escaped references - will never be expanded, regardless of whether - the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' - items: - type: string - type: array - command: - description: 'Entrypoint array. Not executed - within a shell. The docker image''s ENTRYPOINT - is used if this is not provided. Variable - references $(VAR_NAME) are expanded using - the container''s environment. If a variable - cannot be resolved, the reference in the - input string will be unchanged. The $(VAR_NAME) - syntax can be escaped with a double $$, - ie: $$(VAR_NAME). Escaped references will - never be expanded, regardless of whether - the variable exists or not. Cannot be updated. - More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' - items: - type: string - type: array - env: - description: List of environment variables - to set in the container. Cannot be updated. - items: - description: EnvVar represents an environment - variable present in a Container. - properties: - name: - description: Name of the environment - variable. Must be a C_IDENTIFIER. - type: string - value: - description: 'Variable references $(VAR_NAME) - are expanded using the previous defined - environment variables in the container - and any service environment variables. - If a variable cannot be resolved, - the reference in the input string - will be unchanged. The $(VAR_NAME) - syntax can be escaped with a double - $$, ie: $$(VAR_NAME). Escaped references - will never be expanded, regardless - of whether the variable exists or - not. Defaults to "".' - type: string - valueFrom: - description: Source for the environment - variable's value. Cannot be used if - value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a - ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. - apiVersion, kind, uid?' - type: string - optional: - description: Specify whether - the ConfigMap or its key must - be defined - type: boolean - required: - - key - type: object - fieldRef: - description: 'Selects a field of - the pod: supports metadata.name, - metadata.namespace, `metadata.labels['''']`, - `metadata.annotations['''']`, - spec.nodeName, spec.serviceAccountName, - status.hostIP, status.podIP, status.podIPs.' - properties: - apiVersion: - description: Version of the - schema the FieldPath is written - in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field - to select in the specified - API version. - type: string - required: - - fieldPath - type: object - resourceFieldRef: - description: 'Selects a resource - of the container: only resources - limits and requests (limits.cpu, - limits.memory, limits.ephemeral-storage, - requests.cpu, requests.memory - and requests.ephemeral-storage) - are currently supported.' - properties: - containerName: - description: 'Container name: - required for volumes, optional - for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource - to select' - type: string - required: - - resource - type: object - secretKeyRef: - description: Selects a key of a - secret in the pod's namespace - properties: - key: - description: The key of the - secret to select from. Must - be a valid secret key. - type: string - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. - apiVersion, kind, uid?' - type: string - optional: - description: Specify whether - the Secret or its key must - be defined - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - description: List of sources to populate environment - variables in the container. The keys defined - within a source must be a C_IDENTIFIER. - All invalid keys will be reported as an - event when the container is starting. When - a key exists in multiple sources, the value - associated with the last source will take - precedence. Values defined by an Env with - a duplicate key will take precedence. Cannot - be updated. - items: - description: EnvFromSource represents the - source of a set of ConfigMaps - properties: - configMapRef: - description: The ConfigMap to select - from - properties: - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. - apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the - ConfigMap must be defined - type: boolean - type: object - prefix: - description: An optional identifier - to prepend to each key in the ConfigMap. - Must be a C_IDENTIFIER. - type: string - secretRef: - description: The Secret to select from - properties: - name: - description: 'Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. - apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the - Secret must be defined - type: boolean - type: object - type: object - type: array - image: - description: 'Docker image name. More info: - https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level - config management to default or override - container images in workload controllers - like Deployments and StatefulSets.' - type: string - imagePullPolicy: - description: 'Image pull policy. One of Always, - Never, IfNotPresent. Defaults to Always - if :latest tag is specified, or IfNotPresent - otherwise. Cannot be updated. More info: - https://kubernetes.io/docs/concepts/containers/images#updating-images' - type: string - lifecycle: - description: Actions that the management system - should take in response to container lifecycle - events. Cannot be updated. - properties: - postStart: - description: 'PostStart is called immediately - after a container is created. If the - handler fails, the container is terminated - and restarted according to its restart - policy. Other management of the container - blocks until the hook completes. More - info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' - properties: - exec: - description: One and only one of the - following should be specified. Exec - specifies the action to take. - properties: - command: - description: Command is the command - line to execute inside the container, - the working directory for the - command is root ('/') in the - container's filesystem. The - command is simply exec'd, it - is not run inside a shell, so - traditional shell instructions - ('|', etc) won't work. To use - a shell, you need to explicitly - call out to that shell. Exit - status of 0 is treated as live/healthy - and non-zero is unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the - http request to perform. - properties: - host: - description: Host name to connect - to, defaults to the pod IP. - You probably want to set "Host" - in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to - set in the request. HTTP allows - repeated headers. - items: - description: HTTPHeader describes - a custom header to be used - in HTTP probes - properties: - name: - description: The header - field name - type: string - value: - description: The header - field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on - the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of - the port to access on the container. - Number must be in the range - 1 to 65535. Name must be an - IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for - connecting to the host. Defaults - to HTTP. - type: string - required: - - port - type: object - tcpSocket: - description: 'TCPSocket specifies - an action involving a TCP port. - TCP hooks not yet supported TODO: - implement a realistic TCP lifecycle - hook' - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the - pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of - the port to access on the container. - Number must be in the range - 1 to 65535. Name must be an - IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - description: 'PreStop is called immediately - before a container is terminated due - to an API request or management event - such as liveness/startup probe failure, - preemption, resource contention, etc. - The handler is not called if the container - crashes or exits. The reason for termination - is passed to the handler. The Pod''s - termination grace period countdown begins - before the PreStop hooked is executed. - Regardless of the outcome of the handler, - the container will eventually terminate - within the Pod''s termination grace - period. Other management of the container - blocks until the hook completes or until - the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' - properties: - exec: - description: One and only one of the - following should be specified. Exec - specifies the action to take. - properties: - command: - description: Command is the command - line to execute inside the container, - the working directory for the - command is root ('/') in the - container's filesystem. The - command is simply exec'd, it - is not run inside a shell, so - traditional shell instructions - ('|', etc) won't work. To use - a shell, you need to explicitly - call out to that shell. Exit - status of 0 is treated as live/healthy - and non-zero is unhealthy. - items: - type: string - type: array - type: object - httpGet: - description: HTTPGet specifies the - http request to perform. - properties: - host: - description: Host name to connect - to, defaults to the pod IP. - You probably want to set "Host" - in httpHeaders instead. - type: string - httpHeaders: - description: Custom headers to - set in the request. HTTP allows - repeated headers. - items: - description: HTTPHeader describes - a custom header to be used - in HTTP probes - properties: - name: - description: The header - field name - type: string - value: - description: The header - field value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on - the HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of - the port to access on the container. - Number must be in the range - 1 to 65535. Name must be an - IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for - connecting to the host. Defaults - to HTTP. - type: string - required: - - port - type: object - tcpSocket: - description: 'TCPSocket specifies - an action involving a TCP port. - TCP hooks not yet supported TODO: - implement a realistic TCP lifecycle - hook' - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the - pod IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of - the port to access on the container. - Number must be in the range - 1 to 65535. Name must be an - IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - description: 'Periodic probe of container - liveness. Container will be restarted if - the probe fails. Cannot be updated. More - info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - properties: - exec: - description: One and only one of the following - should be specified. Exec specifies - the action to take. - properties: - command: - description: Command is the command - line to execute inside the container, - the working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it - is not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to - explicitly call out to that shell. - Exit status of 0 is treated as live/healthy - and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures - for the probe to be considered failed - after having succeeded. Defaults to - 3. Minimum value is 1. - format: int32 - type: integer - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: Host name to connect - to, defaults to the pod IP. You - probably want to set "Host" in httpHeaders - instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: The header field - name - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the - port to access on the container. - Number must be in the range 1 to - 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after - the container has started before liveness - probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to - perform the probe. Default to 10 seconds. - Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes - for the probe to be considered successful - after having failed. Defaults to 1. - Must be 1 for liveness and startup. - Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not - yet supported TODO: implement a realistic - TCP lifecycle hook' - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the - port to access on the container. - Number must be in the range 1 to - 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - description: 'Number of seconds after - which the probe times out. Defaults - to 1 second. Minimum value is 1. More - info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - name: - description: Name of the container specified - as a DNS_LABEL. Each container in a pod - must have a unique name (DNS_LABEL). Cannot - be updated. - type: string - ports: - description: List of ports to expose from - the container. Exposing a port here gives - the system additional information about - the network connections a container uses, - but is primarily informational. Not specifying - a port here DOES NOT prevent that port from - being exposed. Any port which is listening - on the default "0.0.0.0" address inside - a container will be accessible from the - network. Cannot be updated. - items: - description: ContainerPort represents a - network port in a single container. - properties: - containerPort: - description: Number of port to expose - on the pod's IP address. This must - be a valid port number, 0 < x < 65536. - format: int32 - type: integer - hostIP: - description: What host IP to bind the - external port to. - type: string - hostPort: - description: Number of port to expose - on the host. If specified, this must - be a valid port number, 0 < x < 65536. - If HostNetwork is specified, this - must match ContainerPort. Most containers - do not need this. - format: int32 - type: integer - name: - description: If specified, this must - be an IANA_SVC_NAME and unique within - the pod. Each named port in a pod - must have a unique name. Name for - the port that can be referred to by - services. - type: string - protocol: - description: Protocol for port. Must - be UDP, TCP, or SCTP. Defaults to - "TCP". - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - description: 'Periodic probe of container - service readiness. Container will be removed - from service endpoints if the probe fails. - Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - properties: - exec: - description: One and only one of the following - should be specified. Exec specifies - the action to take. - properties: - command: - description: Command is the command - line to execute inside the container, - the working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it - is not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to - explicitly call out to that shell. - Exit status of 0 is treated as live/healthy - and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures - for the probe to be considered failed - after having succeeded. Defaults to - 3. Minimum value is 1. - format: int32 - type: integer - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: Host name to connect - to, defaults to the pod IP. You - probably want to set "Host" in httpHeaders - instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: The header field - name - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the - port to access on the container. - Number must be in the range 1 to - 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after - the container has started before liveness - probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to - perform the probe. Default to 10 seconds. - Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes - for the probe to be considered successful - after having failed. Defaults to 1. - Must be 1 for liveness and startup. - Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not - yet supported TODO: implement a realistic - TCP lifecycle hook' - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the - port to access on the container. - Number must be in the range 1 to - 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - description: 'Number of seconds after - which the probe times out. Defaults - to 1 second. Minimum value is 1. More - info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - resources: - description: 'Compute Resources required by - this container. Cannot be updated. More - info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum - amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum - amount of compute resources required. - If Requests is omitted for a container, - it defaults to Limits if that is explicitly - specified, otherwise to an implementation-defined - value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - type: object - securityContext: - description: 'Security options the pod should - run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' - properties: - allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation - controls whether a process can gain - more privileges than its parent process. - This bool directly controls if the no_new_privs - flag will be set on the container process. - AllowPrivilegeEscalation is true always - when the container is: 1) run as Privileged - 2) has CAP_SYS_ADMIN' - type: boolean - capabilities: - description: The capabilities to add/drop - when running containers. Defaults to - the default set of capabilities granted - by the container runtime. - properties: - add: - description: Added capabilities - items: - description: Capability represent - POSIX capabilities type - type: string - type: array - drop: - description: Removed capabilities - items: - description: Capability represent - POSIX capabilities type - type: string - type: array - type: object - privileged: - description: Run container in privileged - mode. Processes in privileged containers - are essentially equivalent to root on - the host. Defaults to false. - type: boolean - procMount: - description: procMount denotes the type - of proc mount to use for the containers. - The default is DefaultProcMount which - uses the container runtime defaults - for readonly paths and masked paths. - This requires the ProcMountType feature - flag to be enabled. - type: string - readOnlyRootFilesystem: - description: Whether this container has - a read-only root filesystem. Default - is false. - type: boolean - runAsGroup: - description: The GID to run the entrypoint - of the container process. Uses runtime - default if unset. May also be set in - PodSecurityContext. If set in both - SecurityContext and PodSecurityContext, - the value specified in SecurityContext - takes precedence. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container - must run as a non-root user. If true, - the Kubelet will validate the image - at runtime to ensure that it does not - run as UID 0 (root) and fail to start - the container if it does. If unset or - false, no such validation will be performed. - May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext - takes precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint - of the container process. Defaults to - user specified in image metadata if - unspecified. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext - takes precedence. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be - applied to the container. If unspecified, - the container runtime will allocate - a random SELinux context for each container. May - also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext - takes precedence. - properties: - level: - description: Level is SELinux level - label that applies to the container. - type: string - role: - description: Role is a SELinux role - label that applies to the container. - type: string - type: - description: Type is a SELinux type - label that applies to the container. - type: string - user: - description: User is a SELinux user - label that applies to the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use - by this container. If seccomp options - are provided at both the pod & container - level, the container options override - the pod options. - properties: - localhostProfile: - description: localhostProfile indicates - a profile defined in a file on the - node should be used. The profile - must be preconfigured on the node - to work. Must be a descending path, - relative to the kubelet's configured - seccomp profile location. Must only - be set if type is "Localhost". - type: string - type: - description: "type indicates which - kind of seccomp profile will be - applied. Valid options are: \n Localhost - - a profile defined in a file on - the node should be used. RuntimeDefault - - the container runtime default - profile should be used. Unconfined - - no profile should be applied." - type: string - required: - - type - type: object - windowsOptions: - description: The Windows specific settings - applied to all containers. If unspecified, - the options from the PodSecurityContext - will be used. If set in both SecurityContext - and PodSecurityContext, the value specified - in SecurityContext takes precedence. - properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is - where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA - credential spec named by the GMSACredentialSpecName - field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName - is the name of the GMSA credential - spec to use. - type: string - runAsUserName: - description: The UserName in Windows - to run the entrypoint of the container - process. Defaults to the user specified - in image metadata if unspecified. - May also be set in PodSecurityContext. - If set in both SecurityContext and - PodSecurityContext, the value specified - in SecurityContext takes precedence. - type: string - type: object - type: object - startupProbe: - description: 'StartupProbe indicates that - the Pod has successfully initialized. If - specified, no other probes are executed - until this completes successfully. If this - probe fails, the Pod will be restarted, - just as if the livenessProbe failed. This - can be used to provide different probe parameters - at the beginning of a Pod''s lifecycle, - when it might take a long time to load data - or warm a cache, than during steady-state - operation. This cannot be updated. More - info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - properties: - exec: - description: One and only one of the following - should be specified. Exec specifies - the action to take. - properties: - command: - description: Command is the command - line to execute inside the container, - the working directory for the command is - root ('/') in the container's filesystem. - The command is simply exec'd, it - is not run inside a shell, so traditional - shell instructions ('|', etc) won't - work. To use a shell, you need to - explicitly call out to that shell. - Exit status of 0 is treated as live/healthy - and non-zero is unhealthy. - items: - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures - for the probe to be considered failed - after having succeeded. Defaults to - 3. Minimum value is 1. - format: int32 - type: integer - httpGet: - description: HTTPGet specifies the http - request to perform. - properties: - host: - description: Host name to connect - to, defaults to the pod IP. You - probably want to set "Host" in httpHeaders - instead. - type: string - httpHeaders: - description: Custom headers to set - in the request. HTTP allows repeated - headers. - items: - description: HTTPHeader describes - a custom header to be used in - HTTP probes - properties: - name: - description: The header field - name - type: string - value: - description: The header field - value - type: string - required: - - name - - value - type: object - type: array - path: - description: Path to access on the - HTTP server. - type: string - port: - anyOf: - - type: integer - - type: string - description: Name or number of the - port to access on the container. - Number must be in the range 1 to - 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - scheme: - description: Scheme to use for connecting - to the host. Defaults to HTTP. - type: string - required: - - port - type: object - initialDelaySeconds: - description: 'Number of seconds after - the container has started before liveness - probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to - perform the probe. Default to 10 seconds. - Minimum value is 1. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes - for the probe to be considered successful - after having failed. Defaults to 1. - Must be 1 for liveness and startup. - Minimum value is 1. - format: int32 - type: integer - tcpSocket: - description: 'TCPSocket specifies an action - involving a TCP port. TCP hooks not - yet supported TODO: implement a realistic - TCP lifecycle hook' - properties: - host: - description: 'Optional: Host name - to connect to, defaults to the pod - IP.' - type: string - port: - anyOf: - - type: integer - - type: string - description: Number or name of the - port to access on the container. - Number must be in the range 1 to - 65535. Name must be an IANA_SVC_NAME. - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - description: 'Number of seconds after - which the probe times out. Defaults - to 1 second. Minimum value is 1. More - info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' - format: int32 - type: integer - type: object - stdin: - description: Whether this container should - allocate a buffer for stdin in the container - runtime. If this is not set, reads from - stdin in the container will always result - in EOF. Default is false. - type: boolean - stdinOnce: - description: Whether the container runtime - should close the stdin channel after it - has been opened by a single attach. When - stdin is true the stdin stream will remain - open across multiple attach sessions. If - stdinOnce is set to true, stdin is opened - on container start, is empty until the first - client attaches to stdin, and then remains - open and accepts data until the client disconnects, - at which time stdin is closed and remains - closed until the container is restarted. - If this flag is false, a container processes - that reads from stdin will never receive - an EOF. Default is false - type: boolean - terminationMessagePath: - description: 'Optional: Path at which the - file to which the container''s termination - message will be written is mounted into - the container''s filesystem. Message written - is intended to be brief final status, such - as an assertion failure message. Will be - truncated by the node if greater than 4096 - bytes. The total message length across all - containers will be limited to 12kb. Defaults - to /dev/termination-log. Cannot be updated.' - type: string - terminationMessagePolicy: - description: Indicate how the termination - message should be populated. File will use - the contents of terminationMessagePath to - populate the container status message on - both success and failure. FallbackToLogsOnError - will use the last chunk of container log - output if the termination message file is - empty and the container exited with an error. - The log output is limited to 2048 bytes - or 80 lines, whichever is smaller. Defaults - to File. Cannot be updated. - type: string - tty: - description: Whether this container should - allocate a TTY for itself, also requires - 'stdin' to be true. Default is false. - type: boolean - volumeDevices: - description: volumeDevices is the list of - block devices to be used by the container. - items: - description: volumeDevice describes a mapping - of a raw block device within a container. - properties: - devicePath: - description: devicePath is the path - inside of the container that the device - will be mapped to. - type: string - name: - description: name must match the name - of a persistentVolumeClaim in the - pod - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - description: Pod volumes to mount into the - container's filesystem. Cannot be updated. - items: - description: VolumeMount describes a mounting - of a Volume within a container. - properties: - mountPath: - description: Path within the container - at which the volume should be mounted. Must - not contain ':'. - type: string - mountPropagation: - description: mountPropagation determines - how mounts are propagated from the - host to container and the other way - around. When not set, MountPropagationNone - is used. This field is beta in 1.10. - type: string - name: - description: This must match the Name - of a Volume. - type: string - readOnly: - description: Mounted read-only if true, - read-write otherwise (false or unspecified). - Defaults to false. - type: boolean - subPath: - description: Path within the volume - from which the container's volume - should be mounted. Defaults to "" - (volume's root). - type: string - subPathExpr: - description: Expanded path within the - volume from which the container's - volume should be mounted. Behaves - similarly to SubPath but environment - variable references $(VAR_NAME) are - expanded using the container's environment. - Defaults to "" (volume's root). SubPathExpr - and SubPath are mutually exclusive. - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - description: Container's working directory. - If not specified, the container runtime's - default will be used, which might be configured - in the container image. Cannot be updated. - type: string - required: - - name - type: object - type: array - timeout: - description: Timeout defines the maximum amount - of time Velero should wait for the initContainers - to complete. - type: string - type: object - type: object - type: array - required: - - name - type: object - type: array - type: object - includeClusterResources: - description: IncludeClusterResources specifies whether cluster-scoped - resources should be included for consideration in the restore. If - null, defaults to true. - nullable: true - type: boolean - includedNamespaces: - description: IncludedNamespaces is a slice of namespace names to include - objects from. If empty, all namespaces are included. - items: - type: string - nullable: true - type: array - includedResources: - description: IncludedResources is a slice of resource names to include - in the restore. If empty, all resources in the backup are included. - items: - type: string - nullable: true - type: array - labelSelector: - description: LabelSelector is a metav1.LabelSelector to filter with - when restoring individual objects from the backup. If empty or nil, - all objects are included. Optional. - nullable: true - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains - values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: operator represents a key's relationship to a - set of values. Valid operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator - is In or NotIn, the values array must be non-empty. If the - operator is Exists or DoesNotExist, the values array must - be empty. This array is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator is - "In", and the values array contains only "value". The requirements - are ANDed. - type: object - type: object - namespaceMapping: - additionalProperties: - type: string - description: NamespaceMapping is a map of source namespace names to - target namespace names to restore into. Any source namespaces not - included in the map will be restored into namespaces of the same name. - type: object - preserveNodePorts: - description: PreserveNodePorts specifies whether to restore old nodePorts - from backup. - nullable: true - type: boolean - restorePVs: - description: RestorePVs specifies whether to restore all included PVs - from snapshot (via the cloudprovider). - nullable: true - type: boolean - scheduleName: - description: ScheduleName is the unique name of the Velero schedule - to restore from. If specified, and BackupName is empty, Velero will - restore from the most recent successful backup created from this schedule. - type: string - required: - - backupName - type: object - status: - description: RestoreStatus captures the current status of a Velero restore - properties: - completionTimestamp: - description: CompletionTimestamp records the time the restore operation - was completed. Completion time is recorded even on failed restore. - The server's time is used for StartTimestamps - format: date-time - nullable: true - type: string - errors: - description: Errors is a count of all error messages that were generated - during execution of the restore. The actual errors are stored in object - storage. - type: integer - failureReason: - description: FailureReason is an error that caused the entire restore - to fail. - type: string - phase: - description: Phase is the current state of the Restore - enum: - - New - - FailedValidation - - InProgress - - Completed - - PartiallyFailed - - Failed - type: string - progress: - description: Progress contains information about the restore's execution - progress. Note that this information is best-effort only -- if Velero - fails to update it during a restore for any reason, it may be inaccurate/stale. - nullable: true - properties: - itemsRestored: - description: ItemsRestored is the number of items that have actually - been restored so far - type: integer - totalItems: - description: TotalItems is the total number of items to be restored. - This number may change throughout the execution of the restore - due to plugins that return additional related items to restore - type: integer - type: object - startTimestamp: - description: StartTimestamp records the time the restore operation was - started. The server's time is used for StartTimestamps - format: date-time - nullable: true - type: string - validationErrors: - description: ValidationErrors is a slice of all validation errors (if - applicable) - items: - type: string - nullable: true - type: array - warnings: - description: Warnings is a count of all warning messages that were generated - during execution of the restore. The actual warnings are stored in - object storage. - type: integer - type: object - type: object - version: v1 - versions: - - name: v1 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/crd/v1beta1/bases/velero.io_schedules.yaml b/config/crd/v1beta1/bases/velero.io_schedules.yaml deleted file mode 100644 index ffa190915..000000000 --- a/config/crd/v1beta1/bases/velero.io_schedules.yaml +++ /dev/null @@ -1,401 +0,0 @@ - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.3.0 - creationTimestamp: null - name: schedules.velero.io -spec: - group: velero.io - names: - kind: Schedule - listKind: ScheduleList - plural: schedules - singular: schedule - preserveUnknownFields: false - scope: Namespaced - validation: - openAPIV3Schema: - description: Schedule is a Velero resource that represents a pre-scheduled or - periodic Backup that should be run. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: ScheduleSpec defines the specification for a Velero schedule - properties: - schedule: - description: Schedule is a Cron expression defining when to run the - Backup. - type: string - template: - description: Template is the definition of the Backup to be run on the - provided schedule - properties: - defaultVolumesToRestic: - description: DefaultVolumesToRestic specifies whether restic should - be used to take a backup of all pod volumes by default. - type: boolean - excludedNamespaces: - description: ExcludedNamespaces contains a list of namespaces that - are not included in the backup. - items: - type: string - nullable: true - type: array - excludedResources: - description: ExcludedResources is a slice of resource names that - are not included in the backup. - items: - type: string - nullable: true - type: array - hooks: - description: Hooks represent custom behaviors that should be executed - at different phases of the backup. - properties: - resources: - description: Resources are hooks that should be executed when - backing up individual instances of a resource. - items: - description: BackupResourceHookSpec defines one or more BackupResourceHooks - that should be executed based on the rules defined for namespaces, - resources, and label selector. - properties: - excludedNamespaces: - description: ExcludedNamespaces specifies the namespaces - to which this hook spec does not apply. - items: - type: string - nullable: true - type: array - excludedResources: - description: ExcludedResources specifies the resources - to which this hook spec does not apply. - items: - type: string - nullable: true - type: array - includedNamespaces: - description: IncludedNamespaces specifies the namespaces - to which this hook spec applies. If empty, it applies - to all namespaces. - items: - type: string - nullable: true - type: array - includedResources: - description: IncludedResources specifies the resources - to which this hook spec applies. If empty, it applies - to all resources. - items: - type: string - nullable: true - type: array - labelSelector: - description: LabelSelector, if specified, filters the - resources to which this hook spec applies. - nullable: true - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector - that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: operator represents a key's relationship - to a set of values. Valid operators are In, - NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. - If the operator is In or NotIn, the values - array must be non-empty. If the operator is - Exists or DoesNotExist, the values array must - be empty. This array is replaced during a - strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. - A single {key,value} in the matchLabels map is equivalent - to an element of matchExpressions, whose key field - is "key", the operator is "In", and the values array - contains only "value". The requirements are ANDed. - type: object - type: object - name: - description: Name is the name of this hook. - type: string - post: - description: PostHooks is a list of BackupResourceHooks - to execute after storing the item in the backup. These - are executed after all "additional items" from item - actions are processed. - items: - description: BackupResourceHook defines a hook for a - resource. - properties: - exec: - description: Exec defines an exec hook. - properties: - command: - description: Command is the command and arguments - to execute. - items: - type: string - minItems: 1 - type: array - container: - description: Container is the container in the - pod where the command should be executed. - If not specified, the pod's first container - is used. - type: string - onError: - description: OnError specifies how Velero should - behave if it encounters an error executing - this hook. - enum: - - Continue - - Fail - type: string - timeout: - description: Timeout defines the maximum amount - of time Velero should wait for the hook to - complete before considering the execution - a failure. - type: string - required: - - command - type: object - required: - - exec - type: object - type: array - pre: - description: PreHooks is a list of BackupResourceHooks - to execute prior to storing the item in the backup. - These are executed before any "additional items" from - item actions are processed. - items: - description: BackupResourceHook defines a hook for a - resource. - properties: - exec: - description: Exec defines an exec hook. - properties: - command: - description: Command is the command and arguments - to execute. - items: - type: string - minItems: 1 - type: array - container: - description: Container is the container in the - pod where the command should be executed. - If not specified, the pod's first container - is used. - type: string - onError: - description: OnError specifies how Velero should - behave if it encounters an error executing - this hook. - enum: - - Continue - - Fail - type: string - timeout: - description: Timeout defines the maximum amount - of time Velero should wait for the hook to - complete before considering the execution - a failure. - type: string - required: - - command - type: object - required: - - exec - type: object - type: array - required: - - name - type: object - nullable: true - type: array - type: object - includeClusterResources: - description: IncludeClusterResources specifies whether cluster-scoped - resources should be included for consideration in the backup. - nullable: true - type: boolean - includedNamespaces: - description: IncludedNamespaces is a slice of namespace names to - include objects from. If empty, all namespaces are included. - items: - type: string - nullable: true - type: array - includedResources: - description: IncludedResources is a slice of resource names to include - in the backup. If empty, all resources are included. - items: - type: string - nullable: true - type: array - labelSelector: - description: LabelSelector is a metav1.LabelSelector to filter with - when adding individual objects to the backup. If empty or nil, - all objects are included. Optional. - nullable: true - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the - key and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: operator represents a key's relationship - to a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a - strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. - type: object - type: object - metadata: - properties: - labels: - additionalProperties: - type: string - type: object - type: object - orderedResources: - additionalProperties: - type: string - description: OrderedResources specifies the backup order of resources - of specific Kind. The map key is the Kind name and value is a - list of resource names separated by commas. Each resource name - has format "namespace/resourcename". For cluster resources, simply - use "resourcename". - nullable: true - type: object - snapshotVolumes: - description: SnapshotVolumes specifies whether to take cloud snapshots - of any PV's referenced in the set of objects included in the Backup. - nullable: true - type: boolean - storageLocation: - description: StorageLocation is a string containing the name of - a BackupStorageLocation where the backup should be stored. - type: string - ttl: - description: TTL is a time.Duration-parseable string describing - how long the Backup should be retained for. - type: string - volumeSnapshotLocations: - description: VolumeSnapshotLocations is a list containing names - of VolumeSnapshotLocations associated with this backup. - items: - type: string - type: array - type: object - useOwnerReferencesInBackup: - description: UseOwnerReferencesBackup specifies whether to use OwnerReferences - on backups created by this Schedule. - nullable: true - type: boolean - required: - - schedule - - template - type: object - status: - description: ScheduleStatus captures the current state of a Velero schedule - properties: - lastBackup: - description: LastBackup is the last time a Backup was run for this Schedule - schedule - format: date-time - nullable: true - type: string - phase: - description: Phase is the current phase of the Schedule - enum: - - New - - Enabled - - FailedValidation - type: string - validationErrors: - description: ValidationErrors is a slice of all validation errors (if - applicable) - items: - type: string - type: array - type: object - type: object - version: v1 - versions: - - name: v1 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/crd/v1beta1/bases/velero.io_serverstatusrequests.yaml b/config/crd/v1beta1/bases/velero.io_serverstatusrequests.yaml deleted file mode 100644 index 379cc09f9..000000000 --- a/config/crd/v1beta1/bases/velero.io_serverstatusrequests.yaml +++ /dev/null @@ -1,89 +0,0 @@ - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.3.0 - creationTimestamp: null - name: serverstatusrequests.velero.io -spec: - group: velero.io - names: - kind: ServerStatusRequest - listKind: ServerStatusRequestList - plural: serverstatusrequests - shortNames: - - ssr - singular: serverstatusrequest - preserveUnknownFields: false - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - description: ServerStatusRequest is a request to access current status information - about the Velero server. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: ServerStatusRequestSpec is the specification for a ServerStatusRequest. - type: object - status: - description: ServerStatusRequestStatus is the current status of a ServerStatusRequest. - properties: - phase: - description: Phase is the current lifecycle phase of the ServerStatusRequest. - enum: - - New - - Processed - type: string - plugins: - description: Plugins list information about the plugins running on the - Velero server - items: - description: PluginInfo contains attributes of a Velero plugin - properties: - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - nullable: true - type: array - processedTimestamp: - description: ProcessedTimestamp is when the ServerStatusRequest was - processed by the ServerStatusRequestController. - format: date-time - nullable: true - type: string - serverVersion: - description: ServerVersion is the Velero server version. - type: string - type: object - type: object - version: v1 - versions: - - name: v1 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/crd/v1beta1/bases/velero.io_volumesnapshotlocations.yaml b/config/crd/v1beta1/bases/velero.io_volumesnapshotlocations.yaml deleted file mode 100644 index 1f281fa61..000000000 --- a/config/crd/v1beta1/bases/velero.io_volumesnapshotlocations.yaml +++ /dev/null @@ -1,74 +0,0 @@ - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.3.0 - creationTimestamp: null - name: volumesnapshotlocations.velero.io -spec: - group: velero.io - names: - kind: VolumeSnapshotLocation - listKind: VolumeSnapshotLocationList - plural: volumesnapshotlocations - singular: volumesnapshotlocation - preserveUnknownFields: false - scope: Namespaced - validation: - openAPIV3Schema: - description: VolumeSnapshotLocation is a location where Velero stores volume - snapshots. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: VolumeSnapshotLocationSpec defines the specification for a - Velero VolumeSnapshotLocation. - properties: - config: - additionalProperties: - type: string - description: Config is for provider-specific configuration fields. - type: object - provider: - description: Provider is the provider of the volume storage. - type: string - required: - - provider - type: object - status: - description: VolumeSnapshotLocationStatus describes the current status of - a Velero VolumeSnapshotLocation. - properties: - phase: - description: VolumeSnapshotLocationPhase is the lifecycle phase of a - Velero VolumeSnapshotLocation. - enum: - - Available - - Unavailable - type: string - type: object - type: object - version: v1 - versions: - - name: v1 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/crd/v1beta1/crds/crds.go b/config/crd/v1beta1/crds/crds.go deleted file mode 100644 index d16049295..000000000 --- a/config/crd/v1beta1/crds/crds.go +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by crds_generate.go; DO NOT EDIT. - -package crds - -import ( - "bytes" - "compress/gzip" - "io/ioutil" - - apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" - apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - "k8s.io/client-go/kubernetes/scheme" -) - -var rawCRDs = [][]byte{ - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec~\xe2\xc6\xd2\x0f\x85(5\x13\xd5L\xf4\xcdp\xb9+\x05\xd3\xf1\xeb\x19@\xa1Ѡ>\xe0/\xf2I\xaag\xf9\x91\xa3H\xcd5dL\x18<\x030\x89*\xf0\x1a>;\f\n\x96`z\x06p`\x82\xa7\xb4:\x8f\x93*P\xbe\xbf\xdb<~\u007f\x9f\xec1g\xfe#@\x8a&Ѽ\xa0~\x019\xe0\x06\x18<\xd2\xd2@\a\x16\x80\xdd3\v\x1a\t\x13i\r\xd8=B\xc2\n[j\x04\x95\xc1\x8f\xe5\x16\xb5D\x8b&\x00\x06HDi,j0\x96Y\x04f\x81A\xa1\xb8\xb4\xc0%X\x9e#|\xf3\xfen\x03j\xfb+&\xd6\x00\x93)0cT\u0099\xc5\x14\x0eJ\x949\xfa\xb1߮\x03\xccB\xab\x02\xb5\xe5\x91ή5\x04\xab\xfa\xd6Yօ[\xb7\xef\x03\xa9\x13%\xf4\xe8\a\x81\xc0\x14\f\xd1ĭ\xc3\uee69\x97I\xf4k\x80\x05ׅɀ\xf4\x1a\xee\x1dS\xb4\x01\xb3W\xa5H\x9d\xfc\x1dP;2%j'\xf9\xbf*\xc8\x06\xac\xa2)\x05\xb3\x18X\x1f\x1b\x97\x16\xb5d\xc2q\xac\xc4K\"DΎ\xa0\xd1\xcd\x01\xa5l@\xa3.f\r\xff\xaf4\x02\x97\x99\xba\x86\xbd\xb5\x85\xb9\xbe\xba\xdaq\x1b\xb7R\xa2\xf2\xbc\x94\xdc\x1e\xafhC\xf0mi\x956W)\x1eP\\\x19\xbe[1\x9d\xec\xb9\xc5\xc41\xef\x8a\x15|E\x88K\xdaI\xeb<\xfd\x9f\xc8ts\xd1\xc0\xd4\x1e\x9d\x8c\x19\xab\xb9\xdcU\x9fI\xd2G\xe9\xeeD\xdeK\x93\x1f\xe6\xf1\xaf\xc9\xeb>9\xaa\xfc|{\xffД4n\xda4'j7\x84\xaf&\xbc#\x14\x97\x19jϸL\xab\x9c \xa2L\xbd\xac\x91\x98\n\x8e\xb2MtSnsn\x1d\xa7\xffY\xa2q\xe2\xac\xd6pC\n\x05\xb6\be\x91:)\\\xc3F\xc2\r\xcbQ\xdc0\x83\xbf;\xd9\x1d\x85\xcdʑt\x9e\xf0M=\xd8\xee\xe8\xa9U}\x8e\x1ak\x90C~\xc3\xdf\x17\x98\xb46\x86\x1b\xc33\x9e\x90\xf8C\xa6t\xad\x0f\xbcJZ7\x00\x0emJ?Q\xc6Ja\x1fi#\x9b\a\xf53\x1a˓v\x9f\x0e:\x1f\x06\x87Dt\xd0\xc0\xf3\x1e\xed\x1e\xb5\x93\x15\xfa\x81\xb6]\a\"\x10\x03\r\xa6\xb4\xe7\xd8\x13\x02\vX\xd3\xe6\x15\x02\n\x15\xf5\x8b\x81\xed1\"\xba\xee\xc0\xf1\xd4\xdc*%\x90\xb5u\x00\xbe$\xa2L1\xad\xf4\xad\x99\\\xd5m\xaf;\x19*ƥ\xdb\x19\xce48\xc4d\xfd+\xa9Z\xa6\xb1\xb70'\x9d\\zh\xa4E\xf78\xc0\x10\u05f8ż\x87Ո(\x05إ\x10l+\xf0\x1a\xac.\xbbS\xfbqLkv\x1c\xa4D4\xd4\xcb\bQ\xf5\x0e\xbaA\xf0\x84lH\xa5\x01\x88\x16\u007f\"2\xec\x95z\x9a^\xfa\xff\xb9\x1e\xb5\x06\x83\x84\xfc\x1b\xd8\xe2\x9e\x1d\xb8\xd2a\xb1\xc1\x8cl\x11\xf0\x05\x93\xd2b_\xb6\x99\x85\x94g\x19j\a\xa5\xd83\x83ƛ\xad1\x12\x8cmO\xd7\xf4\x18\xdbz\xf8\xd7,c\x1a\xfdz\xc7Pv\x9bT\x122}\xea\xfa\xe6\\\f\x99\xf2\x03OK&\x80Kc\x99L\xfc:X\x85Sw\x1d0\xce\xce\x1e\xb6^\xadE\x9c\x1d\xed[*NI\x04\xa5!wڼ\xdf\xd5\f\u0087\xd1\xe5n\x99\xd35ʋ\xa1.\x05\x9a0QJ\x9a\xb3\xdeח#\x80+.x\xdb/\xd8\x16\x05\x18\x14\x98X\xa5\x87\xc80\xcdT\xdf\xe6u\xd4\b\xed\x06\xb4U\xad\u007f\xdd\x12\x9b\x8aJ\x8d\xc2\x04x\xde\xf3d\xefͲ\x93\x17\x82\x02\xa9BC\xfb\x97\x15\x858\x0e/\x0e\xa69\xed\xdb\xc4\x16\xae\xdb\xe4f\xee\xc2\xeao\xeb\xba\xcd깺\xcdh\xbc6-+\xd6\xffuH\x19\x15\xf7ɂ\xb9\xe9\r|K\xc1tD\xe4ε\xded\x80ya\x8f\x97\xc0m\xfc\xea<\tFq\xe1(y\xaa\xb9\xfft\x8c8U\xa67\xddqo(ӿ\x91\v\xd5\xd4\u007f\x1a&\x90\xb2\xbf\x0f\xba~!\x03>5\xc7\\\x02\xcf*\x06\xa4\x97\x90qaQw81\xb5\\5͉\xdfJ\x82yK\xe5Z\xcel\xb2\xbf}qޑ\xa9\xd39\x8b\xa8\xd1\x1d\xea}\xca\xe8U\xb7\x8d\xe9$T\xa0`\x90k\xcc}\x88\xf9@\x14\xac\xbf\x90\xe7\xf3\xfe\xf3\aLlj\x02K$\xac\xb7\x84\xf7\x1d4\x9b\xd3\x06\x17y\xd9\x02\x82\x93RE\x17>]p\t\f\x9e\xf0\xe8\xbd\v&\xc11\x84\xb9i\\\xe7Y\x88\x1a)gA\x02\xf5\x84G\x02\x12\xd2\x103c\x97\xb1\u07b7'<\xcew\xea\x90\xcda\xc3MH\xab8\xfa\xb9\x0fD\x00\x8aa\x97\x92\f(\x89\x145\xccܢ`\xa9\x8a\x88-R\xfb\xe4\xe5Ulj$݈\x91\x17\xc63\xc5I\xfb\x9e\x17\x8b\x16\xe8T'\x18\xa4=\x11\x93H\x8fL\xf0\xb4\x9a\xc6\xcb\xf7F^\xc2ge7r\xccYm\xb7\xdb\x17nB\xee\xee\x83B\xf3YY\xfa\xf2\xe6D\xf4(\x9fLB?\x8c\xb6\x90\xf4jح\xbf\x99\x8b\x9a\x15b\xdf6>ªX\xc2\rl\xa4\x8b!<\xad|6\xd1O6\xa5\xed\xdb-/\r%\x9b\xa4\x92+2v\xeb\xa1y\x02\x89\x17\nr\x93\v}\xb4\xaa)\xfdt\x8b >8\xbb\xe0G\xfb̨`\t\xa6\x90\x96DD\xca\xec1\x8b;\x9e@\x8ez7n\b\x9a\xadp:{\xc9\xf4\x8bt\xa9o'\xc9\xd3\x12\xd3\x1c[P\xc6\xe9\x1c\x1a+\xb77g\xfbD\xd6\xcet\x1cL\xe5\x8dw\x9c[\a\x19I\xf2\x1bf\xa8\xc9Ҕ\xceY\x98\xb8[\xac\xbd\x17S\xbeo\xb7=J\xde\xc6\xe5\x8c\x12t\xffv\xa6\x8a\x84\xf6?P0\xaegw\xe8{:0\x11\xd8\x1a\x19\xb2B\xcdI\x1c|n\xc0q\xf3\xc0D7!<\xb0,\xe5\xb4\x06\no\x86U\xd6\xf34.\xe1y\xaf\x8c\xb7\x8a\x19G\x91\x02\x9f\xf2\xb4\\;\u007f\xc2\xe3\xf9eo\x8f\x9fo\xe4\xb97Ͻ\x1d\x1bm\xf9\f`%\xc5\x11\xcei\xe4\xf9\xeb]\x97ER\xb7\xa0\x13\x9d\x9e-sf]4\x17\xad\xb8\x1bV\x9d\xc18Wt\x1c\xdb\x052W(c\x17\"q\xa7\x8c\xf5\x19\xba\x96\xf38\x90\x1b\x9a\x8eiBN\bX\xe6Ͻ\x94\x8e'\x1cN\x91uR\x95\x8eK\x06\a\x13\x9c=\x88i\x00Ʉ\x80\xf3z\x8fz\xfdx\xee\x8f=h\n\x96\x90[0\x01щB\xa1U\x82\xc6L\x89ì\xe6\x9dI\xb8U\xc96\xe6\x83\n\u007f\x880\x95܋m\xa9\xdb\xe8Hs\x92\x9b}\xfb\xd2\xc8\x01\xba\xad\xed\xfe\x9f\x16\xb3\xd30\x02:\x82\xces&g\x8dE\x0f\xb9\x1b?.n\x85\x00ƻ\xeczW\xd26^\xea\xe9\x05\xa1\xf9\xb2\x066\xe7rC\xc0\xe1ݛ\x9ac\x88*\x11Ow\xa9o\xe2Ț\xcc\xd5\a\xbf7\v\xd5O\xb9\x0f\xb5\xe7=jlq\xaa\x9f\x19&wN*\xdb\bϗ\x11\xda\xe3qa \xe3\xda\xd8&\x92\x86\x0e\xb6\xde>F\x91\xb7Z\xbf\"D\xf9ɏk$\x80\xf6\xea9\x9e\x14\x8e\x1c\xce\r5:\x06A\xe0\x19p\v(\x13UJJb\xb8MJ\x13x\x92ze:kd}[\xb2\xb1]CY\xe6K\x16\xbe\"\xe9\xe1r\"\xd7\xd1\xec\xfc\x91\xf1\xa9LUl'\xb1\xc9\xf2\x1cU9a\xd4\xea\xd6bӃ\x1f\xd7:\xe2\xcd\xd9\v\xcf\xcb\x1cX\ue23d\x88\xa2\xce2\xf3\x1c\xdb\xfc\x85g\xc6-iw\a\x95T\xbdUnS\x14\x02\xed\xb2h`\x8b\x99Ҵ\x17\rO\xb12\x99\x81\xe7J\x02\x83\x8cqQ\xeaE\x1a\xed\x04\x8a.\xf7\xec\xc3&\u007f\x1b\xa7}ɴ+Z\xfel\x9ar\x91\xab6\xa5U\v\xbd\xd4Q\xbb\xd3\xf8\x96.R\xa1\xb9\x93\x19\xf5\xb6^R\x10%&\x8f_ݤ\x06m\xbe\xbaI\xbd\xf6\xd5Mj\xb5\xafn\xd2W7i\xb2}u\x93\xbe\xbaI\u007fU7i\x1a\x93\x15\xe5\xad\x06\u007f\x9a\x99}\xf6\bu\x1c\xb1Q\xc8\xe1T\xff\xc6\xd7^/\xab\xcb\xdb\f\x8f\x19\xa8\xbb\f%\xdd+*8\xef\xf3\xb9>\xfa\xaf\xd5|U\xa8\xe7\x84?\n\xaf/,\x9d,\xdd[P\x887T\x9b9_^2WTҮI\xac\n;bQ\xa2\x8aS\xf4V\x1f+ٝ\x9b٬``B4kS\x98\xae\x89\xf2\x85\xea\x15gK?f\n>\xa6\xcb6\xc7)\xd4q\xed\xdb$ҭ\x12\xc3/L\xa1ɺ\x8c\xf1j\x8cp\x92\x81\x96\x1dޭۿX\x15j3\xe0\x99\xdb}o\x01T4\xe9B\x16\xb9k\x16GF\x99\n\xd7\a\xba\x94\x03\xa5Arq9X\x17Sݬh\x92\x13~*|Pt\xd2~\x9br\xed\x97\xd4n\xbc\xbab\xa3]\x931\xa8dO;\xecXZB\xba\xbc&\xa3]s1bd\x16Tb\x9c\\i1\x1foMVU\xbc\xa2\x96\"\xd6IL\x19܉\n\x8a\x05>\xc7|\xb5īj$\xe80o\x02\xeb\x93*#\x1aU\x0f\x13 \x97\xd5C, \xc9\\\xed\xc3\xc9\x15\x0f\xdd*\x83\x89E\xcc\xd59\x8c\xd70L\x00\x1d\xacnXR\xb90\x01\xb3\xaaix\xc3z\x85\x99*\x85\xb7\xa9$\xfc\xad\xbe\xe7X\xcd\xc1L\xa5\xc1\x8cg:\x85\xd5L-\xc1\xf2\n\x82\x19\xfa\xbc\xb2Z\xa0\xaa\a\x18\x9c\xf3\xd4\x1a\x81v\x15\xc0 ȅ\x95\x01#g\xff\x83 \x17\xd4\x03̜\xf8\x0f\x82\x9d4\x8c\x13\x121\xfa\xd3\xd0%4\x98\xb1L\xe2\xf7\x97\x9c\xd7,E\xe9\x14\xf5\x84G\xbc\f\xb9\t\xc4\xda\x19\xa1\xcel\x9d\x12\xeax]\xcd\xf5jz\xd8}\xb6\xaa\xaa\xf87\x81\x1f\xb9L\xbd$81nXt\xba\x86I\xc5\x15\x95KQ\xfb\\C ;\x1e\xbd\xc1\x82i\xba\x95\xbb=\xfa\x18߬\xe1\x96%\xfbvG\xd83㢼|\xa0\xaa\xf4\xbc\n\x80\xae\xe2\x18\xf7\xe5|\r\xf0QUqe\xf3*\x8c\xe1y!\x8eP\x1a\x84\xf3\xf6\x90\xd3\xfd\xfb\x01~\x1b\xc9\n\xb3W\xf1\xaeᤋ\u007f\xdf\xee;\x10\x17Ǜ\x86\x89PeZ\xc1\x1ed\x17\x93G\xb8{$\xff\x84nq%\xf5\x1d\xb6\xe0\x85D\xbf\xbd{\xc5퇷\x8c\x93\x8dU\x9a\xed\xf0\x93J\x1aw\xc4\xc7\xd6\xdf\xeeۺ\xd1\x1b\xf4Q\xccF\xc5\x12\x1e\x16\xaf\x96\xb6\x87\x0e\x85=!A\x1cd\xbeN\x1c8\f\xfb\xaajt\x87Y+&\x17\xf1\xf0\xf0\xc9#ny\x8e\xeb\x0f\xa5\xcfA\xac\n\xa6\r:\xfa\xc5\x05\xf9A[\xf7\xe7^=\xf7\u0557\n+\xfd\xa1\x8b\xafFJ?S\xa2c1\xd6\xfe\x1aj\x14\xb0H\xa6iq|\x1c\x1e\xd3\b\xa3\x1aL\xf1;Xec\xa3z\vl\xdc\xc1w\x81\xaa/\xc6z\xabە\xc3~\xc5\xf0\xbde\xcbli\xe6n.S\xa7\xf8\x0eA8\xac(5]\x8e\xf4\x00\xbc0\x9e|y9df[oCL\xf1\xe4\xa6ߟ^\x01ЩG\x8a2\xc2\xd5=\xe4gf\xaa\xdc\xef\x801\xae\x81\xf9q\xe4\xc7:X\x98\x02\x1eP\x82\x92\x94\xea\xa5ˇ\xfe\x85\x8a\xee\x98~\xea\xa5\x01#d\x92\xcbB(\x96Ɲ\x1bmNx\xd9\xe0\x81\xf4\x91>\xa0\xbe0\xa3\x10\xe9\x96u\xa6\xf4\xd0\xf2\xbb\x92\xe5\r\xc35\xa4\xcc\xe2j\x00\xe0\x02=6 Rt\xee1s똺\xf8\xddAG&\xf1\x1a\xb8?3\xc9\xd1\x18\xb6\x8b\u05cd\x9f\x9d:ڡD\xb2w\xbd5\x87(\xa2ι\xb7\xaf\xde\xfad\x04Kl\xc9\x02\xf8\x98}i\xf4\xba\xe8\xef9\xa1v\x90qA\x1d\xc3c\aA?\x0f+\x12.-\xee\xb0\xed\xd9\xe3K\xc1\xf5\xbc.\xbf\xad\xba9\x8aP։vx\xfd\xf4\a\n\xbe\xe3N!:\xc6\xee\x98\u07b2\x1d\xae\x12%\x04ҡy\x17\xa3߇\xaf\x1e\xea\xc0\xc3\x1e\xbd\x05}l\xf6\x8c\x1eO\x10f\x0f%\xbe\xf3q\x19,\xaa\xe3`\xce~U\xba\u007fΘs\xa9\xb4\xf7\xbc)\xfa\x8bC\x17\xebs\xba\x91=\x89\xef\x9d\xebQ\x1d\xac6t\x15Fa\x1a\xb6\xf3C\ap+\xf8\x8c]\x13\xe5\xcf\xd40}\xac\xde\u007f\xe9u\xd8\xc8;\xadv.X\xe9\xfdt\x13\xb5R\xef\x97;\xa6-gB\x1c=\xf8\x91Y{\x9f?\xa0\xd3\v#\x86`\x88\x80\x01\xb3i\x1a\x86Nu4ĥ\xe75\x9d\x83mUi[\x1b\xaeް=\x8e\xc7\xf9\xd6\xf0YY\x8c)/ކ\xe8, \x1a\xbb\xc2,S\xda\xfa\xd0k\xb5\x02\x9e\x05\xc3҃\xea\xb43%m\xfdC\"\xc0m\x9d\x80\xa8e\x93|A\x8d̐lZz̄NLX\x928\xff\x04\xaf\x8ce\xa2\xa7\x03^\x9d\xa9%{\xed\xa4\v\xd3_z\xe6\xacG\xe4M\xb3wU\xa2]\xe6[\x1f\x93\x100O/:\x95\xf6ZO\fg#\xb6\x88\x12\x9e5\xb7\xd6\xe9\x9bf.\x1b\xac\xd30B\x80Q\x90\xb1\xc1\xeb\xee\xe3:\x8f~U\x96\x89\xcdX.\xa6\xed\x02V]\xe3rhp\u007fQʱaKK\x1f\\\x8e\xafK\xe2&\x8et\x8cK\xf6L\xee\x9c\x00iU\xee\xf6Q\x02G,\xc5p\xfe\xb9t\bA!ʝ\x13\xe9\x90\x13\xb6\xa5\x96\x8d\xe83d\x89\xd3\x06\xaa,y\x82\xb2\x18.\x9a\xf0\xcf\x1c\x85G\xaa\xae\xc25\xf6U\xa6U\xbe\n\xf4\xa7x\xfc2D\x86\x9a+\xe72QL\x13n\x92\x8e\x80%\xb6\x17\x05J`&\xe02[25\xc5\xc8\xf1@\xcd2m\x979a\xf7\xad\xae3\xfe\x17\xc1\xc5t\r\xf7!\xba\xedob\xadr\xb8\xe9>\x11\xe6\"S\x19\xdf\xc3\xf2\xb1\xb4g\xbdqn\x99F\n[\xfc\xf5\xd1\x1eĖC\xd5r\xa0ڨ\xff1\xbeS\xfdB\xd8\xed\xbc\x17\xf5\xd8\xe9\xdc9\x03t;\xb8\x86\x17}\x9fox֏/\x8aB\xf0\xc4a\xfb\xed\x17:\xdb;,\xf0*.&\x1d\n\xf2\x1e*\xdf\x00>`\xa11q\xbb\xb2\x8f\xfc\x9d@g\xef\rb\xdbS\xb9X\xecصCD\xf3\xdeZ̋\x81\xb9&b\xc4zИ\xe2c\xb1Co\x01\xf1!\xb6\b*\x14\xb1\x8c\x06\x85\x8b\x17R\xb9\x1a\xa7,\xa4\x1a4\xb6\x10S&N\x01e\xe5\x90)\xaab\xae7\\\xd53\xd3.О\xde=\u007f\v\x9d\x06\xa2\x900\xfem\xe3\x90F\x18\x12\xf1\xfb\x83\x02\x91\x01=\xde\xf9T?\xc3\xf8\xae\xfe\x8fȷ\n\xcf.\x1e|)$i˴\xb1\xb5\x03*\xe1K\x9d `I\x82Nv?w_`\x9a\xb2R\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcYKs\xe3\xb8\x11\xbe\xebWt\xcd\x1e|\x19Q3\xd9KJ\x97\x94F\xceVM\xe2\x19\xbbF^琤j!\xa2)!\x02\x01\x06\x0fi\x95T\xfe{\xaa\xf1\x10)\x92\x92\xec\xad\xdd\xe5\xc5&\xd9\x04\xbe\xfe\xfa\rM\xa6\xd3\xe9\x845\xe2\x05\x8d\x15Ź5\x02\u007fv\xa8\xe8\xce\x16\xbb?\xdaB\xe8\xd9\xfe\xe3\x1a\x1d\xfb8\xd9\t\xc5\xe7\xb0\xf4\xd6\xe9\xfa\x1bZ\xedM\x89\xf7X\t%\x9c\xd0jR\xa3c\x9c96\x9f\x000\xa5\xb4c\xf4\xd8\xd2-@\xa9\x953ZJ4\xd3\r\xaab\xe7\u05f8\xf6Br4a\x87\xbc\xff\xfeC\xf1}\xf1a\x02P\x1a\f\x9f?\x8b\x1a\xadcu3\a奜\x00(V\xe3\x1c֬\xdc\xf9\xc6:m\xd8\x06\xa5.\xe3^\xc5\x1e%\x1a]\b=\xb1\r\x96\x01\t\xe7\x01\x1e\x93OF(\x87f\xa9\xa5\xaf#\xac)\xfce\xf5\xf8\xf5\x89\xb9\xed\x1c\n\xeb\x98\xf3\xb6h\xb6\xccb\x80\xccіF4.\x00\xfb\x14\xf6\x83U\xdc\x10\x1eҎ\x10\xbf\x02\xeb\xcb-0\v\x8b=\x13\x92\xad%\xce~T,\xff\x1fV\x8b\xb0\x9fN\xab\xbbc\x83s\xb0\xce\b\xb5\xb9\x00E2\xeb^\x98\x14\xfc\xc4\xc4\x10\xd7\xc3@\x06\x84\x05\xb7E\xa0\xaf\xc1\xd1\x03\xba\x8b|\x01\x11\x86\x90\xf9\x82\x03\xb3aI\x80}\\\x03y\a,\xad\r/g/\"j\xba\xefc\xce\xd6/\x06\x96문\xd8\xe0\x8de\xc8l\x05NJy\xe9\x86\xda\xde\xc7\x17]m\xc8\x1aY\x9f\xceN\xf7\x9d%\xe2nk\xad%2\x92\xd9\x18\xed\x9b9\xb4\xbe\x12?J\x9e\x1a\xbd<\xda;\x99\xfb\xa1\xbb\xbe\x14\xd6\xfd\xf5\xb2̃\xb0q\xd7Fz\xc3\xe4%O\r\"v\xab\x8d\xfb\xdan=\x85\xb5\x95\xf1\x8dP\x1b/\x99\xb9\xf0\xf9\x04\xa01h\xd1\xec\xf1G\xb5S\xfa\xa0~\x10(\xb9\x9dC\xc5dp0[jR:,ް2\x98\xcf\xfa\xb5Ia\x9b6\x8c\x8e6\x87\xff\xfeorr\x01\":\xbc\xd4\r\xaa\xc5\xd3\xe7\x97\xefW\xe5\x16k6O\x9e2\x12\x16=\n\xc8\x03Y\xc7ɶh\x10^\x02\xdb\xd1\x01m\xd2*\xad\b\xa0\xd7\xff\xc2\xd2e_l\x8cn\xd08\x91Q\xd2\xd5IR\xa7g=,w\x046\xca\x00\xa7\xb4\x841\x10RrA\x0e6(\x02\xba\x02\xb7\x15\x16\f\x06\x12\x95k\x8d{\x02T\x01S\tV\x01+\"\xdaX\xb2\x97\x97\x9cr\xd9\x1e\x8d\x03\x83\xa5\xde(\xf1\x9f\xd3\xca\x16\x9cN\xb1\xe70\xb9A\xbeB\xeeQL\x12\xcd\x1e\xdf\x03S\x1cjv\x04\x83\xb4\ax\xd5Y-\x88\xd8\x02\xbeP\xb0\nU\xe99l\x9dk\xec|6\xdb\b\x97\xd3r\xa9\xeb\xda+Ꮃ\x90\\\xc5\xda;m\xec\x8c\xe3\x1e\xe5͔̊\x99r+\x1c\x96\xce\x1b\x9c\xb1FL\x03p\x153eͿ;9\xc3]\ai//\xc5+\xc4\xc4E\xde)\x1a\xa2\xcd\xe3g\x11\u007fK/=\"V\xbe\xfdy\xf5\fy\xd3`\x82s\xce\x03\xdb\xedg\xb6%\x9e\x88\x12\xaaB\x13\rW\x19]\x87\x15Q\xf1F\v\xe5\xc2M)\x05\xaasҭ_\xd7\u0091\xa5\xff\xed\xd1:\xb2O\x01\xcbP\x9c`\x8d\xe0\x9b\x90\xd9\n\xf8\xac`\xc9j\x94Kf\xf17\xa7\x9d\x18\xb6S\xa2\xf46\xf1ݚz.\x18\xd9:=\xce\xe5n\xd4B\xa3Q\xbaj\xb0<\x8b\x13\x8eV\x18\xf2e\xc7\x1c\x86\bHA{F\xe9\xe5\xc4x9xC\x00\x97%Z\xfbEs<\u007fރ\xba8\x89\x9dak\xd0\xd4\u0086\xae\x04*m\xfa%\x8d\xa5\xbaҽr\xfe)zoP\xf9\xba\x0fa\nߐ\xf1G%\x8f\xa3/\xfef\x84\xebo0j.\xba\"\xac\xd5Q\x95Oh\x84\xe6W\xd5\xfd\xd4\x13>)\xbd\xd5\a\xa8\x82\xdb*'\x8f\x94W\xecQ\x95\xfd\xbc\x99\xaf\xc5\xd3\xe7\x9cCcp\xa4XJ\xdc\x14\xb0H1\xa9+\xf8\x00\\XjKlX\xb2O\x0fuY\xf4v\x0e\xce\xf8W+]jU\x89M_\xd5n\xef5\xee\x15W\x17\xedq\xb5\f{P\xa2!\x0fh\x8c\xde\v\x8efJ\x9e/*Q&\f\xdeĪS\x85\x82\xd8\xd7n4v\x82\x02\x069\xc5(\x93W\xed\xb5<\x89\x85\x8e\x96\t\x15\xfd\xb3\xfd<$\x0eS\xa7B\xa8\x1c*\x9ez\xa73\x1c:\xe4\x1f\x8b\x1c\x0e\xc2mcZ\x93\xc3h\x82+\x11E\xd7\x0e\x8fÇ=\xcc\xcf[$\xb9X\xf6\x10,\x96\x06]\xf0(\x94\xe4$\xe40\x05\xc0\x17oCRd\xb1\x11\x18Y\x15\xf2\xb7;<\xf6\x89\xbda\xc8Ԗ݂zG\xfdJ\x06j\xb0B\x83ʍ&d\x1a \x8cB\x87!'s]ZJ\xc7%6\xce\xce\xf4\x1e\xcd^\xe0av\xd0f'\xd4fJ\x14OS|\xccB\xab7\xfb.\xfc\x19U\xf2\xf9\xf1\xfeq\x0e\v\xceA\xbb-\x1a\xb2R\xe5ev\xa8N'\xf2>\xd4\xc5\xf7\xe0\x05\xff\xd3\xdd[\xf9\xd0M\x8c\x8c\x9b\x9c\xac\x82\u007f\x1f\xa9\x8d\np\x88\x9aU\xb4\x836@Ս\x8c['\xeb\xc5\xfc1f\xbd~\x17ܽ(\xd1P\xee\x1f\xe6\xc5\x1d\xf6S\xe2\xc5\x10J]\xfb\xd5\xf8\xc9\r\xbcP\\\x94\xd4$\x9d{~\x9e]\xf8h\x9f\xff\xea\x14\u007fY\xd5\b;U\xaf\xabH\x1f\xbb\x92\xed\xb8\x17\x93M\xaaJ\x16\x1d57\x16\x14R\xd5b\xa6\xcfU\b\xf4R+Eq\xe64\xb0Sں\xb3\xfd\x1c\xfd\x86\xa8_\xfbr\x87\x03\xa2\a*|\nb\x99\xd3\xf8\x11\xa1\xf0\x16C\n\xbd\x0e\x00nypɖhn\xa3X.H\xecT\xd8\x18,\x17\xb0\xf6\x8aK\xccX\x0e[Tԥ\x8b\xeaH\xad\xe2\xf3\xc3j4.\x13\x8f\xa1\aH}vfs\f{\xcc\xc2sX\x1f\a\xb5\xfb\xa6j\x8d\xc1J\xfc|S\xb5\xa7 \x96\tn\x98ۂPVpJ\xa2C\xbaG\x9a\xa9|\x9d\xea\xf4c\xca\no4\xc6\xe5\xf8\x8d0^\x1b\u0099ϫ\x91\xf1\x94\x84Nz\xe7\xfb\x94\xb7σv<6G\xb4h\xc7\xcf\x1fb\xdfS\x0eJ\xdb\x19\x8c\x97\xa1\xfc\x95\xee)\x9fo\f\x03\x94\xaa\xb76\x06m\xa3\x15'\xff{]\xef\xd4\xc2\xfd5:\xa81\x03Nϳ\xd5ٛ\xcc\xf9ͱ \x0e\xf8o\x1b\f\xe2\x91V\xb7\xfd\xd6\xebp\xd6Й\r~\xe31\xe0]g\x0e\xa0\xc9R\x81W\xa1[\nU\xb8\x80\u007f(\xb8\xa79\x91j\b\x9f\x13F\xea\x10\x86\xf5\\\xe9\x03}\xdcY-,\x00Z\xc5:J3\x10M\xe2q\xac\f\xaf\x0eBJ\xaa\xa4\x06k\xbd\x1f\xa9\xa4\xd4\xe6\x19\x94G`\x96\x88\xd8\xff\xa1\xf8P\xbc\xfb\x9dg\fɬ\xa3\xa1\x01\xf97܋\xfe\xa9Ȑ͇\x81|\x0eޓk\xd3\xcdOyܜ\x99$\xf6\xd3@\xfdJH\xea\xc5F\"\xbd\xad\xe2\xc3\xe3\xc7O\xab\x87;\x1bZf\x1a\xec\a\x8b\x1e\xc8|6\x00\xa4\x9eY\xa7y\xde[\x87f\xc4\xd8'[\t\vJ\x83\xd4js\x16\n\xf1J\xd3=uI\xd1u\xb4\x01\x8e4\x98S\x94\x97[\xa66؞\xd8$\xec\x1d\x94\xe4\x18C\xa4\xe7\xde\xd1z\x83P\xe3\xae\xf0\n\x1b>\x8bak<8\xe4mE\xc7\x0fxO\xa8\x93-/\f\x137\xb8\xeeI\xe7\x1aJDN]>\x80n\xaf_6,\x0eϵoj\xff\x8b\x8f\xb8\x87\xea3\xdb\x1ev\xff\xfe\xba\x87\x9f\x17\xae\x97W\x92\xc8\x1a\x96\xde\xd0\b\xd4\xe6\xdd\x10Lc\xb9\xf7u\xc7\x1c\x8b\xb3\xdf$\xbao\xfa\xbfW\xdc\xd4e\xa4\xde\xf4\x1e\xb5\xbf\xea|l\xef\xd2\x0f/\xf1\x94>\xbc\xa0\xb1\x92\x8aK\x87ȔQғ\xb6\x88Q\xf5h\x1c\xf2\xaf\xfd\xe3\xfaw1\xec\xf2\x99{\xb8-\xa9\x9eǟ\xa0\xe0\xef\xff\x9c\xc4U\x91\xbfd\x1c\xf4\xf0\xff\x01\x00\x00\xff\xff\xed\x93\x00\x8d\x01\x1b\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4\x96\xcdn\xe46\f\x80\xef~\nb{\xd8Kdz\xc1^\n\xdf\xda\xec\x16\b\xda\x06A\xb2ͥ\xe8A#q\xc6ldI%\xa9Iӧ/$ۙ\x9f8\xc8\xf6\xb0\xbe\x89\xa2\xf8\xf3\x91\x94լV\xab\xc6$\xbaG\x16\x8a\xa1\x03\x93\b\xffQ\fe%\xed\xc3\x0f\xd2R\\\xef/6\xa8\xe6\xa2y\xa0\xe0:\xb8̢q\xb8E\x89\x99-~\xc2-\x05R\x8a\xa1\x19P\x8d3j\xba\x06\xc0\x84\x10\xd5\x14\xb1\x94%\x80\x8dA9z\x8f\xbc\xdaah\x1f\xf2\x067\x99\xbcC\xae\x1ef\xff\xfb\x0f\xed\xc7\xf6C\x03`\x19\xeb\xf1/4\xa0\xa8\x19R\a!{\xdf\x00\x043`\a\x0e=*n\x8c}ȉ\xf1\uf322\xd2\xee\xd1#ǖb#\tmq\xbc\xe3\x98S\a\x87\x8d\xf1\xfc\x14ԘЧj\xea\xa7j\xeav4Uw=\x89\xfe\xf2\x9aƯ4i%\x9f\xd9\xf8倪\x82P\xd8eoxQ\xa5\x01H\x8c\x82\xbc\xc7\xdf\xc3C\x88\x8f\xe1gB賈\xad\xf1\x82\r\x80ؘ\xb0\x83\xeb\x12u2\x16]\x03\xb07\x9e\\\xc53\xe6\x11\x13\x86\x1fo\xae\xee?\xde\xd9\x1e\a3\n\x01\x1c\x8aeJUo)\a \x01\x03S$\xa0q\n\x10b@\x88\fCd\x841Zi'\x93\x89cBV\x9a\t\x96\xef\xa8\u007f\x9eeg\xceߗ\xe8F\x1dp\xa5cP@{\x84\xa9\xee\xe8@j\xe4\x10\xb7\xa0=\t0V,a\xec\xa1#\xb3PTL\x80\xb8\xf9\v\xad\xb6pWб\x80\xf41{W\xdal\x8f\xac\xc0h\xe3.пϖ\xa5\xe4W\\z\xa3s\x81珂\"\a\xe3\v\u05cc߃\t\x0e\x06\xf3\x04\x8c\xc5\a\xe4pd\xad\xaaH\v\xbf\x158\x14\xb6\xb1\x83^5I\xb7^\xefH牱q\x18r }Z\u05fe\xa7M\xd6Ȳv\xb8G\xbf\x16ڭ\f۞\x14\xadfƵI\xb4\xaa\x81\x87:0\xed\xe0\xbe\xe3i\xbc\xe4\xfdQ\xa4\xfaT:A\x94)\xec\x9eŵ\x87_\xe5^\xfaw,\xf3xl\x8c\xff\x80\xb7\x88\n\x95\xdb\xcfw_`vZKpʼ\xd2>\x1c\x93\x03\xf8\x02\x8a\xc2\x16y,ܖ\xe3P-bp)Rк\xb0\x9e0\x9cB\x97\xbc\x19Hen\xbfR\x9f\x16.\xeb\xbd\x01\x1b\x84\x9c\x9cQt-\\\x05\xb84\x03\xfaK#\xf8ͱ\x17²*H\xdf\x06\u007f|ݝ*\x8e\xb4\x9e\xc5\xf3]\xb4X\xa1\x85\xb1\xbcKhK\xcd\n\xb8r\x96\xb6d\xeb\x18\xc062<\xf6d\xfby,O\x88>\x0fp{$^\x1a\xd8\xf2\x8d\x06ʭr*\u007f%Y\xa8u\"Ɠ^[\x1d\x99y\x93\x82\x1a\xcd\xf2\xbf8\xd4\x133\t\x9b\x991\xe8d\xa7\xde\x02K\x87\xbe&wd\x8e,\xe7y\x9f\x84\xf3\xb9\xaaԿ\x96\xa1 `\xc2\xd3t\f\xb47\n\x8fȥ\xc5m\xcc\xe5\xee@\a.\x9f\xf1\x9aP\xf48\x16\xa5\x94/q\xb4(Ҟi\x91\xe2\xf0\"\x9aW\xebP\xbe\xf2'4\x1b\x8f\x1d(g\\\xac\x9fa6O';\xa97\xf2\xa2\xd8'I\xdf\x14\x8d%\xde8\xde\xcb\xf8\x16\xf0\n7\xe4\xe1\xdc\xcb\n\xae\xf1\xf1\x85\xec*\xdcp\xdc1\x8a\xbcغ\x19I՟\xddW0Yh\xb83\xd1\xe1\x81qqXU\xe8\xab\xe9AQ7\x00\xea\xaf\xd8\x1d\x81\x15\x8dlv3\xeaC\x17\x1bk1)\xba\xeb\xf3\xe7Ļw'\uf0ba\xb418\x1a_C\xf0ǟ\xcdh\x15\xdd\xfd\x1cG\x11\xfe\x17\x00\x00\xff\xff\"\xf7\xf4 \x8c\t\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4WOoܶ\x13\xbd\xebS\f\xf2;\xe4W \xd2&ȥЭuR h\x1a\x04\xb6\xe3K\xd1\x03E\xceJ\xac)\x92\xe5\x90\xeb\xb8E\xbf{1\xa4\xb4\u007f\xb4Z;=to\x1a\x0e\x87\x8f\x8f\xef\r\xb9U]ו\xf0\xfa\x0e\x03ig[\x10^\xe3\u05c8\x96\xbf\xa8\xb9\xff\x9e\x1a\xed6\xbb7\x1dF\xf1\xa6\xba\xd7V\xb5p\x95(\xba\xf1\x1aɥ \xf1\x1dn\xb5\xd5Q;[\x8d\x18\x85\x12Q\xb4\x15\x80\xb0\xd6E\xc1a\xe2O\x00\xe9l\f\xce\x18\fu\x8f\xb6\xb9O\x1dvI\x1b\x85!\xaf0\xaf\xbf{ݼm^W\x002`\x9e~\xabG\xa4(F߂M\xc6T\x00V\x8c\u0602r\x0f\xd68\xa1\x02\xfe\x91\x90\"5;4\x18\\\xa3]E\x1e%/\xda\a\x97|\v\x87\x812w\x02T6\xf3n*s]\xca\xe4\x11\xa3)\xfe\xbc6\xfaQO\x19ޤ \xcc9\x88\x9cmp\x91w\xb6A\x91G\x99V\xf0\x1f\xe8\xe5\x10\xb3r\xfd\xfe\xe6\x16\xe6E\xf3\x11\x9cr^t\xb2\x9fF\a\xe2\x99(m\xb7\x18\xca\xc1e\x95qE\xb4\xca;mc\xfe\x90F\xa3=%\x9dR7\xeaH\xb3l\xf9|\x1a\xb8\xcam\a:\x84䕈\xa8\x1a\xf8`\xe1J\x8ch\xae\x04\xe1\u007fN;3L5S\xfa<\xf1\xc7\xdd\xf24\xb1\xb0\xb5\x0f\xcf\xedl\xf5\x84\x16V\xbe\xf1(\xf9\xbc\x984\x9e\xa7\xb7Zf\v\xc0\xd6\x05\x10\agO\xb45Gu\u05fc\x99A\x89\xd0c<\x8d-P\xdc\xe6\x14^\xf8a\x10\xa7-\xe4\xff\xd8\xf4\r\xf7\x01\x9a \x94\xce\xf0]\xb3\xa8wi\xf55\x8d\xaeb\x98\xa5\xca[g\x1e\xd9\xe8\xdcz\x8e\xd1,\x17\xe5\x1f\xda4\xae\x15\xaf\xe1nj\xf4\xa3\xeb\x9f\x18\xbdr6\xb2\xa0\x9fH\xb9s&\x8dxc\x85\xa7\xc1=\x999ߩ\xfb{f\x99v\x8d\xdcj\xf1\x12\xa4i\xf8\x1a)\x99ՅV\x858\xff\xf2\xbd\xfa\x1c\xcb|5\xcd,\xf3\x84\xd2q\x11\xf8>\x0f\x16#ҡ\r<\xe88\xc0à\xe5\xb0R\x15\xf2\xb4|@\xdc_\x88\x9c\xd4ٱ\xff\x0e6\xebX\a<\x93G\x9dEs\x16d\xc8\xd5Z\xf1\x85\xe7\xd6\vד\x17\x9eul\xb9\xa0\xbfճ9{&U\xa6\x10\xd0ƩF\xbe\xad\x96\x13\xbeŴ\xb3\xe2\xbf\\\u007f|ҹ\xef\x0ey\xf9\x89&\xb4-8|\xc0\x9at\xcfw+\x8f\xb1w\xb3\xb3\x96\x04\x94\xdf\xf1\x1d\xff\xec\xa9\xe1W\xaf\xc3ѓ\xe5\x02\xb4\xf7\xfb\xb4\xd2XЖ+b\xf9z\xc9\xe5\x90\xf2\xb5+\x85=\xc3\xd6!(4\x18QA\xf7X:\xe3#E\x1c\x97x\xb7.\x8c\"\xb6\xc0\x17G\x1d\xf5\x99P\xf8\xf9):\x83-Đ\xd6U\xb4\xb2Y?\b:\xb3\xd5\xc9>?s\xc6\xda\xf1\xef\xcd\xf5\xc4\xf9Å\x0eV\xc3'|8\x8b}\x0eN\"\x11.\x8dq\x01\xfd\x8a\xb8\x17\xa1û\xfd\xcd\xe1+K\xb1\x9e\xde\xe9y\x00 \xbfz\xd5\x11uӛq\x8a\x1c\x1c#\xa4D\x1fQ}Z\xbe\xd4_\xbc8yz\xe7O\xe9\xac\xd2\xe5O\x06\xfc\xfa[U\xaa\xa2\xba\x9bqp\xf0\x9f\x00\x00\x00\xff\xff]]l+\xe3\f\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4Y\xdfs۸\xf1\u007f\xd7_\xb1\x93{\xf0\xf7fB\xea\x92\xfbN\xa7\xa3\xb7\x8b\xddt\xdc\xde9\x9eȗ\x97L\x1eV\xc4JD\r\x02(\x16\x94\xa2v\xfa\xbfw\x16 %Q\xa2e\xf9\xdaK\xf9b\x13\\,\xf6\xf7~\x16\x9a\x14E1A\xaf?Q`\xed\xec\f\xd0k\xfa\x1a\xc9\xca\x1b\x97\x8f\u007f\xe4R\xbb\xe9\xfa͂\"\xbe\x99\x94+\xd74\xad\xd5q;M\x01\xa9\x17mt\x81\xa7\x8a\xd6d\xa6\xacW\x05\x86\xaa֑\xaa\xd8\x06\x9a\xa2\xd7E\x12ܦH.\x1b\xf5]\xe8➯\x0e$\x8d[q\x1bǠ\xedj\xb7\x9c\x02\xecI\xbbK\x80\x81f\xc0n[\x96\u007fo^Y\x12\xab|\xfc\xd3\xfc\x01\xfaC\x93\v\x866O\xd6\xdeo\xe3\xbd\xe1\xc5P\xda.)d\xc7-\x83k\x12G\xb2\xca;mcz\xa9\x8c&;4:\xb7\x8bFG\xf1\xf4\xdf[\xe2(\xfe)\xe1:%4,\bZ\xaf0\x92*\xe1\xd6\xc256d\xae\x91\xe9w7\xbbX\x98\v1\xe9\xf3\x86?\xacCC\xc2l\xad\xddr_(F=t\x94\xfbsO\x95\xf8K\x8c&\xfb\xf4RW)\x05`\xe9\x02\xe01yy\xc0v,5\xe5\xc9Ua\x1e]\xc0\x15\xfd쪃$\u007fB\xa6wc;z\xa9\xa4\xb6\xe54\xa5\x8e5p\xa6\xf1\xacE\xe4YJ\x97\xb9\xc7X?{\xea\xd5\xed2\x1f\x93*Rt\x80\xe05U4(\xed\xa0-GB\x95\x17GX\x02H\xe2\x06\xea\xe8_\xe7\xfaӕ\xb9};\x10[\x03\xe6\xfe\x06\u007f\x99\u007f\xb8\x9b\xfe\xd9eYGybU\x11\v\x1b\x8cԐ\x8d\xaf\x81۪\x06dQA\aRs\xf9R6h\xf5\x928\x96\xdd\t\x14\xf8\xf3\xdb/c6\x03x\xef\x02\xd0Wl\xbc\xa1נ\xb3\x95w\x05\xb5\x0f\x10\xcd\xd9\x10;~\xb0ѱ\xd6㊣\x04R\xa7\xf0&)\x1a\xf1\x91\xc0u\x8a\xb6\x04F?\xd2\f^I\t9\x10\xf1\x9f\x92\r\xffz5\xca\xf3\xffr\x92\xbe\x12\x92WY\xb0]\xcfr\x0eO_\xacN\x8f+/\xedJW\xf3\x0e\xf9\x1c\uf514\xd8Ժ\xaa\xfb!a_=Gs\xa4A\x95K.\xda\xed\xef\x1e\xb6b\xc86\x88<ۢ\x1bC\v\xb4J\xfeg\xcdQ\xd6_l\xb9V_\x90\xa4\xbf\xde\xde|\x9b`n\xf5\x8b3r\x14\x10\xe7\x98\xf0\xeeV\x89\xf9\x96\x9a\xc2Y8\xf5q@\xda\x03\xbb\x11$\xb9\xa3\xb9\x18\xc9E\\\x9d\x00(T*]4\xa0\xb9?\x03\xb2\xce\xe8<\x10\xfe\x01W\f\x18\b\x10\x1a\xf4\xe2\xa7G\xda\x16\xb9I{\xd4\xd2c\xa5\x8dvxeA\x80\xde\x1b=\xd2N\xbbV\xdc\xc1\xc5\x0ey\xcbX\x8b+\x1e\xd7w\xc4\xeay\xf7Yk\xe7\xf1b\f>wGg\\\xb2\x83\xd0\xd1\xed\x81\xeai\xfc\x9e\x00\xd7'\xec&S\xa0\xa0\xabCъ\xf1\xd1e@!\x90~\xb0\xe0\x9d\x1a\xbc\x0f\xe3l\xf0)\xeb\xf3\xec\xf4\x161\xb6|\xf1\xfc\x96\xa8{\xeb\xe5z\x10;\x1e\t+\xfc\x96\t\xaer\x82\x1d\x87\xd7T\xe7\\x}J\x9f.D\x82\xcabE\xddH\xbd~\xc94y>\xec6\x00.\\\x1bw\x03\xe2 ů\xb8\x8b\x9e˧ӑ\x11l\x18\xb2(\x80\x99;\xf8hL\xdaq\x98\xd6\xfbK\xd4$ς\xc4-\xffi\x86\x03\xf8\x1a\xf9\xbcq\xee\x85b,yv5\xe8L\xf6@\x9a\b\xdb\xe6\xf8\x84\x02\xeehs\xb2vk\xef\x83[\x05\xe2\xe3\xd0(\xfa\xf89Q\xb6\x80\xf7)\xce/ַ;\xe0\xbc\xca\x1d\x11\xd4\xce\xf4\xe9\xe9\"\x1a\xb0m\xb3\xa0 z/\xb6\x91xX\x84Og\xfe4E\xec\x8dv\xb0\xbb\xbfB\xc8|\xba\xa1\xa8B\x9bn\xd9$g\xa2\x03\xa5\xd9\x1b<\x9d\x8az\x15\x12\x92\x90\x94\x91\x94\xdeGk\x9f\xa6\x9eB\xfa\xf4\x92[\x8a$͍\xb3\xa3\x18\xb7\xcfOm\xe3\x1f\xfe\xffIġm\xa4ՠ\xa8w_ŀ\xef\x84\xff\u007f\x9b\xf7\x93\x8d\x95-z\xae]\xbc\xbd9\xeb\xed\xf9\x8e\xac\x8f\xf2=hI\xb5+\xdd\xfbuD\xbdˇ--?9\f.N=\x8e\x18\xe2e\xcdc> }\xa6o$\xbe\xa4J\x98\x93ǀ\xf140\xd3}\xf0\xf5\xf1\xaf,\xaf\x81u\xba\x16\x13\xec\x93\xc1P\x1euYډ@;\x17r\xac\x9er\x1c4\x82A\xe1\x1f\x8a\xfe-j\xfeH<\x1c-\xed\u007frz\xb3\u007fKqYt?1\xa5\x0f\x9dZ\xea\xe0\xf0\xeeV\xb5[\xd9\xc3\x10\xac\x04\xb2\x93\xba;\xfe\x91\xe9U\xbe*\xe9\u007f5J\xaf\x95\xb3\x19\xcd\xf2\f>\u007f\x99@w\xd7\xfa\xa9\x97C\x16\xff\x1d\x00\x00\xff\xff\x81\x16-\x05\x9e\x1b\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4Y_s۸\x11\u007fק\xd8\xf1=\xb87\x13R\x97\\\xa7\xd3\xd1\u06dd\xddt\xdc\xde9\x9eȗ\x97L\x1e b)\xa2&\x01\x16\xbb\x90\xacv\xfa\xdd;\v\x90\x92(Q\xb2\x9c\xe9\xa5zI\b,\x16\xbf\xfd\xed\x1f,\xe0I\x96e\x13՚O\xe8\xc98;\x03\xd5\x1a|f\xb4\xf2E\xf9ӟ)7n\xbaz\xbb@Vo'O\xc6\xea\x19\xdc\x04b\xd7|Dr\xc1\x17x\x8b\xa5\xb1\x86\x8d\xb3\x93\x06Yi\xc5j6\x01P\xd6:V2L\xf2\tP8\xcb\xde\xd55\xfal\x896\u007f\n\v\\\x04Sk\xf4q\x87~\xff\xd5\x0f\xf9\x8f\xf9\x0f\x13\x80\xc2c\\\xfeh\x1a$VM;\x03\x1b\xeaz\x02`U\x833h\x9d^\xb9:4\xe8\x91\xd8y\xa4|\x855z\x97\x1b7\xa1\x16\v\xd9u\xe9]hg\xb0\x9bH\x8b;Dɚ\a\xa7?E=\x1f\x93\x9e8U\x1b⿏N\xffb\x88\xa3H[\a\xaf\xea\x11\x1cq\x96\x8c]\x86Z\xf9\xe3\xf9\t@\xeb\x91Я\xf07\xfbd\xddھ7Xk\x9aA\xa9j\x92i*\\\x8b3\xb8\x17\xa4\xad*PO\x00V\xaa6:\U00091c3b\x16\xedO\x0fw\x9f~\x9c\x17\x156*\r\x8afעgӛ(\xbf=\xefn\xc7\x004R\xe1M\x1b5µ\xa8J2\xa0şH\xc0\x15B\xe7\x15\xd4@q\x1bp%pe\bE\xe5\b-8[o\xc6\xd1:\xa8\xd4\n\x81\\\x83\xb0ƺ\xceR\xaf\xa0a\xad6b\u007f\xef.\x890\x05\xad\xf2<\xec\x06F\xb5>~\xb8\xfd0K\xa8$\x84\x96\xb1\x8e\xc9)S\x1a9\xf3\xe5\xb0O'\x97\xc4d\xa4#\xa4\xe0`\aE\xa5\xecHY\x83\xd84Dv\xcb gI~\xfd\xdal=<\xb6\xfb\xdf\xc8\xf1}X\x18\xfeO\x87\xe0Ef\xc5\xd6\xf9E\xb3\xee\xf7\xe2\xf9\xacY\xd2\xc4{\x8b\x8c\xd12\xed\n\x12\xa3\nl\x99\xa6n\x85~ep=];\xffd\xec2\x93@\xccR$\xd04\xb6\xe1\xd3\xef\xe2?_eE\xec\x8c/3%\x8a~\v{d\x1f\x9a\xbeڜ\xbe\xaf\xbb\xf4T\xba\x9ew\x8d\xc7\xe1JI\x89ue\x8a\xaao\xd2w\xd5s4G\x1a\xa5S\xc9Uv\U000fb1ed\x10\x19\xbc\xe0\xd9d\xdd]0SV\xcb\xff\xc9\x10\xcb\xf8\xab\x99\v\xe6\x82$\xfd\xed\xee\xf6\xdb\x04s0\xaf\xce\xc8ц4\xc5D\xeb\xee\xb4\xd0W\x1a\xf4g\xbb\xa9\x8f\x03Ѿ\v\x1c\xe9\xe3\xb62\x177rdUK\x95\xe3\xbb۳\b\xe6[\xb1~\xf7\x1d\xe5]\xfb\xd6k\x92\x10=ӷ\x9dD\x92ԜE\x91\xfa\xee\xb1.\xb8Ð:\x868\"\x1d\xe8W!\x91됴9\xfbH\xb2\xf1\x0e~ \xd1:=\xf8\x1e\xfaw0\xb5#}0\x9c\x8cx\xf12Ê\x03]~\x9d\x89\xe2=g)?\xb9S\x12\xcf\uebfa\xd0\x14N\x9a\xb9\xe1\xe3\xcd9\xcf\xdd\x1c\xcb\xc7\x17\x02\xaf\x13.6\r\xc6\xdbBD\x00kE\xfd\x16\xc7~\x83=mia\xac\x84\xa2\ful\xb6\xa4\x0f,\x95\xa9Q\xc3\xf6\xe9\b\x1e\xe5>\x17\xaf\xcc\xd7ǵ\xb2W\x13\bu\xbc\xe7\x8d\x00>\\U:\xdf(\x9e\x81\\\x933Qp0oC]\xabE\x8d3`\x1f\x0e'O\xa6A\x83Djy>\x0f~M2\xe9\x86\xd5-\x00\xb5p\x81\xb7W\xac.!:\xf3\xaf\xa9\xf3\xf8\xe5\x17\xbcJ\xd1y\x10\x0f\"1\x16Wۤ<\x17X\x10o/\xa19\xdc\"\x83{\\\x1f\x8d\xdd\xd9\a\xef\x96\x1e\xe9\xd0\aY﨣\xf6;\x83\xf71\x02.6\xb8\xdb\xe0\xbc͝\x10T\xae\xee#ױ\xaa\xc1\x86f\x81^\f_l\x18\xa9g\xa0O\xf4\xe3\x1bj\xecyw\xbc\xed\xd6\xf7\xd5*)\xea:\xf8B\xd9\xf8$#\xd1\xc9\x0e\xb4\xa1\xb6V\xc7-|oC<\xf6$8%Cvq\xd1g\x97\xa4t\x9c{͝:¹uv\xb4#\xebS\xc1X\xfe\xd3\x1fO\x9e\x8f\xc62.\a\xa5\xb0\x9b\x15\n\u007f\x16\xfd\xffk\xdd'\x0f_b\xe5\xf9\xb2\xd25\x1f\x88\xbeT\xb5\xa2ⱚ\xb5_~\x8e\xcb\xcdp\x93oQiF\xa89\x18\xda=ؿ\xdd}E\x17e\xdd\x03}\x9c\x80d\x96\xdeۼ{\x8c\xeaFv\a\x96*\xa4\xd7B}\u007f\xf8B\u007fu5xp\x8f\x9f\x85\xb3ڤ\xbf.\xc0\xe7/\x13螨>\xf58d\xf0\xbf\x01\x00\x00\xff\xff\x98\xaaEc\xdc\x18\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4W\xcdn\xe36\x10\xbe\xfb)\x06\xdb\xc3^*y\x83\xbd\x14\xba\xb5i\x17\b\x9a\x04\vg\x9bK\xd1\x03E\x8d\xeci(\x92\xe5\f\x9d\xbaO_\x90\x92lٖ\xbd\xc1\x02\xab\x1b\x87Ùo\xbe\xf9!\xb5(\x8ab\xa1<=c`r\xb6\x02\xe5\t\xff\x15\xb4i\xc5\xe5\xcbO\\\x92[noj\x14u\xb3x!\xdbTp\x1bY\\\xb7Bv1h\xfc\x15[\xb2$\xe4\xec\xa2CQ\x8d\x12U-\x00\x94\xb5NT\x12sZ\x02hg%8c0\x14k\xb4\xe5K\xac\xb1\x8ed\x1a\f\xd9\xc3\xe8\u007f\xfb\xa1\xfcX~X\x00\xe8\x80\xf9\xf8\x17\xea\x90Eu\xbe\x02\x1b\x8dY\x00X\xd5a\x05\x01YH\a\xf4\x8eI\\ \xe4r\x8b\x06\x83+\xc9-أNn\xd7\xc1E_\xc1a\xa3?=@\xea\xc3YeC\xab\xd1\xd0.o\x19b\xf9}v\xfb\x9eX\xb2\x8a71(3\a$o3\xd9u4*\x9c)$\a> c\xd8\xe2\x1f\xf6źW\xfb\x89\xd04\\A\xab\f\xe3\x02\x80\xb5\xf3X\xc1c\x82\xea\x95\xc6f\x01\xb0U\x86\x9a\xccH\x0f\xdey\xb4?\u007f\xbe{\xfe\xf8\xa47ة^\x98,;\x8fAh\x8c1}\x93\xfc\xeee\x00\r\xb2\x0e\xe4\xb3Ex\x9fL\xf5:Ф\x8c\"\x83l\x10\x86\xbc`\x03\x9c݀kA6\xc4\x100\xc7`\xfb\x1cO\xccBRQ\x16\\\xfd7j)\xe1)\xc5\x19\x18x\xe3\xa2iR\x19l1\b\x04\xd4nm\u9ffde\x06q٥Q\x82\x03\xc5\xe3GV0Xe\x12\t\x11\u007f\x04e\x1b\xe8\xd4\x0e\x02&\x1f\x10\xed\xc4ZV\xe1\x12\x1e\\@ ۺ\n6\"\x9e\xab\xe5rM2V\xb4v]\x17-\xc9n\x99\xeb\x92\xea(.\xf0\xb2\xc1-\x9a%ӺPAoHPK\f\xb8T\x9e\x8a\f\xdc\xe6\x82.\xbb\xe6\x870\x94?\xbf\x9f \x95]J\x1bK \xbbދs\x95]\xe4=\x15\x19\x10\x83\x1a\x8e\xf5\xf8\x0f\xf4&Qbe\xf5\xdb\xd3\x17\x18\x9d\xe6\x14\x1cs\x9e\xd9>\x1c\xe3\x03\xf1\x89(\xb2-\x86>qmp]\xb6\x88\xb6\xf1\x8e\xac\xe4\x856\x84\xf6\x98t\x8euG\x922\xfdOD\x96\x94\x9f\x12ns_C\x8d\x10}\xa3\x04\x9b\x12\xee,ܪ\x0eͭb\xfc\xee\xb4'\x86\xb9H\x94~\x9d\xf8\xe98:V\xec\xd9ڋ\xc7i1\x9b\xa1\xd3\xfe\u007f\xf2\xa8S\xc2\x12k\xe9 \xb5\xa4s\x0f@\xeb\x02\xa83\xfdrbx\xae9\xd3W+\xfd\x12\xfd\x93\xb8\xa0\xd6x\xef\xf4\xa4\xcd/\xa0\xfae\xee\xc4\b+\x8d\xb8\xbeQq^\xf1\xc42\x80l\x94L:T\x14\xd9}\x9b\xcf\xc4q\x91\xf2L\xbbJ\xedj\x95\xd5\xf8)\u05ceջ\xab\xb1<\xcc\x1cH\xa1l\xdc+\xb8V\xd0NM\x8e(k<\v\"D\xfbf\x90\xfdL\xbekRi\xb5\x84\xe1*\xc0Չ\xf2\xc8s\x1b\x8d\x19,\x15\xdau^\t\xd5\x06\xc7Fn]8\x83H\xbd\x8d]\xdf\xd5\xdf\xc6\xef֙\xd8\xe1\xfen\xb8\x8a\xfc\xf9XwZ \xbd`\x00\x91B\x80p|\x05N\xbf\xa1&\x18\xbck\x06\x00C\xd1r\x8a\xf3\x8d\xd8Sr)\xe0\xd14,\xe6\x8b\xffHc\xae\xa2\x8e\x14N\xb3y\xb4y\xc2\xd7W\x87\x81(\x89\xfc\xf6q\x90\xd5Gbu\f\x01\xad\fF\xf2M\xf8M\x03\xc1(\x96I[\xa47\xd0\xd5<ߟ돐\x92)\x90$\x98vѫ\xe2\xb9~i]\xe8\x94T\x90F{\x91\x0e\x9d\xec\xa7\x17\x98\xaa\rV !\x9en^\x9e\bȬ\xd6\xd7#x\xe8u\xfa\xabp8\x00\xaavQ.\x10\x9b/\xc5+\xd4^E\xe47\x8a\xaf\xe3\xf9\x9c4\xe6Ҋou\x8e6v\xa7.\nx\xc4\xd73\xd9\nUs\xdas\x05<:\x99۸\x10\xd3L-\x9f\x88\x0eO\xec\x9b\xc3*\xd7]1<\xa9\xf3\x06@~\x996\x93\x14sߛ\x83\xe4\xd0 Jk\xf4\x82\xcd\xe3\xe9\x93\xfaݻ\xa3\x17r^jg\x1b\xea\xff\a\xe0Ͽ\x16\xbdUl\x9eG\x1cI\xf8\u007f\x00\x00\x00\xff\xfflC\xbf\xee\x8e\f\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec}\xfds\x1c7r\xe8\xef\xfe+P<\xd7#\x99\xe3.\xa5\xbb\xe4*Q\xa5\x9e\x8b'Q>\x96-jK\xa4\xe5w\xe5s\x1c\xecL\xef.\xc2\x19`\x0e\xc0,\xb9\x89\xf3\xbf\xbfB\x03\x98\x8f\xdd\xd9\x0f`HJ\xf2\rR\x953W3=@\xa3\xd1\xdfݠ\x05\xfb\bR1\xc1_\x11Z0x\xd0\xc0\xcd_j|\xf7\xafj\xcc\xc4\xf9\xf2\xe5\x144}\xf9\xd5\x1d\xe3\xe9+\xf2\xbaTZ\xe4\x1f@\x89R&\xf0\x06f\x8c3\xcd\x04\xff*\aMS\xaa髯\b\xa1\x9c\vM\xcd\xcf\xca\xfcIH\"\xb8\x96\"\xcb@\x8e\xe6\xc0\xc7w\xe5\x14\xa6%\xcbR\x90\xf8\x05\xff\xfd\xe5\x8b\xf1\x1f\xc7/\xbe\"$\x91\x80\xaf߲\x1c\x94\xa6y\xf1\x8a\xf02˾\"\x84\xd3\x1c^\x11\tJ\v\tj\xbc\x84\f\xa4\x183\xf1\x95* 1\x1f\x9bKQ\x16\xafH\xfd\x0f\xf6\x1d7\x11\xbb\x88\x0f\xf6u\xfc%cJ\u007f\xd7\xfc\xf5{\xa64\xfeK\x91\x95\x92f\xf5\xc7\xf0G\xc5\xf8\xbc̨\xac~\xfe\x8a\x90B\x82\x02\xb9\x84\x1f\xf8\x1d\x17\xf7\xfc-\x83,U\xafȌf\xca\xfc\xb3JD\x01\xafȵ\x99EA\x13H\xbf\"dI3\x96\xe2\x12\xed\xbcD\x01\xfcbr\xf5\xf1\x8f7\xc9\x02rj\u007f$$\x05\x95HV\xe0s~~\x84)B\xc9G\\\x9f\x99\x04n\x04\xd1\v\xaa\x89\x04\x9c\n\u05ca\xe8\x05\x10Z\x14\x19K\xf0+D\xcc\x1cHR\xbd\xa3\xc8L\x8a\xbc\x865\xa5\xc9]Y\x10-\b%\x9a\xca9h\xf2]9\x05\xc9A\x83\"IV*\rr\xec\xc0\x14R\x14 5\xf3\x885\xa3AJ\xd5okk86\x8b\xb4ϐ\xd4\x10\x0fة:\x12\x80\x94(D\x00\x113\xa2\x17L\xd5K\xc2e4\xc0\x12\xf3\b\xe5DL\xff\v\x12=&7f\a\xa4\"j!\xca,5\x14\xb7\x04iP\x92\x889g\xff]AVf\x81\xe6\x93\x19\xd5\xe0v\xda\x0f\xc65HN3\xb3=%\x9c\x11\xcaS\x92\xd3\x15\x91`\xbeAJހ\x86\x8f\xa81y\x87[\xc2g\xe2\x15Yh]\xa8W\xe7\xe7s\xa6\xfd\xe1ID\x9e\x97\x9c\xe9\xd59\x1e\x016-\xb5\x90\xea<\x85%d\xe7\x8a\xcdGT&\v\xa6!ѥ\x84sZ\xb0\x11N\x9c\xe3\xd9\x19\xe7\xe9\xef\xaa\xcd:n\xccT\xaf\fA)-\x19\x9fW?#ioŻ!qK9\xf65;\xff\x1a\xbd\xe6'\x83\x95\x0f\x977\xb7M\xaab\xaa\x8ds\xc4v\x83\xd0j\xc4\x1bD1>\x03i7\x0ei\xcb@\x04\x9e\x16\x82q\x8d\u007f$\x19\x03\xdeF\xba*\xa79\xd3f\xa7\xff^\x822\xa4+\xc6\xe45\xb2\x102\x05R\x16)Ր\x8e\xc9\x15'\xafi\x0e\xd9k\xaa\xe0\xc9\xd1n0\xacF\x06\xa5\xfb\x11\xdf\xe4|\xed\a-\xb6\xaa\x9f=\x8b\xea\xdc!w\xbao\nHZ'ü\xc4f\xfe\x18τl\x1d~\xf3ʸ\x01\xb2\xebX\x9aa϶aA\xed\xdf\xd7&\xf1\xe7\xea1C+\xe6\xf3%g\u007f/\x01Y\xa8=\x93\xb0\xc9.d\x83\x9d6\x87!\x81\xf1گ\x9d\x184\x03\x1e\x92\xacL!\xadؤ\xda9\xd3ˍ\xc7Q\xc8P\xc6\r\x8d\x1b\xa6n\xa6\xcb\xeb\u007fE\x06I;fi\xe8\x8cq\v\x8d0\x8eK\xec\xc0\xac\x19LC\xbe1\xad\x1dk\"(\xb5\xe84\x83WD\xcbr\xfd\xdb\xf6=*%]u\xa2\xc2K\xd9\xc30Q=\xed\x8ey\xc6\x12ܲ\xea0#2\xbe$<,\x84\xb8۽\xf6\xbf\x98'jnD\x12\xd4N\xc8\x14\x16tɄt\xabu\"a\n\x04\x1e )5J\xe05\xa8%2E!I!\x94\u07b6\xeem\xa7\x8b4\xa5\xea\xe6?mE\xd8\xc6z\x1c\x13\xf0[i\x96\xd7b\b\x82\x83\x99cn\x98_\xfd\xac\x14\xa5}Vu~\x81l\xc3\x02\x99R\x05)\x11n\xaf\xcb\f\x94\xfbR\x8a\x8c\xa6>=g[\x00W\x8b\xb6\xb22\xa3SȈ\x82\f\x12-\xe4:\xf6\xf6\xe3Ў\xfd\x9c`\v\xf6:x\x82㞎\x976ف\xd8\n\x93\x90\xfb\x05K\x16V\x8c\x19\x1aD($\x15\xa0\xf0\x90\x18\xb5jս8\xb2{\xaf\xed\xd8qL\xea\xb1\xf3\xc0\xac\xc3\xda<:\xf5\xd8\xcbL걇\xad\xb4qYk\x91\xff0\xa8\xf4\xdc1\x980\xaf6^|L\xc2D5ߨ\xa2W3\x02y\xa1Wg\x84i\xff+\xaa\xf3h9mEO\xf5\xed/n#Bi\xfaj\xfd\xbdG\xa4鞻P}\xfa\x8b\xd9\x04d\xf67\x8e\xd7\x1f\xb8\x01\xdf7\xdf9#lVm@zFf,\xd3 \xd7vb\xd7r\xc5\xee\x9d苂\xfd\x92ʌ\x9c\xeadq\xf9`4\x10U;<\x0e\xc2\xc6\xfa\xabVq\xf3\xbak[\x98\xee\x84J\xd0xb\x12rk\x92\xdd\"\x06\xeb_\x8c\xbeG.\xae\xdf@\xba\x1d)\xe4\x10\n\xdbX\xc2\xc5\xda4\x9b\x9fuz\xe8a\vpJJ\xa5\xc3[\xf3\xfa\x8cPr\a+\xab]\x18c\xbf\x00I\xcdg\xcc\xc3{!J@\x1b\x1f\t\xea\x0eV\bę\xed{\xde=l\xeb\xed\xb8\x83\xd5\xfe\x87\xd6\xd0ff\xe3\f,\x8b?\xf3\x03\"\x00M\xbeCQF\xd0\xe9\xe29̾E\x91CY\x84\x1f\x1e\xdb\xc1˫\xb6\xa9\xe1\x90\u008dHG<\x94\x14b\xd3\xe5\xda5\x16T\x91)\x00\xf7H\xdbs\xe8\xedx:I\x9b3~\x85\xc0\xc9\xcbG\x95ˤFQ\xc4\xf6y\xe4V\x1bX\xfd`%ǡȾ_\x80\x84\x16\rl\xba\x88Q\xaf\xe3B7\xec\xf4\xc3\x10m\xe7q\xacȌI\xa5\x9b\x93T\xa4T\x87ml\xd0n\x99\x19߲\x1cD\xa9\x83qzY\xbfۊ\xbd\xe5\xf4\x81\xe5eNh.ʽB\xd7\x0e#\x03X^\x05\xc9\x1cF\xef)\xd3Ƞ\fT\xf4Tha\xb0^d\xa0\x0f\xd3;\xa703L$\x11\\\xb1\x14\xa4\x0f\xd7\xda}b\xc2\x1c\xbb\x19eY\xb9\x19\xb4\xe8\x1aaf \xbf\x942\xc2\n|o\xdfk\xf8\xd8\x16⾍\x98\x03\x97\xbe\xa0K lF\x98&\xc0\x13\xb3\x17 -\x83\xc5\x0f8$ J\xf6\xea1v\x1c\u008c\xcd\x00^\xe6\x87,|\x84\xe7\x92\xf1\x1d\xee\xa4\xe6\xc3o)\xdb\xe5\f\xf4#h\x9b\f\x8d\xc5\x1e\x80\x1f\xebw\x9f\xe1\x00\xd4\xcc`\xa72R\x8f)\x90\x0f@ӕ?\x05Tkc\x06\xe2\x8e\v\"K\xde\xe4b\x8fL\xff\x87\xdbP\xee\xfb\x8fe\x1e1\xce\xf6n\xe4\x9aw\x9b\xe9\xa6\xf6a\x00<\x99\xf6a\x80W\xa2(ܽq\xd5z\xdd\b\x05\xaf\xb4\xe2\xac+\n9X\x13\x99\x821\x00!\xb5\xee\xa2BTf\xbeM-\xe9\f\xe7v\xae\xebpeb\xcd\x11\xeaL\xb9f\xd2U\x83\xd0\x0f\xf1Wڱ\x12%\xb9\xa7\\{ҮԪB\x1cD\xdba\xfbh\a\x95\xf3\x83\x9f\xdd\xc8\xe8\xf2J\xa3O\xac\x02\xae\xe5\nS~\x0e\x9b\xae\x1d\xc6\xf0KErgT\x84\x9c\xce\xe1\xf8X\x91\xd7\xef\xdex}\xc1\xb0\xff\x83\xb9\xbb\x1d\xcc\xc6\x18\v)\x96,5\xaa\xccG*\x19\x9df\xc6\xc0\x9c\x81\x04\x9e\x80\"_\x9f|\xbc\xf8\xf0\xcb\xf5Ż\xcb\xd3\x00\xd0\xc6(\x85\x87\x82rCq\xa5\xf2Ҹ\xdao3y\xe0K&\x057\xa8\t\xc1\xc3ՌP\xb2\xf43M\xaa<(c\xd8dKH\xcf\\|ĭ \x04\x1f\x96M2^\x94\xda{\x12\xefY\x96a\x96\x15O\x16\x94\xcf\r\x96n\x17!@\x1b\xf8#j\xc55}0sF\x15R%\xb4\x80\x14\xe9\x97\xd0\x00\x90\xa9(\xcdҿ\xfe\xfa\x8c0xE\xben|bL.\x1d\xd4z\v\x03 \xe3j9,AZ\x1d\xd7n\xe0\x19\x910\xa72\xcd@)Á\xee\x17\xa0\x17p\x98\xd3\xd2\x0e\xeb\xfap[\x06\xde\xebi\xa8\xaf+\x93-\x00pG\x96\xdb]\x95\x929f\xe2<\x15\x89:\xd7TݩsƍH\x19\xa5T\xd3Q\x83\t\x9d[\x890r\xd2i\xe4m\xbcQE\xac翓%\xe7\x8c\xcfG\xb4z\x8a\xf1\x11\x1d\xa9\x05d\xd9\xf1\xc1\xd3\r`\x9d\x0ema\xd6X\xf3\xa5\xc3]\xd5A\x86\xb2\x1dm\xfevY\xb13\xfb\xd51\xb9\x16z{&\xd1\xf6Q1r\xc4븓\xe3]^\xdf~\xf8\xeb\xe4\xfd\xd5\xf5m\x18\xa3k\xb2\xc8\xed\x8c/\x00f7\x8b\xec`|\x81\xc7d+\x8bl3\xbe\x00\xa8{Y\xa4c|A\x9cr/\x8b\x8c\x14\x1c\xbbXd\x83\xf1\x85\xcc\xf5\x00\x16\x89k\b\x809\xb0\xc8\u007f0\x16\t|\x19\xc9\x1e\xbfwj{\xe3(W\xfb\x1c\"\x9a\xb5\xc0\x18/\xe3m.ы8\x82\xb1\xddv\n\xf1\xe5G\xda\x0ea\xf3\xe62\x03\xe0\x92\x9a\xf4}\xa6*\n\x82\xca\x02\n!\xf8p\xedގݑ\x8d\xee\xb1\x19\xefp9\xe0\xb1x \r\\\x8c\xc9;\x17ӥ\xe4\xf5/Wo.\xafo\xaf\xde^]~\bA\x06\x89=#ć\xe6{\xa1\xe4\xf8\xf1L\n;\xb6\x18\x16\x85\x84%\x13e\x95\x9e\x1b\f\xb7\xf3xn\x9c\xb6\xf0\xe9b\xe0`E\x14\xc8%K\xa0\xfb3\xa1\xfby\x80\r\x14\f\xb1K!h\x89\xf9`\x88\x8f\xaa\x16\xd8q\x80r\x10\f\xf3\t\xac(;\xf6\xdbR\xc1 k\xc5b\x8b\xba\x10\f\x11Ջ70\xa3ef\xfd\x13GG\xe3å\xb5\x1d\xfd\xd8\xcb[)\x0er 7G\x8b\xc5\xdc\xd8\xe2\r\xef;}\f\xc6{\xec\xd2\xebZ\xc2\xd5\x1a\x10\x110\xb3\x12\xbc\xc5\x11\x90\x9bS\x8fXyFl\x18m\xc6\xe6\xefh\xf1\x1d\xac>\xc0,\x1c\xc0:\xb21\xf3\xce%\xaba\x81a\x04Db亝V8\xeb\xeb\x87\x0frx>b\xd7h\xe1\xe2\xd6eM\xa2ff\xd0\x12\xb3\x18\xd2\xe7\x00\xf9\x11\xa3\xb9\xf8\xd1\x16\xd7M\x15\xc6\xf1\xbe\xe8e\x1djz$\x82'Phu.\x96FJ\xc2\xfd\xf9\xbd\x90wƖ0\x9c}d#\x01\xea\x1c\xd3\xf0\xcf\u007f\x87\xff\x13=\xa3\xdb\xf7o\u07bf\"\x17iJ\x04\xb2\xd1R\xc1\xac\xccl\x8aO\x84\x1c\xf6\xa3.\xec=\xc32\xd33R\xb2\xf4\x9bPF\xeaGoz\x10\x85\xcd\xf3z\x14\x9a\xb8\xc1\xe8\xe4*¤m\x0fCRչ7\xa6-\xd3\n\xcfO^\xaapV\xed\xc7\x14\xa2U>7-D\xf6T\x88\f(\x8f\x80qh\xf8\xabk\x1c\x92V\xd85\x0e\x0e\x91u\r\xa4\xf5ǐ\x05ǵ0\xb0)r\"\\:\x92:\x15\xe2\x15QeQ\b\xa9UU0<6\x87=\\\x97%\x8d\x9a\xe3qU\xbdsF\xfe\xb3\xfa\x11s\xca\xd5O\xc7\xc7\xff\xfe\xdd\xe5_\xff\xef\xf1\xf1\xcf\xff\x19\xf7\x95\x1ab\xa3\xa3C\u007f\xb0\xaa\x80d\xccE\n\x86\x1d\x9f\xd9?\x9d\x05q\x91`x\xff:\x1a1JS]\xaa\xf1B(}59\xf3\u007f\x16\"]\xffK\x05+\x82\xe4\x11\x84sw\x8b\x84\x90ѢQ\xdfJ\xc1\x8a\xb4h.\xe3z.\x18J\xc5\xe6\x15\x13\xaa\x17F\xa7\xbb\x97Lk\x88a\x1bv\x18;\vd\xae\x88\x98\x9d\x19>V\xab\xe1˗G\x9fL\x9d\x98\xf9%>\xca\x16 \xae\x9cJ\x81\x90\xe3%\x88W\xb4\xbc}Z\xe5\\E\x83\xbc\x98\\\xf9\xd6\x1a\x9f\b\xdd\xfd\xe4G\xb5U\xcf-E|\x1a\xe9\xdb'\x90&\x1ev\x9c\xee3k\xbbl^\xd9\xfc\xe9C\xea嶏\x8ca\a\x0e\xcaӺ\vlj\xfdq\x9c\x14e\x1c'v\xef\xe7\x90\v\xb9:\xf3\u007fB\xb1\x80\x1c$\xcdFJ\vI\xe7\x91l\xdeO\x13\xa7W\xffe?\x16Ǚ\x1b\x8bߜe\xb83\x878o^RJced+/\xff!\xfd$\x92\xa7\xa2\x98\xae& \x87\x8e6Iש\xa8},\xb4\x9aG\xa0\x93c)\xb22\auVi\xf9\xd1`\r4\xe0K\xb2\xa4R}2[%eK\xa6\x0eK\x9e\xec\x1a\x94\xaf\xdeG1\x1f\x82\xfc\xd3N\x9fq\r\xf3h\xd3f\xd4\x1f\t\x9d&\x97/\xba\x16\xa5.\xcax\vi&dNu\x15\x97x(\x84BǦoL\x11\r\xb8\xa5\xaf\xbc<\x8a\x84SP\xadA\xf2W\xe4?N\xfe\xf6\xfb_G\xa7ߜ\x9c\xfc\xf4b\xf4o?\xff\xfe\xe4oc\xfc\x8f\u007f:\xfd\xe6\xf4W\xff\xc7\xefOOON~\xfa\xeeݷ\xb7\x93˟\xd9\xe9\xaf?\xf12\xbf\xb3\u007f\xfdz\xf2\x13\\\xfe| \x90\xd3\xd3o\xbe\x8e\x9c\xf0è\xf6a\x8c\x18\xd7#!Gv\xeb\xf7\x94K\xef\x1a~;\x1e\x87\xef|\xf0:E?QJ\x9a:\xd7'b\x10\xfdԣ\x1e\xcb\xef\xa5\x1d)H$\xe8\xcf\xcb\xe7j\xe7Ԩ\x818Vuk\x8b߀\x1b\xb6\xaf\x89g\xd1S\xdb\x18\u061c\x8b`\b\xb6\x8fw\x8a\xdaV\x86\x1e\xfe\x1d\x04\xfb\xff\xfd\x18\xdcă\x9b\xb89~\xbbn\xe2\x1b{V\x06\x1f\xf1\xa7\xf1\x11G\xbe\x1a\xb3\xca\x112\xa5\x904\xa8\x98\xb9E\xe5{\x85\x05\xa6;s\xbe\xeafI\xa4\x10E\x99Q\x1d\x1b\x9fޞ\x922\xf6\x020&\xf7\xa5θ\xb5A\xf4\xbcw\xbe\xd1E\x96\x11ƭ\xc8\xc3I\xf94\x10\tֶ'T\x91\xa0C\x04K\xe0ڰ\x15\xbe^ͩ\x88\xd2Tj\xc6\xe7c\xf2\xe3\"\xc8\rku)\x977\xc18\xc9\xcbL\xb3\"\x03R\xb5뫪\xf5C\xa0*%\x12F\xb5OJ\xb1\xedk\x94\xf6\xe8E\\hz\x17\x02\xb3\x90\x90@\n<\x01\xec\xeaR6Z\x10NW\x84rrɗ\xf8\xb5\xa0է\xa5M\ued2aS5\xaf\xd6\xd7l\xeeC\x00\xd8O\x92\x82h\x8e\xa9K\x01iww\x0ebzn\x83\x8cr\xed[\xe9T\xb1\xca\x105\"V)\xae\xf24\"\f\x86\rm\xb8\x8e\xb2V\xdalx\x94P\x8a\xfc\x19\xf3TbUӧRK?/\x95\xf4\t\xd4\xd1\xc7SE{\xa9\xa1}T\xd0]\xeag\xb4)X\x9f\x1d/\v\xe3U\xc7>jc\xb4\xfaVH\x98\xb1\x87^<\xe4\x82W\xfbBX\n\\\xb3\x19\x8b\xd0\xe8\x8d\xd6#\xa1\x00\x8e5\xa7@\x93\x85m\xeb\xc6۩ \xe1\xf4\xfb\x89\xb3\xa2\xad%\xff\x18\x8c\xfa\xa6\xcb\xe70p݁\xeb\xee\x1f\xbf-\xae\xeb\x0e\xc2\x17\xc9r\x9f\xc9\"\xc5\n\xc8\xd8\x12\xcd7\x8d*J<\xf5\xcd\xfb!\x02\xd6zȩ\xac[\x13\x9c\xe3\xf7B\x0e\x1f6$\xf4\xfd\xd6j!d\x9b\x03\x8b{\xb2`sCf\x19,!$\xeci\xb5k\x92SN\xe7\xb6\xe5\x9b\x16>|E\x84$\x86\x91H\x96\x06\x15U\xd6f(.҈5Æ2A\xd3\xc6m>!\x8b\xcf\xd8\x1d\x907Pdb\xe5:\xbb\xf1\x94\xdch\xaa\r۹\x01\x1d\x92\x90\x15\xc1\x1ep\x1d\x932\xcb&\"cI\x80o\xbeMjWHcE\x99e\xa4@@c\xf2\x9e\xa3|\xb8\xc8\xee\xe9*(\xdex\rK\x90g\xe4jv-\xf4Ěv\xedj\x05\v2\x00\"\x9b\x91W\xf6^\x1b\xa2\xe9\x1c]\bu\u007fe![\x9f\n\x00\x8b\x02\xe2\x9e)輘\xe5\xf9\x8e\xda\xef\xf0\x9bF\x14ڿ\x9f\x94`26\x83d\x95d\xb1\\\xe9\"\xc1tɺ\xado\xe3|\xaa\x95\xd2\x10\xa2\n\xb96:\xe8\xc4`\xd8\x1e\xad\x10\\\x81m#\xe5\x8fj5\xe3P\xf7\x93\xeaUf\x19\xa7\xa2\x15B\xe9\x1bM\xe5A͊\xea\xd1>\x8d\x13\x0fĐzB\xb3\fR\xc2\xf2\x1cRF5d\xa1~e߭\xae\xe5\x83ë\xc8\\#\xb4p\xf9\xbf\xa0<\xcd@bo.\xe7ukA\xd7 s\xc6iX#\x01R\xa5+\xa1\x83\x10RB\x93D\xc8\xd4\xf5C\xf2\x1do\xa8\f\xf5\x8bT\x1c\r\xb5\x9d\x06\xbd\xaeg\x9d\x05\u009df\"\xb9S\xa4\xe4\x9aeu\v4\xdf\xff\xcc]\xa2\x15\b\xf3p=\xba\xc1F\xaa\xff\x1cUge\x84\xb7̜\xff\xae\xfe'\xfc!Li\x8d\xb7R\x0e\xe91\xb99\xd6\xfa\xac\x01\x92\x03&\x02\n\x0e\xf1\xa1\xe2\x990j\x88!\xa3\xba\x13`%@\xc6\xd8&/\x02j\xfb\x8e\x05\x8al\x11{\x05ѻ\x83Z2\xb5G\xbf\xb8|p/\x90\xe68\xa0\x8dfd\x04.c\x1c\x9a\xfd4\x19v\xf9k\x9f\xb9\xd8L&\x03\xc4Y\x90$e\x12;˯|=a$L\xdf0\x12\xfbj\v\xa1\xc9\xc9\xf1\xf9\xf1ix\xa3\x8d6L\xdf\x19\xc4\xe8\xc8\x19X\x19\x19ڏ\xa8k\x96F\rby\x91\xad\x10\xbf\xc7\xe9\x19a\xb1\xd1VW\xe8(K\xee\xf7ȵs9#\xea\xb0^v\x9bCK\xea;W[X\x06\xb4\x96\xa5\xd5\x1f\"\x81\x9e\x1c\xffz|F@'\xa7\xe4^\xf0c\x8d$0&\xb7\xc2\xd8\xf9\x910\xab\xa5\xaeDI8\xd8fk\xf0Pd,a:X\xda\xfaa\xc46\x11\xa5\xb6\xed\xc3\xf0\xa2*l\x8fs\xf9\x10\xbdK\xb6\xce\xc3\xf0\xc1\x17x>\xad\b'T\x91\x8c-\xe1|\x014Ӌ\xd8\xf9\x1a\x8a\u208f\xfe\x1b\xa4\xc0\xd6;\xdc\xc1\x8b\xf3\x99\x04G\x88\x9a\xa3w\x8eD\xb8\xa1\xbe\xfenT\bވ\xedo!P\xf5#\x1b\xb7\xbf\xdd\xdeN\xbe\x05\xdd\x160\x11h0\xb3\xf1\xb9\xdf\xe8\xd6\x059\x13r\xe3\xea\xc2\xfd\xa3\x9flZ\b\x15\x81\x11\xb2y'\x9eҶ\x1f\xb95\x0exL|\xcc\x0e-\xdae;.\xb3\x8e\\Mb\x93\x84\xfe*J\x83\xa5)\x9df\xab\xaaˡ\x02M\x8e̴c\x93l\x19\xc7=\xfc\v\xd0\x14\xbbIr\xa5\x81\x06u\x11\xaaG\xcf#\u0558\xc7c(\x19\xf6>Å[\u0601\xedR7G\xa3\xb5\x8e\xa3\xf31\x9e\x1e\xebw\x8a\x951\x12\n\xcbX\xdd\xfc>\x01\x03\xdc\xe0\a\x16\xf7\xee\xf7i\x8f\x1c9\xea/\x93\xb4\x8bs=FKգ\x1a\x8bq\x8bts\x00\xa2g\xd67/\x95\xf4̔$]\x91\x1e\x8b\xa3\x1e\x10]U^h\xba\xd4\xfax\x84J\x85ȶ@\xcd\xf1t\xe8\t\xcd\xd8Y\x1f\x8f\x80\x9f>\xc9~$&%\xae\xfdr\x1f\f\xf4\xcay'=\xb5%,\x05\x89,9\xdd,8Ղ\xd0$\xc1n|\xb1\xe5\xb9F\x18 ;»\xeb\x83Z\x905\x80\xf4#\xa8B\x84\xfa\xff\xfc\xe8Q\x18\xf5\x18eQ\x8fP\x14\xd5\xd1[M\x12^\xe6S\x90\xb1\xad\x06|\xb3\x01\xa9[\x04\xb2\x96Q\x19\t\xfa\xdaN\xcd\a1\xbd:A\xf9\x817gm\x8e\x97f\x96\u007f\xfa\x97\u007f\xf9㿌-\x02\xaa\xfc\xccX\x9a\xbe\xba\xb8\xbe\xf8\xe5\xe6\xe3k\xecs\x15\xb7\xd0'\xa8\u007f\xc2\xf2\xfaH\x89ҎG# \x83\xb5RaK\xa8xW\x8b\xb1\n\x9c\xbf\xd8:dU#\xf6\x14m. C\xf9\x04\x9c$^(\x8d\xf0\xb8<\xa7\xed\xab\x93\xe2F$w\xbd\xad\xdf\xe3\xdb\xd7\x13\v\xa86\x80#0O\xb9w\xc92\xbe\x14\xd9\xd2\xde\xf1t\xfbz\x82\x88\x89\xd9K\xf3.\xfa\xd0\xd1U\xb62\xf3\xf3\x95\xcf6\xe9$\x02&\xcb\vw\x9b\x19%\x12hƔf\t~)&\xe8凙exv\xcaga\xe5\x1f\xbf\xf7I.\xb5\xc1\x1f\u007fl\x1dC\xe82\xf8c\xcd\x14\xeb&\x88+\xfe\x19\xb4\x8aG\xd2*\x9c6!\xfd\xfdt\x83V\x113>G\xad\xe2ˑx\x91/\x16\x12n\xb4(ze\aX\x10\x8f\x92\x1b\xe0o\x1e\xda\x16\xbe'i\xf0&\xda[:/&W\x95\xefY\xb4\x82\ue61a\x11\bS\x95\xc9\xc2\xc798(u\x8ei\x00ea}N\xfe\x8a\xb0\xd0Pb!\x01\xef[\x12\xfc\xac\xaa9GD\x00\xb7?\x82NB\xcf\x05\xfaE\\v\x84\x8b\xaa\xf9M\xea\x97l\x90H\xaa\x16\x80\xdd\xe5\xe1\x81\xd5סS%\xb8\r{\xbaMc\xc1\xa63S\xa4\xa0J\xd9\xc0\x97\xae\x17`?1\x11\xe9\xf1q\xa8\n֘\f\x99K\x9a\x00)@2\x91\x12샖\x8a{N\xa60\xdf\u007f\x8b\xea\xfap\xf4j&鏁\xd1v\x00\xa3\xa1\xd5\xed~\x81@?\xb4.\x01p\xcd;\x12Q\xe7G;|\x84\xd2W;-\x06˵\x90\xf8K\x9ae\xab\xfa\x90\x05Bu\xd5\u007f\xbaښMd\x87\x9e\x03ܚgϏ1\xa4\x8c\xff\x16\x81֭\xf4\x857b\xd3d\x11N\x05\x81i\xecC\xfa͡cH\xbf\xd99\x86\xf4\x1b?\x86\xf4\x9b!\xfdfH\xbf\x19\xd2o\x86\xf4\x9b\xd6\xf8,\x1csC\xfa͐~\xb3>\x86\xf4\x9b\xe01\xa4\xdfl\x1fC\xfa\xcd\xce1\xa4\xdf\xec\x18C\xfaM\xf8\x18\xd2o6\xc6o-P6\xa4\xdf\xfcV\x03eC\xfa\xcda/\x0f\xe97{ǐ~3\xa4\xdf\f\xe97\a|{\xd0*\x86\xf4\x9b߶V\xf1\xe5H\xbc\x1e\xfd\x9b\x82^\xf2\x19'\x13)\xa6э\x9c&\x18\x9bf\x89KW\x11\xb3\xa8\x90\xba\x9fʸ\xbe`\xbdѧ\xd7\xf7\xcc\b\xba\xec\xd6^\xc2\xedSh:\xfb\xa5\x846\xb18<\x82\xee\x1b/\xa9\xf3B\xd8\xffW\xc7\xcf\x1b\x81s\xeb\xd7:\x9c\xe5\xc7\t\xd2\xf0\x88\xf9!\xd1\xf2:\xf6\x1d\x9a\xf0\xb4-R\x1e\xad\x95\xf5\x8d\x92\xc7\xeb'\xd1\xd1\U0006724c?UT|gD\xbc\x19ێ\x80\xbd\x11\r\xdf\x16\u05ceQ\xac\x1b\xb3{\xa4\x98\xf6\xcexv32\x1dc\xf6nIJ7\xa2\xd2\x11P\x9bq\xecΈt\x04\xcc:\x86\xbd-\x1a\x1d\x01\xf4\xf2\x81駋D?b\x14::\x00\xd3KY\x8d\xf5\xa5F\xea!.\xf1\xf4v!A-D\x16\xc8\xe3Z\xfc\xed\x1d\xe3,/ss\xb0\x95aLlY嵆r\f\xcfs\xacd\xb7!&\x03\x96\xa5\x80\xd7\xd1Q\x96\x857\xe6\xc2&b\v\x8a\x96\xbc*\x93\x04 52\xa9\xd1\xd7/\x10\xe2\x1f\xc7՚\xab\xdb\xf6_\x86љ\xbd$\r\xad\xa3?\xfe!b\xbfí\xaa\xa8\x14\x83\xfd\xe9\x05\b7\x10\u007f}S\v\xe2\x05z\x9c\xb3\xe1)\xd2\tv\xa4\x12\x90\xbf\x8a2\xc6\xcaߞF\xb0\x96\x10\x10#\x17cS\bz\xf0\xc4^\xa9\x03\xbb\xd3\x06\fn\xa2\xb0\xb05e\xa0\n\xfeǸ\xc0b\xd3\x05\xa2%\xd5Ӥ\tlO\x11 ,\xce\xd7\xd0/=\xa0oj\xc0\xa3\xdd_VǼ{\xdeH\xddǫ\xd9ד\xd6+\r\xe0i\xd0\xd1?\xf8\xfd\x89\ue24c\xdc\xc7\xf8p\u007f\xafP\u007f|\x98?.Ŀ;\xbc\x1f\xe9\x84\xef\x15\xda\xefA,q\xce\xf7H\xc7{_\xa7{O\x87\xfb\xee\x10~\xe4\xc6=\x81\xa3}\x87\x93\x1d\xdd\xe5\x11 \xbb\x1d\xec}]\xe5\x8f\xec&\x8f\r\xbc\xef\x0e\xba7\xc2\xe7Q\x8apG\xc0=>t\x1eM\xbfq\f=\"x\x10Ɋ\x19g\x9a\xd1\xec\rdtu\x03\x89\xe0i\xa0V\xb3v\x89Ju*\x95\x05f\xed\xe4\b\xd7l]'\xb8\xa0\xee\x86\x89\xf9n\x8b\x04\xfbo\xfc_\xc4=\x113\r\x9c\x9c0\xee\xf7\xfe4\x9c\xe79ý\xf6\xd6T\x87לݗ/<\xe8\xe0Z\xc6/α\x82.%\xa5\x9eʓ\xe6\xc0?\xb6+́\x9d\x95\xa1\x9e\xec\x96;\xcd:\xe4\xda|;p\xc3\xea\xeb\xb5^\xe2\x9c=\xc7@\x8f\xae+\x96\xff\xed\x13Qd\x12\xd4\xde\x04\xa8:\x9d)\x10\x85\x9d\xc9O\xedT\xa6@\x88\x1d\x89O\xddiL\x81p[IO\x11)L\x9fԛ\xf8HiK\xbbS\x96H!bl\xec\xa8t\xa5\xc1R:h\xecNK\x1a,\xa5Ok)}\ued80f9\x88R\u007f6f\xc0\xfd\x82%\x8b\xa6\xb6\xc1rPD\x94\xf1)\xd4F\x8fpS\xea\f\xb6=\xed\x055\xbf!\xcb!\x82\xc2\xc2\xdc\xde\x1d>\x9f\xb5\xde+u\"P\xc0z\xa9\"\x94\xbc\xb9\xbe\xf9\xe5\xfb\x8b?_~?&\x974Y4[=qB\x03\xc5\x1a\xf2\x9a\x05]\x02\xa1\xa4\xe4\xec諒\x99\x90\x9cT_9}\xa6;\xc8#$\x87\xe1,\x01\a\xbd\xb5)\xdf3\x85\rq\x10\x86kQ \x14\x84^\xfeږ%\xe4\xd2\x00\xb1\xfa!ʝ\x05H s\xb6\f2T\fL\x9b\xffChZ5}0\a՜\x12&8\xa1SQ\x06\xb1\xc6\x05\x10\x0eڜ\xe0\xca/%\xb8j\xf5\t+\x15\x04]\v8-\xf1:\xb3B\xb2\x9cJ\x96\xad\x9a\x13\xa4٘\\\v\xafq\xaf\xc2t\x81&\xea\u07bc\xbf\xbc!\xd7\xefoI!\xb1ՒͶ\xc1\u007f\x0fܨ)\x98m\xb1\x9b\x9c\x8e\xc9\x05_Y0\x96K3E\x8c\x9a\r\x1a/\xe5\xe3]\b\xd6\x1e\xbd\x9a\xf4ک\x1f\r\xd31p\fv\xb5 S\xc6\xd3(k\x04\x1e4H\xc3\xccݎ?\xdbm\xe9f\xf2\x9f\x1d\xc1\xda\xe8\xc6լy\xfb\xab\xfe\xbcH\x96\x98\xe9\xfdE(}\xed\x98O\xfb\xaeZ3\xdb`\x88\xa8r\xe5T'\x8b6g4\xea\xbb\xd25\x83\t\x87\x9c\n\xccӵ)\xae\v\x16\xecf\xfe4\a4&\xa1\xa4E\x97\x8fIAk&7\xfa[\x9d^l\x1b5\x86\xfb~,kvʺYl\xda\x10a1N\xa8-:\xbb\xf3\x1e\xc4\x14\xfc֥[\x86\xd3%\x94\xdb\x1a\x94\x19Hi\xfbwMó\x8f\x15\xc8%K \x98\b\xa3y\\!\x85\x16\x89\b\xbeO\xbf\x9dXဠ\xd7ݺw\xdfE\xd2\xd2\x0fo&g\xe4\xf6\xf5\x04\xaf\xb4\xbey};\xe9\x93]K\xc8\xd1\xed\xeb\xc9\xd13!3\xc6\xd53j\xabFAo\xfa\xad\v1i\x9e\xe7\xc2\xff5\x1f\x9a1\x12F9-Fw\xb0\nP\x1ccq\x13\x81\x99\xcd\xe9\xdaE\xe7\xf4Єd\t4e\x9fI\x8d\x9cc\"\xf5\x9c\xba\x8b\xe5r\xb1\f\xf2\xa3\xa0\x19\xe5a\x03O\v\xc1\x8c=\xe2Z:7+\xe8\x02\x80\xee\xbcs~\xa8\xa0\x1b*\xe8\xaa1T\xd0\r\x15tC\x05\xddPAw\xe0\x18*\xe8\x86\n\xba\xc3\x17:T\xd0\r\x15tC\x05ݎ1T\xd0\xed\x9d\xcfPA\xb7k\f\x15t\x8d1Tе\xc7PA\x17\xf8\xf2PA7\xe4\x85\xee\x19C\x05\xdd\xe7\x9c\x17:T\xd0\xed\x1a\x9f{\xd6\xecPA\xf7\x99x\xe9\xc9PA7T\xd05\xc6PA7T\xd0Uc\xa8\xa0\xdb:\x86\n:;\x86\n\xba-\xe3\x1f\xd7R\x1a*\xe8>/K\xe9s\xb7\x05\x86\n\xba\xa1\x82.\xe8\xad \n\xf3W\xf2\xc7Vl\x1d\xbf\x16yQj \x1f<\xa0\xea@\x85\xe5\xa7b\x86p\xa3h\xeb9\x9b\xa4'\x82\xcfؼ\x94X&un\xeff\x1f%va\xa3\nC\xa3jv\xe7O\x9d敱\x9c\x85\x14љQW\xa5M\xa2\x95\x9c(\xf9\xdaO\xba\xf6\x92\xad\x05\xd5\x1a$\u007fE\xfe\xe3\xe4o\xbf\xffut\xfa\xcd\xc9\xc9O/F\xff\xf6\xf3\xefO\xfe6\xc6\xff\xf8\xa7\xd3oN\u007f\xf5\u007f\xfc\xfe\xf4\xf4\xe4\xe4\xa7\xef\xde}{;\xb9\xfc\x99\x9d\xfe\xfa\x13/\xf3;\xfbׯ'?\xc1\xe5\xcf\a\x029=\xfd\xe6\xeb\xc0\x89>\xaa\xc4j\x1f\xc0\xef\x91V\xeah\x1e\xb2\xe6\x9c>\x18.\x1a\xba\xfd\xb9(\xb9\xb6i\xa1\xf6TW\xc4o#\x9f\xcfq\xe1\xffS\x9dD\x12/\x82]\fx8\x90{\xc7p \xc9\xf1\aG-\xebG\xd2*6\x8fx$\xbd\xa0\r=\x93W3R͑)\"r\xa6\x8d\x95>\x13\xb2Y\xe9\x1a\x9a\\\xcat\xcb\x14ul\t\xb3\xb7)\x16%G_7ߨ#\x12z\x01\xf2\x9e)trQ^\xfb\x14\x90a\x8cR\x981\x1e\x9c\x96\x81\xaaf\xb0\xc7\xf9sdU\x11/)HJ\xc9\xf4\xea\xb5\xe0\x1a\x1e\x02l\xf26\xd1\xdf80D\x146\xdb\xd5\xe78\xd9\x14\xf1\x10f[r\xac\xea\nސBd,Y\x9d\xfb\x05!\xe6\xe1A\x9f\a|\xfb\xb0/j\xaa\xee\xea\xfd\x87\x911\x19\xeam\xde\xf8\xfeS+\x8b(\x99'\x92-Y\x06s\xb8T\t͐&\xfb\x98\x8a\x17[`\x06\x9e,\x83\x02)2E\xee\x17`N.\xa1f\x8d\xe8\xb0H('s\x1a\x9c*\x94\x9b\x1d*\xfc\xc4\f\x99\x19.\xa0\x15)\xa8\x04\xae=\xf8P\x96\x88E\xd9S!2\x97\x13\x9f\xad깻\x02\x14.~\xe1p\xff\x8b\xf9v\xb0{>\xa3\xf3\xaa0F\x81\xde\xf0\xd6\xc4N{\xdb6\xd9t\xeb\x12\b\xcd\xee\xe9*t\xba\xf7\vX\x9f\x1fS\xaf\xc8\xcbS<\x9bT\x91ꋡ\x9c\xf6\x0f\xa7\x187|}1\xf9\xe5\xe6\xaf7\xbf\\\xbcywu\x1d\xc3\x16\xcdNAХp\t-\xe8\x94e,\\\t\xdb\xc8fj\x82B1\x94\xa6\xe7\xa9\x14\xa1\x89\xb1\x88eYr\xce\xf8\xbcQ_\xdc'W\xb9\xd9\xf6\x02\xc9l֞\xec\\R\x1e\x9e\xb58]\xad\x11\x83,\xb9f\xf9\xb3\x15\xe6дoQ\xceE\x9aB\xdaBE0\xbc\xc7ɾ|\xed\xa7\xb0\xaa;nD\xc0$d\xf2\xfe\xe6\xea\xff\xadQ⪈O\x16{\xe6:\x06B́鹫\x1fl\x85ᰯ\x9d\xe3K\xaaO\xa9\xe4y\x9fx\xfa\x87\x92\xb7\xbbn\x15\xb1R*\x17)\x8c\xc9ĊdPmX\xf1\xad \xa8\x04b\x00r\xcdh\x96\xad\x88\xb1ޖ4\x03\x9b\xc0\x8f\xb5s\xc1\nVw6Ռf*\x90=\xc7\xcaU\xa3\xb8\xbc3&j\x8f\x9d\xab`\x90\x14\xb8\xd0\xce^\x8e\xa0{1CX\xc4\xda̍\xa4\xb5\x96\xfc\x8aP\x0ek\xb1ʔ\xc7\xf4\xa4\x9a5FD\x02a\x96\nT\xb7X\xad\xac\xe8\x88\x1c\x10\t4\xc5\xdaނ\xea\x85ͪȩ\xba\x83\xd4\xfe\x10\xa5\x15;/\x83\x9dm\xb5\xe8\xdbU\x01d\x06T\x97\xc1\xa1\x19Ԇm\x8e\np:\xcdB\x1d\x18\xd1\xed\x13h\xfa\x9eg\xab\x0fB\xe8\xb7U)j\x0f\xb2\xfd\xd1\xd94\xedȅQpC\x19\x03\xcem\x84\x1b\x87l\xa0Q)\xeb\xa9-\xd4\x19\xa3\x9e\x93\tȒ_\xa8o\xa5(\x03E\xfa\x86j\xfd\xed\xd5\x1b䅥\xb5?\x80k\xb9\xc26\x00ጠ۾\"?\x98s\xe7NZ\xa8\xca\xe2Y\xc0\x8c\x94\\\x81\x1e\x93wtEh\xa6\x847낭\xd9\tf\xf95\xfd/ct\xcfY`d*t(_Y\x03\x87,`\xf3+\xa1\xbe=\x83L\x1b\x90\xad|qf~kPC\x81\xd2;P\xa4\x90\x90@\n<\t\xa4\xd5Fl\xf5O\xff\xfc,i[H\xe5ׂ\x1b\x06҃ίx\xca\x12j\xa5\x1c\xd5m:\rUTJ\xa5\xbdMN\xb1\"\x1a\xd9G\xa9@b\v/-K\x88\xd9\xea\xef\xca)d\xa0\xad\xcb\x02\xbbwQm[\x0f\xb0\x9c\x06\xdf\xeeNu%ڴ \xc0U)\xc19\x855I\x05\xc4䗹E\xffp\xf5\x86\xbc 'fէH\xea3\xca2,\xf9\xd34\xf8\xa2\xf45\x8f\xc7\xccO\x0fQ\x89'\x9e\x04wqB&|F\xb8 \xaaL\x16\x1e\x97L\xf0\xca\x1d\xe4rk#\"k\x1b\xccg\x1b;\tu\xb7\xd7\xcc\xe7\x1f\x87\x9d\xf4\x12}?(\x90=%\xdf\x0fO.\xf9\xe2\xddJ\x86\x9f\xb4w\n\xd9\x00\xc9AӔj\x1av\x1d>B\xe4\x8d~1\x03!\xaf\x01\xfd\xc2䢂\xef\x19/\x1flrk_\xe7\xea\xcd%\x02#.xb\xed\x84P\x81S\x14\x19\xb3-\xf2\xd6:A[F^\x85\x13{\t\b/Ӑ\x91\xd3,\x13F\xa8\x87k\xfe\x94\xa7\"\xdfX\xb61\xe6\xa0\xd5G|\x8c\x1c?\x14\xfep\xacj\xa0\xbd\x8eU\xbc\xfb:\x83%\x04\xb7?\\\xef\x8bn`\x18\xa3\xce\xd3\t\x02\x8d\xf0\nft\n\x99U\xbe\xec)Q\x9b\xa7$\xd2[\x18\xe5j\x94\"\xeb[\xa2\xf8Ad\x98'J+\xe4\x18\xa0\xbf\x01\xdc\xe0\xab\xfdp\x83^\x9a\x16n\"\xbdɟ\x1bn\xca`\x8d\x8b\xac\xe3\xc6(mm\xdc\x18\xa0_\xe7\xc3|/&\x9c'\\\xf3?_@I\x11\xd1\xc0S\xc6\xe7\xe8ݏi\x06)!\xa3Xx\xea\x18֝u\xfc\x1c\xe3\xa9s\x13\x8f\x00\xeb\x0f\xa9\xdf.ԍ\x99\xe0n\xf6\x82\a\xa7\x00\x12۽\x19U\xcf\x19\xf28ú\x8f\xbe\xf7\xe4\x15܃\xf9SK\xe6#\xbb\x86\xca\x17\x18\x13C2\xe3\x8e\xf1\xd4U\xf4\xb4P\xee\xdcK1\xa7\xccZOc\xf2\x11\xdb\xf6{6F%\xbc\"\u007f\xe3\xa4By\x04\xe8ў#\x1cs.ܑ\xda8\xc2\x1f\xacy\x16\x17>\xb1\x93\xdd\x19\x9e\x8b\x80藾>\xd5\x1f8\x9e\xb6\xf0\xc4U;M.: \xfb]\xdf\xe2:\xe8r\x06\x84\x06C7\\\a\x9f\xcc\x190\xcf\x15}-\xcd75\xa3\xd9M\x11\xda͛\xac\xd3\xe2\xb7\xefn.\xda\x00\xe3Z7\xdf\xe3\xcd^\x06\xd7\x06\"\xa1iΔ\xc28\x05L\x17B\xdcE\x80<\xf1)\xd4s\xa6\x17\xe5t\x9c\x88\xbc\x91M=Rl\xae\xceݙ\x1c\x19\xbc\x9cF|\x83\xf1\x8c\xf1F&\x05\xde`\xe5|\xe0f!\x11 \x93\n\x9bHp\xeer\x10\x97\x04\xb9\x89\xee\xeb\xb8\"~\xec\x85\xf7\xacJ\xcb&\xe9]G\xb5<\xdcC~\x91\xf8p\xfd\xd2\x1b5\xf1\x96\x10\xeb݈\xd1C\xcd\xfeY\xe3\xf3y\xbdZ>(\xf4\b\x18\xc6\xe0\x90\x03e8\x99\x13'>\xf1&\x134%\x86\x87\x06@\x15\x92\xdcS\x99\x13J\x12\x9a,\xe0̖[\xa5\xa5\xc4\xf6\xfa\x1ah\xba\x1a)\x1d\x16\xf74\xaa\xb3\xf3\x06a~i\xf2\x8c\x8d\x1e\x86\xbb|\xb6\x8f\xe1.\x9f\xe1.\x9f\xc6\x18\xee\xf2\xa9\xc6p\x97\xcfp\x97Ͼ\x97\x87\xbb|\x86\xbb|v\x8f\xe1.\x9f\xe1.\x9f\xce1\xdc\xe5\xd3\x1c\xc3]>\xbb\xc6p\x97\xcf!c\xb8˧9\x86\xbb|\x9ac\xb8\xcb\xe7\x901\xdc\xe5\xf3\x0fܡz\xb8\xcb\xe7\xf3\xeaP=\xdc\xe5\xb3k|\xee\xfd\xbb\x87\xbb|\x86\xbb|\x86\xbb|\xf6\x8e\xe1.\x1f7\x86\xbb|\xbe$\"\x1a\xee\xf2\x19\xee\xf2\x19\xee\xf2\xf9G\xb5\x94\x86\xbb|>/K\xe9s\xb7\x05\x86\xbb|\x86\xbb|\x82\xde\n\xccmLY@\xd7\xefC\x9a\xd9\x05wo\xf7\x8d0\b%\xd3r6\x03\x89\xba!\xcel#\x8f$\x00\xaco9\xec\xb3\r}\xbe\x87\x02}\x86\xdd\xf3l\x91K\x88\xf6\xdf9%\xdf\xcd㞮\x14\x91\xa0\xc2\n\xbd\x18'\x97\xef\xdf\xd6\x06Ux\x17\xbe\x986D\xb8\x92\xf7<\x89\xcdg\xad\xb7\xbe\xa3\xdc-\x04\xa36\x81,Ʉ\xb2\xb9M\x16\xc5ɂr\x0e\x99\xb3?\x82\x92{\x16T\x91)\x00'\xa2\x00n\x8b0(Q\x8c\xcf3 Tk\x9a,\xc6f\xf6!*\xb2\xdbv\xd7\x1e\xbd\x9e\xa5\xd2\x12hn\xb7_B\x1e֘\xdeL\x8f\xd0D\n\xa5H^f\x9a\x15\xd5\x04\x89\x02\xac\xa3Q\xa1\xa9\xbc~S1A\nl\x1a\x8f,\xe1\xac^\x81EJ\xc84\x9b\rr\xd1B;\xc3{9\xf2B\xaf\xaaL_ 3&\x83\xaa;\x93\x8c\xa1!\x80\xeb\xb5\xcd\x0fp\x8egh\tjls\x8e\x18\r\x91%\x16\xa5\x96C\x8dQ$\xdd\x14?\x1b>c\xf7rc\x8a\x8d\xee\xf9uZs\x88\x86\xe4\x99\x1dvL\xf5\xcc\xe4\xacyI\x8b\xaf\xbd\b\xf22`:X\xcd4\xdd\xfa\x91\xf49,\xcdه\x04\xd82\xe4\xec\xd3-\x9c\xefI\x19\x9f\x06\x993\x8e\xb9\xc4\xef@):\x87IP\xd8j\x9bA\x87\x91\xab\x9aD\x82Tz,\xc3բ\xa1Y\xd5i\x93Ǫ9\xe5\x00\xa0\xb9]]\x95#\u007f/\x99ր$\x8b\xad\x8e1N\x1f\xa4\xd3oL\xac\xd9r\xf6\x9d\xff\x9c\xfdL\x88\x00T\xa8\xe7\xf0\xd4\xe6\xccO\x81L%\x83\x19\x991N3\x97Cx\x86\xad\x10Ch\xcb:C\x942ƾ\xe0>E\xcdceL~\f\xaeuײ\xe4\t&0\xba\xba>,!g32ǼFi\xf3\xdc\xff\xf9ſ\xfd)\x00\xe8tetR\f\x92k\xa1iVm[\x06|n(\xca\n\b\x9a\x85x\xee\xea\x82\xe0j\xf7\xf1r@\x8b\xe0\x97\u007f\xb8\x9bF\xa9\xeaZ\x90\xf3\x14\x96\xe7\rz\x1ceb\xdeu\xed\xe2\xe1jr\x84a\xddq\x84\xf1\x16\x9f\xc8C\xec{\xab\x92\x85\xb8\xb7M\xc4{\x9d\xb7:%\xbe\x10E\x99\xd9`\xc6۪\xbdBXO\x9b\x8d\x12\xd5N\xee\x15f\x9a\xfbi\xad\xc9\x1b\x97\xac\xeb\x97\x11\xb4v\xac]sN檥j)aL\xde\xd2,\x9b\xd2\xe4\xeeV|/\xe6\xea=\xbf\x942\xa8\x1f\xaaǙ-ѡJ\x93dQ\xf2;{\xb7\x99\x9fz&B|2\xa2\xd4E\xa9}\xd9O\x03\xa3\xd5\xdamә\x90\xa3a\xd5!\xa7\xba4f\x06\x0fx\xea\xee\x999ʜ\x80Y}\x8807|!\x13\xf3jΪy\x90\xff\xf0\xe2\x9f\xff\xd52\x90\x90\xd5K\xf2\xaf/\xb0\xb8@\x9dY\x81\x83\xd2\xdb(\x8c9\xcd2\x90\xb1\xac\xc1\x90x\x17+xRN\xa0c\x0f\xfd\x13\x98\xae\xb7\xb7\u007fE\xbb\x95i\x05\xd9\xec\xcc\u058b\xfaF\xf8\x01 \x8fQ\xb5:v\xb2\xd0\xe8\xef\xcfm\x1c.EV\xe6\xf0\x06\x96,\xfe\x8e\xdf\x16\f_\r\x931\xa5\x89\b1i\xa6\x99H\xeeH\xea\xc04r\f\xd7/\x18:\x1c#\xc1y\x94[\xd7ո\xac\x91\x92\x9c\x16E\xa8s\x18+\xf8$\xbdo-\x13\xb9\x05v\xb8\x89r\xe6\xc6F8\xec\xc7Ôa\xfff\x03?5\x18\xbf\xe9\x05\rnHO|=\xceFg\xe2\xaa\xfd\xb9\xfdN0\\\xaf\x0f\x99\xddB.\x1a\xea|\x8e\x0e\x04\xc4䗶0\xcb+\x1fzN\xb5\xb3\x13\xa2\"HHu\x05HŔQ,>\"E\xbf\xce(˝k+\x18bx\xc8)\xfa>\x8ep_\xfd\xa8A\x93A\xaf\x05\"\xb7G5zH\xb6\xa5e@x\x9fJ,o\x9e\x88ԁA\x96jo\xbe1\xc6`\xe0\xe6o)\xee\xeb\xa3\x04\xf4c\xce\x1fkܴy\xb3\xf9%\x8a9[\x88\x9f\x88%\xe3\xb4{sd\xe4\xc5n\x01\xfd\xbav4\xdd\x1b\x8e\x80\x1a\xe6\x8e\xf3*\x8cm\x8eG0pC1nj\xe4\xf8\xd5\xf1\xb3\xf1e\x8bd)\n:\x8f\xb8\x01u\r\xd7\xeb\xc0H\n\xd6\xc0\x88(i0\xe6(\xc2S\xbe\x05)B\x85\xb4j\xcd\x15\x01\xd2\x16b\xd5\xf2ԛ,\xb6\xef\xc3}p\xce7!T\x8a\x92\xa7֧^\x87Wޭ!\xe2Z\xf0\xf0\xe92U\xb5\xd64\x8a/\x83\fK\u007f\xa7\xa0\xa9\x91T/\xc7/_|9\xe2\x1bװ&\xbe\xa3\xfa\x1e5\xf8ҳ\xad\xde߃\xd5\v\x03\xef\x9c۱\xbe\xb8\x8a\xc5]7c\xe73\xba\x97LC\xe3v\xef\x134\x8d\x8c\x85\xdb\xe8\xf6s\x1a\x9e]\xd0\xf3V\xbc\xf8{A\bQ\xe5\xf4\xd1\xf9\xbde\xd4\xc1X@&\xd3\xe5\x91V\xb1\x10;DE\x13\xd5G\xe1m'O\xecL\x8e\x15v\x1e\b\xde\xea\xe8\xe3\xe0\xb6\xe9\xf2\xa1\bn(\xdeڪˇ\x82\xa2\u07fbh\xefY0\"\x9c0\u07beg\xb1\x10;\xf6\xecϰ\xa0\xcb\by\xa6X\xce2*\xb3\x95\xd9\xec\x1b\x8bA2-5\x01\xbedR\xf0<\xe6\xfe\xd3%\x95\x8cN3 \x12\xb0\xc3N\x02\x8a|}\xf2\xf1\xe2\x03f\x16\x9d\x1a\xc9\x19\f\x13\xfc\xae\x94\x8a\xf1\xf9\x06\xf57\xa6ۏ\xb7\x1c\x1dm\x10\xb0Nj\xa1\xacpI\xcc\xd3\n\xafFc\xc8K]\xdaKC\x1f\x92\xacTl\xf9\\\xf2\"\xceJ\xab\xb4\xdd߀\x91\xe6\x1a\xac\xbca\x01\xfca\xad\x8dLMp\x1b\xddZ\x02\xc3\xc1\xa8\x94\xd5]\xbe:S6\x828\x84\xbf\u0530\xd9\xd8\xd59\x93]/)\x9b~\x1e\xde\f|#\xb5\x06;\xf9=\xaf[9\x8cz\x03(0\x90\xf6B\xa8\xce\xe5\b\x1e2\xe5\xb6Rj\xdfs\x8d\xb5\xad\xbf.\xa7\x0f\x98\x80G\xf1@\x1e\xb421\xb3I\x11\x1f!\x03)\xbcи\xa7LW\x95\t\x8c3\xfd:\xec\x16d4Tl\xff\xb8C\xb6;`\xa3\x0f܉\x83\x1e۷M\xbb\xc9i\a\xf9\xec\xf9\xfa\xf6\xefn}\x91\xf1$+Sx\x9d\x95J\x83\xfc\x00J\x94\xb2\xc3ÿ\x16\x1d\xed|\xa7Qtp\xefB)\x89}d\xa4\x12Qt\x1czY\xbfZ\xe9\x14nB\xa9/,\xc4z\x15k8W\xdd\x17\x94\x16\x12:\x13\xa1x\x99ek\xe9\xef\xb2\xdc \x15\xf3\x94\xd1\x10:3\x83\xb7k\xea~j\xc6DS\x05=\x10M\x8d\xc7\xed51*c\t\xba\xb1\xb9\xff\a\xfb_f\xb6\xee\x13\x1b\xeb\xb2;g\xf3l0y\x11\xa3\x8bg\xd8\xeb\x9b\xd7\xf0m\xbd\x9c\xfd\xec\xfa\xa2\xb7\xb8\xd1v\x1c\x91\x03дIk\xfe\xf3A\xa4T?\xbd\x86\"O!\xfb1\xb4I\x1cM\x1cՔ枛\xd2\xe4\xae,>\a\x84\xe1\xb5?7\x90\xa1\x1c߉\xac\xef\x9bOZD\xe5\xa0\xe9\xf2\xe5\xb8\xfd/\xc6Fe\x99\xc6,\xd4\x0e\xd5\xe9\xdef\xd2\x19<\x19\x15\x82\xf1\x94-YZҬEe\r,\xd5\xc8\xc4\x12\x05\x96m\x1a\xe7\xd8$̽\xdd\xc2)\xf1\xe9PAgp\x97w\x14]%F\x19v\t\x91]L\xb4\xed\x80[{\xc1b\xce\xc5\x1dݭK\xca\xe3αf\xa3\xc9o)]\xbcu\r`\xfcS\xb8ދ\xeb7\xdd\n\xc8\x0e\xe7uk\x92\x17;&\xe2\xceD\xb5\xbd\vZ\xb9E\xb7IM̔Wg\x84\x92;X\xd9\x04J\xca]\xcbL\x0f\x02/mq\x02\xfe\x0el\xaa\x82}\xaf{\xe1\xfb]\xd6w\xb0\xc3\x1b\xd4Z\xae\xf9\x9e\x0f\x00\xe3\xba\xcd\x0fU \xafZ\xaa\xbb\ak\x97<\xde\x11\xad;@\xfa{\x8c\x1c8\xed\n\x81\x12\f9\xd9\xed7s7\xd6\x1aށ#\xb8Z\xb0\x02\x8bpv\xcc\x1a\xef\x00\x9eylW7\xa4X\xe0\x96\xa2\xae\xf8\x19\xb9\x16\xda\xfc\xcf\xe5\x03SZ\xedi\xfc\xfcF\x80\xba\x16\x1a\x9f\xed\x85\x12;\xa9\x03\x11b\x1fF\x02喷a)\t¯\x96w5s\xb7H\xd8\xf5\xedX\x04S\xe4\x8a\x1b&\xe3V^u\xa8V\x0e\xb8\xaf\x17\u208f\x90#y\xe8;\x80V\x9bƔG\xa5\x90-|m\xf9\xd0\x0e\x98S \xee\xf3\xe8õ\xef`zn\x91\xd1\x04R\xdfۖ\x1a\\P\rs\x96\x90\x1c\xe4\xce;\xaf\vç\xb6o\xdd\xde0\xd8A\xcan_\xd5\xf4\x0e\xba\xdf\x1b\xed\xde\xdeh\xc5\xd5\xf1{\x14p\x9d\xab\xa7\xa9\xef\xc89\xd9ß\xf6\xe0gSf؏:AK\vC\xd9\xffc\xd8)\x12\xca\xff\x92\x822\xa9\xc6\xe4\xc2U\x12t~\xb3\xf9\xbc\xd3<\x9a\xa0\rT\xa6\x88\xc1\xf9\x92f`۶QN\xc0\x16\xc5v\x82\x14\xb3\r\x89f\fm\xa1,\x17\xafB\"Gw\xb0::k\x9d\xbcm\tlGW\xfc\xa8ʲo\x9f\x03/gl{\xe0#\xfc\xb7\xa3\xf1\x86\x10\xec\x04\xbbS0\ue808\xad\xffTi\xba\xeflb\xcd\xfa>\x1fF\v;\xe8`\xa3\u007fM\xf3k-Bh\xaa\xa5-\x15~\xf3sT\xceAw)\xfbNW\xc50\xfb\x98\\\xf0\xd5\x06\xd4\xee2\xeb\xcaD\xaa(\xaah\xb5=\x17\xd2%r7\x01\xb9\xb4\x19Es\v\u007f}O\xb6\"\x1d\x05\x9d\\µHa\"\xe4f\x86A;Ұ\xfet\x87U\xd8X\xba\xc8R̙\xc6G7\x16\x8az\xa8\xd3A\x1fDŽsߝ|ܽ\x8a\x0f\xd5c\xbb\xa7O\xb1\x9f\xafۍ\xc9\xc7-+P\x9c\x16j!49Y2\xea\xaaND\x99\xbaN\xf0r#F\x11\xb96\x95, -3\xe8\xba,d\xa3\x81\x8f\u007f\xd0ka%g\u007f/\xdb\xf7\xa6xύ{z\x93\xb0k\x83\xac\xc8JI3\xf7g\"\xb8\xe5\b\xea\x15\xf9\xe9篈\xc3\xc0G?\x0f\xf3\xe3\xff\x0f\x00\x00\xff\xff9\xa7\u007f\xb0\x93\xba\x01\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec`\x86F\xaf\xa5>\xb3\x05&\xb4ڽ\xd1eq\x05\xcd\x0f~R\xc0\xc4\xef\xe2[\x98\xcf]\x99\xb4\xee\xa7N\xf7'i\x1d\xffTd\xa5\x11Yk=\xee\xb5Rݗ\x990M\xff\x19@aТ9\xe0/\xeaA\xe9G\xf5\xa3\xc4,\xb5W\xb0\x13\x99\xa5\x9fm\xa2\v\xbc\x82τI!\x12L\xcf\x00\x0e\"\x93)\xef\xd3\xe3\xa6\vT\x1f\xbel\xee\xbe'aR:\x1c\xd6\x11\xe1 \x95\xbb\x1d\x1a\x82T\xec\x85E[\x19\x89q\xf2L\xa9=53\xc5\xe2\x93\xfd4\xec%F1\rƶ\xc0\xe6t\x04&0\xc2ds\xcb\x02\xa4J\xe5A\xa6\xa5\xc8@*\xeb\x84J\xfc\xbeD\x8d\xdbо`\x9a\xf5'\x98{3Z\xe1O|\xe9x$\xad\x10\xb4\x81\x9c\xdc\xef\xe9P;\xba\x06\x8cn\u007f+Ȟyc\r\x86\xe2\xe9\xb0X\xcaή\xb1\x17\x17\x13\xc0k\xee\xf8\xa0-\x13[\xcc\xc0b\x86\x89\xd3f\x8c,\xf3L\xf7-\xce\x16\x8e\xd0s\xc0*6v\x9f\xb6\xdclp\x12(\x90\xc9\u007f\xdc\xcbd\xef\xe3+\x92)\x86\x04\xa9F˶@\x14Ev\x1c\xdf,\xccKBXh\xda\x1c4m\xd60\xf4a\x0e\x9b\x88\xa6E\xd9Ӧ\xcdX\xd6.\x9dk\x11y#s\xe54\x9e%Л\x93\xc9/-\xd0D`Ig\xab\xcd\x0e0/\xdc\xf1\x02\xa4\xabz\xe7aR\x18\xd4\xe0\xf0\x87`\xd4s\xf4aӟ\xfb\xc2\xfa\xf0\x02\\\xaaQ\xf8\x9ff\x12;\x9bo\xc1\xd7,`Ч\xf6\xbc\v\x90\xbb\x9aA\xe9\x05\xecd\xe6\x90\xe3\x9ei\x14[\xaeo\x96S/E\x968\xafI-\x17.\xd9\xdf\xd4G\xc8\xd9\xf1=\n\xf5\xa7\xfbX\xb9:It\x9d\xfc,d\xe0\f\x834\x98\xfb\xbc\xc5-\xeb@\xd3Ñڇ\xcf\x1f1\x9d&\x14\xc4J\xe4\xc9v>\xf4Pn/\x1f\x8e\x01\xf1\x9b\t\x01U}\xc2\xf2\xf9\xa8\v\x10\xf0\x80G\x1f\x05\t\x05\xc4(AK\x8d\x1e$N\x89ĉ16\x11\x0fxd@!\xd7\x151?^4|{\xc0c\xdc\xc0\x1e)\t\xb3\x90\t\xf04\xa5\x0e&\b\xa7L\x96\x90\x118s\xc9\x1a\x02N\xc7l\x12\x96\x98\x9b\xaaU\x9cx\xd6vk6vҸ\x0fx|g=\xc3H;\xf6\xb2\x88\xde0\x19`\xb0\xc8zTe2\xefD&\xd3z)\xaf\x0f\x1b5\x15uw\xdbg\xed6\xea\x02n\x9e\xa4%\xf4T\n\x1f5\xda\xcf\xdaqϫ\x11֣\xff,\xb2\xfa\xa9\xaczʛy\xa2G;A\x1a%\xf4\xbem\xfc\t\xb3f\x95\xb4\xb0QtV\nt\xe147Ì\x17KF)/-gB\x95V+v\xb4끵\xa2a\x06\xf6h\xd3\xe1N\x1b\xbdֲ\xd1P\xe9@\xe7Q\xbb%\xdf\xe3!\xf8\xf4}&\x12L!-\x99\xa8\"\x1a\xa2uF8\xbc\x97\t\xe4h\xee\x11\n\xf2\x05\xb1܈\xb6Ͼ-\x96\xb9\xd8Рj\xc1Ч1(\xadH\xaf\xa3\xc6U\xec\x8f\x18<\x98\x8f\x9e\x1e\x1c\xb37v\xd0\x1c\xc7DP[\xa4)'lE\xf6e\x91\x97Xĝ\xd3\xf8\xc1\xa3\xe7\xfdk.8Q\xfaor\x91,\xec\xff\x81BH\x13\xa5\xe5\x1f\xf8\x8a/\xc3\xce\xec\x90uk/DkH\v\xc4\xf1\x83\xc8\xfa\xb7\x1d#[\xd4d\x810\xf3\xa1\x80ޝD>\x17\xf0\xb8\xd7\xd6{\xe4\x9dđ\xe4p\xb7I\v\xe7\x0fx<\xbf8\xb1K\xe7\x1bu\xeeC\x84\xbe\xd6G\x80\xad#\x0e\xad\xb2#\x9c\xf3\xec\xf3_\x17NEKg\xe4@\xbe\x0f\x8e\x0f\xc4\xe9$[E\x134\xb5\xbe|\xa4\x10z\x1a\xfbH\xd9,\xb4u\v\x10\xfa\xa2\xad\xf3\x19\xd1N\xc0\xbb,\xdf\x06^\xaeB\x9e\r\xc4Ρ\x01봩\xae\xfa\xc8H\xf6\xd2\xc6\xc4E;w\xe0 \xc6\xd6\xd9;\x0f\x96\x0es\xe7\x8d~{\xfb{\xee\xef\x00\xe9\xef9\x88\t\a0\f\xb90:Ak\xe7\xc4&\xca\xc2\xcf$6뤦\xf0\x87%\xbe[\x9b\x15ֹdkՖ\x84\xc2D\xce\xc5lj\x9b\xa7V^\x96\xcc\a\xfd?/\xb2˱\x03\xd6\xfa<\x17*ʁ\x9d z\xed\xe7V*\x16@\xf9#\x8a\xb9/\xd9\\,\x89\\\x83\xf0\xfd~\x82\x81\\\xaa\r/\x02\xef_%|\xa8\x8d.>\xef\xf8p]\xcdnXPw\f_\x92\x8e\xb5B\xf3}\x85\xc1\x0e'O\xb3\xfa\x8b\xc2f\xa5];\xf5A\x90\v\x9d\xbe\xb3\xb0\x93ƺ\x06\xd9h\x98\xd2\xf2%\xe9\xeb\x9d\xe5ԍ1\xcf<\xca\xfd\xec綒q{\xfdX_\xe8\x8f_\xfc\x0e5\xbe\x1eC\x90;\x90\x0eP%\xbaT\x9c4\"c\xc0\x8bxv\xc4\v2\xc4\xfa\xbd\xa6\xa1*\xf3XB\xacX\x12\xa5\x9a\xc9/\xb5'\xfc(d\xf6Zlt2G]\xce8\xe6\xa6u\xeb\x11\xfc\xdcN\xa5F.\x9ed^\xe6 rbD4\xc9)ڐ9ve\x00\x1e\x85t\xec\x91\b2\xbb'\xa7\xa3A&:/2t\b[\xdci\xc3\xfane\x8a\xb5\xeb\x0fr\xa1\xc7\xef\x1d\xfbM\xc0NȬ4\xd1Vw!7\x96\x9d\x90\x82\xe1yكO,\n+&_T:::\xa4\x9d\xf3\x04\x85Y\x12\xd0~1\xf8\xd2\xe1ca$ɢ\x9e\x8b g r|ٍ \x83\x88\nu\x1c\v!g`2\x16o!\xe4[\b\x19\a\xf7-\x84|\v!\xe7\xdb[\b\xf9\x16B\xc6Lx\v!\xdfB\xc8\xe5(\xfc\xf6!\xe4\x8dW<\x85\x1b9t\xe2\xf0~\xdd\xfd\xc5\xe9P\xff\x04\x8f\xd2\xed\a7\xc5ߠ\xd0qQݷ\v\xa3+Y\f\xdfx\xf5\xa9\nڀ\x92\xd9pM\x03\x11\xbc\x9a\xdf!7\xfc\\\xf8C\xe9\xb3\xf4w\xee\x98\x14[#\xf5\xecʨn\xddӨ\x81_~\x81\xb7\xa4\xac<\xbe\xf6i\xaeTiI\xc5S\xbb\x9ai\x02dl\x9dS܉w\xb6\xa6\xe9\x19\x95L\xd1u\x95/r]\x19W\xab\xf4\x1a\x15J\v꒺\xf5F3p\x97U#E\x92)\xa6\xf2hq\xbdQ\xa8\xed\x99\xd9OD\x95\xd1h\xf5\xd0\f\xe8\x81:\xa6\xf9\x9a\xa19\xf2wPy\x91J\xa1g\xd4\a\xbdl\x1d\xf1KE\xddS\xd5>\x115>\x11q\xf9\x1c\xa6\x11U<\xcbjw\"h\xf8\xcc:\x9d\xba\ngt\xed\xa5\xd59\xddڛQ\xb0159#\x157\xa30'+qb\xeblF\xa1Ϻ\xef\x19ə\xfcy\xe8\x9b\xe7\xaa\xcdy\xc9췕\xb4_\xb3MmR43g\x83x\x84g\x90\xed\xe6\xf8z+\xf7>ި>\xfa\xa5Q\xed3ǰ8\xe8\xfaӂ\x04~\x92*\xf5RD\xaaЊ>\xf8\x85\x00.\u007f\xaaC!V\xc6a\x1e\x86H\xb3wֱX\b2\xcb)l\x8f>\xc3b\xd7p#\x92}w\xe0 Ƚ\xb0t~΅\x83\xf3\xfa\xd8xYͣ\x9e\xf35\xc0\x8f\xba>\xa5\xb7?\xfa\xb32/\xb2a\xebVZ\x84\xf3.\x98矂F\xe4\xc4*Qؽ\xae\xbe\xf0\x9e=\b}\xeb\x8e\x1f\xc8DT\xdfw'\x99.\xd3\x1a\xfe({\x85:\u0097;\x8e\xb5\xf8\xbbפ\xf9\"8DS\xd5ɦ\xff\xc1\xf0\xf0\xc7\xfa\vH2\x96\x99\xb0N\x1bq\x8f\x9ft\xd2z\xcdd\x8a&\xdd\xf1\x9d\xc7*\x82-\xacr\x8f\xa1Ho\xf8\x14\x17v\xd4\a\xd7\\9\x04\xddi\xd27\x84鰙\x9c\xd4X\xe7\xb2\xd9M\xdd\xde~\xf2\x1bq2\xc7\xf5\xc7\xd2g\x85V\x850\x16\x89\xb6\xd5\x06\xfd\xa4\xed\x98\x15\xdb\xebG\xc8t\xd8\xfd\x0f}\xfc\r\xf2\xe5\x06\xa7\x9f\x16\xef\xc2?\x16P\tdE\xaey\x11\xbe\x1b\x9e\xd7:\x88\xb6\x98\xc6\xfa<&\xbbc\x90\x84\xb5:\x91lM\x1e\xa5\v_5\xbd\xc6\xf7\xed\xe3\xf1Ѩҗ\x16\u007f~Th\xbeV\xeaf7\xca\xf3e\xf2\x15\x8b_N\xa6U\xcc\x1c2\x00d\xb9z\xc3O=\x94\n$\xb1\xfe}$oz\x99T\xd5k\x1e}bM\xea\xf5\x98N\x0fE\xb2\xab\xa1'4V\xf5{\x1eg3t\xb4N\xb8\xd2\xce?\x88\xc2\xc3 \x11\x85+Mp\u007fIi\xf8\xe3~\x02\x81\xfe\x1b\xf8\xe5O\xa2dº\b\x9e}\xaa\x875\xe7t\xeb\xfc}Kel\xe0QX~s\xc4߶\xb4\x88\u007fj\x16\x87\u007f\xf0\x9e\xef\nR\xe1pE\xb0\x973m@\xbe\xf9\xf1\x83\xc9\xdd}\xa1\x11\xf5\x9dn +O\xab\x9eL\x18\xd9\xc9Х\xdd\n>\xe3\xe3Iߍ\"\xc4\xfb\xd9t\u007f/\x87\xe9]\xfd\xdeU즚\x17\xb2\xf8\x1a\xf4D\xe7\xbbf\xaa7\xb8\x97\xab\x15Yւ\xe7\xaf<-\xfc\xbf<\xf5-\x9c\x80Ih'\u007f\xea\xfd6bx&\x8cΰ\xc1\x19P\x92^W\xf3&\xda\xfb\xe6?^z\x15\xde@;\xf8\xea\x01~t,m\xc9Jpơ\xa7\xd1<\x91$X\xb8p\x17\xd0~\f\xed\xfc\x9c\xff\xa9\xde:\xe3\u007f\x13\xad|\xa8k\xaf\xe0o\u007f?\x83\xe08\xef*<\xa8\xf3\xbf\x01\x00\x00\xff\xff*\xdd\xe8w?N\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4VM\x8f\xe44\x13\xbe\xe7W\x94\xf6=\xec\xe5MzG{A\xb9\xa1\x06\xa4\x110\x1aM/sA\x1c\x1c\xa7\xd2mƱCU\xb9\x87\x06\xf1ߑ\xed\xa4?\xd2\xe9\x9d\xdd\x03\xbe\xa5\\\x1f\x8f\x9f\xfaH\x15eY\x16j0\xcfHl\xbc\xabA\r\x06\xff\x14t\U0004bad7o\xb82~\xb5\xbfkP\xd4]\xf1b\\[\xc3:\xb0\xf8\xfe\t\xd9\a\xd2\xf8\x1dv\xc6\x191\xde\x15=\x8aj\x95\xa8\xba\x00P\xceyQQ\xcc\xf1\x13@{'\xe4\xadE*\xb7誗\xd0`\x13\x8cm\x91R\x84)\xfe\xfeC\xf5\xb1\xfaP\x00h\xc2d\xfe\xc9\xf4Ȣ\xfa\xa1\x06\x17\xac-\x00\x9c\xea\xb1\x06F\x8aF\xa2$0\xe1\x1f\x01Y\xb8ڣE\xf2\x95\xf1\x05\x0f\xa8c\xe0-\xf90\xd4p\xba\xc8\xf6#\xa8\xfc\xa0Mr\xb5I\xae\x9e\xb2\xabtk\rˏ\xb74~2\xa3\xd6`\x03)\xbb\f()\xf0Γ<\x9c\x82\x96\xc0L\xf9Ƹm\xb0\x8a\x16\x8d\v\x80\x810]\xfc\xe2^\x9c\u007fu?\x18\xb4-\xd7\xd0)\xcbX\x00\xb0\xf6\x03\u0590\\\x0fJc\x1be\xa1\xa113c\xb8촆\xbf\xff)\x00\xf6ʚ6\xf1\x9a/\xfd\x80\xee\xdb\xc7\xfb\xe7\x8f\x1b\xbd\xc3^e!@\x8b\xac\xc9\fIo\xe9\xf1`\x18\x14\x8c@A<(\xad\x91\x19t B'cL0\xae\xf3ԧp\xa3c\x00\xd5\xf8 ;\x84甓\xf1\xe9ը0\x90\x1f\x90\xc4L\xe8\x93ɩ>\x8f\xb2\x19\xc6\xf7\xf1\x11Y\a\xdaX\x91\xc8)\xc6XW\xd8\x02\xa7\a\x82\xef@v\x86\x810\x91\xeb\xe4\x12]\xe2\xa4\x03\xe5\xc07\xbf\xa3\x96j|=\xc7,\x06\xdb\xc62\xde#\t\x10j\xbfu毣g\x8e4ĐV\xc9T@\xd31N\x90\x9c\xb2\x91\xfe\x80\xff\a\xe5Z\xe8\xd5\x01\bc\f\b\xee\xcc[R\xe1\n~\xf6\x84\x89\xc0\x1av\"\x03\u05eb\xd5\xd6\xc8ԑ\xda\xf7}pF\x0e\xab\xd4W\xa6\t\xe2\x89W-\xeeѮ\xd8lKEzg\x04\xb5\x04\u0095\x1aL\x99\x80\xbbԐU\xdf\xfe\xefX$\xefϐ\xca!\xd6\x13\v\x19\xb7=\x8aS\x8f\xdc\xe4=\xf6G\xae\x86l\x96\xf1\x9f荢\xc8\xca\xd3\xf7\x9bO0\x05M)\xb8\xe4<\xb1}2\xe3\x13\xf1\x91(\xe3:\xa4\x9c\xb8\x8e|\x9f<\xa2k\ao\\\xae%m\r\xbaK\xd294\xbd\x11\x9e\xaa4槂u\x9aK\xd0 \x84\xa1U\x82m\x05\xf7\x0e֪G\xbbV\x8c\xff9\xed\x91a.#\xa5o\x13\u007f>N/\x153[G\xf14\xeb\x163\xb4н\x9b\x01u\xccY$.ښ\xce\xe8\xd4\x06\xd0y\x02\xb5dR\xbd\x89!O\x99\xafA1Έ\x8cc69b\x0f\xbe\x85ciT$\xf9N1^\x8afh\x1e\xa3\xc6<\xb25\x1dꃶ\x98\x1d\xe4I\x81o\x81\x88\a]\xe8\xe7\xf1Jx\xc0\xd7+\xd9#\xf98'Ӥ>?\x8b\xf9\x87\xfcs\xd9\x1aǟ\u007fM\xd6I\xbf\xab\xf3\x91{6jG7@\xc1\xb9ؑ\xdeE\xf1\xcc)\\N\xe4٭\x11\xec\xafp,\"\xb9w\x9dO\xbf{\x15C*\xc9}\x82cR\xc7\x18\x19ѕ\xbb[9\xcdg>\x8a\xbe\x80\xc0|\xd2\xca\xf0\xf5\x86qt\x18\u0085\x98e² \x8e\x91\xaeċ\x1d3\"\v֪\xc6b\rBan\x99\xed\x14\x91:\\V\xc5TF\xa7\xe5\xe8\xb3\x05r\xa5\x1ek\xffu\x87\xeeV\x85ë\xe2\xa5\xdcd7\xd0\x1cn\x19\xae\x8f[\u07bcIrY\xd6\x10\xa7n)报/ b!K\xb9T\x17\xb6\x83+\x126\xe7\x9aS\xef_\x14\xfc\xb4,̑\xdf\b\xbe\x90ԙ\xe8\xb4\xd4ޝ\xbeRa\x97\xe3\x12\x9b.\xc6W\xb4g/g\xf1\xa4\xb6\x13\x17\xa7\xd9\x1a\u05ecA\xb0}\x98\xaf\xb0\xef\xde]\xec\xa2\xe9S{ך\xbc\x81ï\xbf\x15\xd9+\xb6\xcf\x13\x8e(\xfc7\x00\x00\xff\xff\xad\x01\x9a\xeb\x00\f\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4V\xc1\x8e\xdc6\f\xbd\xfb+\x88\xf4\x90K\xed\xc9\"\x97·`\xdb\x02A\xd3`\x91M\xe6R\xf4\xa0\x91\xe8\x19veI\x15)\xa7ۯ/$\xcb;\xe3ٙ\xa4E\x11\xdfDS\xe4\xe3\xe3#\xa1\xa6m\xdbF\x05\xdabd\xf2\xae\a\x15\b\xff\x12t\xf9\xc4\xdd\xc3\x0fܑ\xdfL7;\x14u\xd3<\x903=\xdc&\x16?~@\xf6)j\xfc\x11\ar$\xe4]3\xa2(\xa3D\xf5\r\x80r\u038b\xcaf\xceG\x00\xed\x9dDo-\xc6v\x8f\xae{H;\xdc%\xb2\x06cɰ\xe4\x9f^u\xaf\xbbW\r\x80\x8eX\xae\u007f\xa4\x11Y\xd4\x18zp\xc9\xda\x06\xc0\xa9\x11{\x98\xbcM#\xb2S\x81\x0f^\xac\xd7s\xb2nB\x8b\xd1w\xe4\x1b\x0e\xa8s\xee}\xf4)\xf4p\xfc1\x87\xa8\xb8暶%\xda}\x8d\xf6\xaeF+\x0e\x96X~\xf9\x82\xd3;b)\x8e\xc1\xa6\xa8\xecUdŇ\xc9\xed\x93U\xf1\x9aW\x03\x10\"2\xc6\t?\xb9\a\xe7?\xbb\x9f\t\xad\xe1\x1e\x06e\x19\x1b\x00\xd6>`\x0f\xefs\x05Ai4\r\xc0\xa4,\x99r\u007f\xae\xc9\ato\xee\xden_\xdf\xeb\x03\x8ej6\x02\x18d\x1d)\x14\xbf+\xc5\x001(X\xd0\xc0\xe7\x03F\x84ma\x0eX|D\xae\xc0kH\x80\xa5\x02\xee\xaa)D\x1f0\n-\x04\xe7\xefDaO\xb63_\x9b\xf1\x1f\xe9ͦ\xccʇ\x9f\xee?\u0092\xb4\xb4`\xcdya\xfbx\x8d\x8f\xc4g\xa2\xc8\r\x18\xe7\xc6\rя%\":\x13<9)\am\tݚtN\xbb\x91$w\xfaτ,\xb9?\x1dܖ\xcd\x02;\x84\x14\x8c\x124\x1d\xbcup\xabF\xb4\xb7\x8a\xf1\x9bӞ\x19\xe66S\xfau\xe2O\x17\xe2\xdaqf\xeb8DuU]\xec\xd0\xe5I\xbd\x0f\xa8W\x83\x92c\xd0@ur\a\x1fA\xadجS|9Zw\xe2zi\x80a\xde\xe0\x03\xed\xd76\x00eL\xd9\xfe\xca\xde]\xb9w\x95\x9e\v\xb5ޖ\x1cY\x8e\xb9\x80\x10\xfdD\x06c\xbb\xd4V1\xa4X\x8b,\xbb\xb1k.\xe5:c\xb8\x16V\u009d\xc3[!\xb8\xabN\x19C\xa6u\xb94\xef\x1d\xac\xeb\xaf,C\xb5\xc7˹\x9fՙ\x15L\x11WS\xd8>\x85\xfe\xaa:DI\xe2\xff\xaa\x8fr\xa9z\xee\xaaFt\x8a\x11\x9dԈ\xe0\x87\x15|\xf5\xff5\x12\x0e\x8a\xf1\x8b\xfc^\x8e}\x97\xef-\x94[\x1aP?j\x8bs\xb8\xb2Ο)\xea_C\xcd\x1f\xba4\x9e\xa3j\xe1ͤȪ\x9d\xc5g\u007f>9u\xe5ߕ\x06_\xe8ۙ\xe9\xf8¹9\x9e\n{\xed\U000a2e59\x9f\byk\x9a\x1e$\xa69y\x95Z\xb5\x1cŠ\xb4\xc6 hޟ?f^\xbcX\xbdG\xcaQ{7\xcf)\xf7\xf0\xdb\xef\xcd\x1c\x15\xcdv\xc1\x91\x8d\xff\x04\x00\x00\xff\xffJ\xbeWz\r\n\x00\x00"), -} - -var CRDs = crds() - -func crds() []*apiextv1beta1.CustomResourceDefinition { - apiextinstall.Install(scheme.Scheme) - decode := scheme.Codecs.UniversalDeserializer().Decode - var objs []*apiextv1beta1.CustomResourceDefinition - for _, crd := range rawCRDs { - gzr, err := gzip.NewReader(bytes.NewReader(crd)) - if err != nil { - panic(err) - } - bytes, err := ioutil.ReadAll(gzr) - if err != nil { - panic(err) - } - gzr.Close() - - obj, _, err := decode(bytes, nil, nil) - if err != nil { - panic(err) - } - objs = append(objs, obj.(*apiextv1beta1.CustomResourceDefinition)) - } - return objs -} diff --git a/config/crd/v1beta1/crds/doc.go b/config/crd/v1beta1/crds/doc.go deleted file mode 100644 index 16e695f91..000000000 --- a/config/crd/v1beta1/crds/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Package crds embeds the controller-tools generated CRD manifests -package crds - -//go:generate go run ../../../../hack/crd-gen/v1beta1/main.go diff --git a/design/2082-bsl-delete-associated-resources_design.md b/design/2082-bsl-delete-associated-resources_design.md new file mode 100644 index 000000000..da5ac0bf0 --- /dev/null +++ b/design/2082-bsl-delete-associated-resources_design.md @@ -0,0 +1,40 @@ +# Delete Backup and Restic Repo Resources when BSL is Deleted + +## Abstract + +Issue #2082 requested that with the command `velero backup-location delete ` (implemented in Velero 1.6 with #3073), the following will be deleted: + +- associated Velero backups (to be clear, these are custom Kubernetes resources called "backups" that are stored in the API server) +- associated Restic repositories (custom Kubernetes resources called "resticrepositories") + +This design doc explains how the request will be implemented. + +## Background + +When a BSL resource is deleted from its Velero namespace, the associated custom Kubernetes resources, backups and Restic repositories, can no longer be used. +It makes sense to clean those resources up when a BSL is deleted. + +## Goals + +Update the `velero backup-location delete ` command to delete associated backup and Restic repository resources in the same Velero namespace. + +## Non Goals + +[It was suggested](https://github.com/vmware-tanzu/velero/issues/2082#issuecomment-827951311) to fix bug #2697 alongside this issue. +However, I think that should be fixed separately because although it is similar (restore objects are not being deleted), it is also quite different. +One is adding a command feature update (this issue) and the other is a bug fix and each affect different parts of the code base. + +## High-Level Design + +Update the `velero backup-location delete ` command to do the following: + +- find in the same Velero namespace from which the BSL was deleted the associated backup resources and Restic repositories, called "backups.velero.io" and "resticrepositories.velero.io" respectively +- delete the resources found + +The above logic will be added to [where BSLs are deleted](https://github.com/vmware-tanzu/velero/blob/main/pkg/cmd/cli/backuplocation/delete.go). + +## Alternative Considered + +I had considered deleting the backup files (the ones in json format and tarballs) in the BSL itself. +However, a standard use case is to back up a cluster and then restore into a new cluster. +Deleting the backup storage location in either location is not expected to remove all of the backups in the backup storage location and should not be done. diff --git a/design/restore-with-EnableAPIGroupVersions-feature.md b/design/Implemented/restore-with-EnableAPIGroupVersions-feature.md similarity index 99% rename from design/restore-with-EnableAPIGroupVersions-feature.md rename to design/Implemented/restore-with-EnableAPIGroupVersions-feature.md index eea929dfd..c5f9e1c15 100644 --- a/design/restore-with-EnableAPIGroupVersions-feature.md +++ b/design/Implemented/restore-with-EnableAPIGroupVersions-feature.md @@ -1,6 +1,6 @@ # Restore API Group Version by Priority Level When EnableAPIGroupVersions Feature is Set -Status: Draft +Status: Accepted ## Abstract diff --git a/design/Implemented/velero-debug.md b/design/Implemented/velero-debug.md new file mode 100644 index 000000000..2a6fe7128 --- /dev/null +++ b/design/Implemented/velero-debug.md @@ -0,0 +1,122 @@ +# `velero debug` command for gathering troubleshooting information + +## Abstract +To simplify the communication between velero users and developers, this document proposes the `velero debug` command to generate a tarball including the logs needed for debugging. + +Github issue: https://github.com/vmware-tanzu/velero/issues/675 + +## Background +Gathering information to troubleshoot a Velero deployment is currently spread across multiple commands, and is not very efficient. Logs for the Velero server itself are accessed via a kubectl logs command, while information on specific backups or restores are accessed via a Velero subcommand. Restic logs are even more complicated to retrieve, since one must gather logs for every instance of the daemonset, and there’s currently no good mechanism to locate which node a particular restic backup ran against. +A dedicated subcommand can lower this effort and reduce back-and-forth between user and developer for collecting the logs. + + +## Goals +- Enable efficient log collection for Velero and associated components, like plugins and restic. + +## Non Goals +- Collecting logs for components that do not belong to velero such as storage service. +- Automated log analysis. + +## High-Level Design +With the introduction of the new command `velero debug`, the command would download all of the following information: +- velero deployment logs +- restic DaemonSet logs +- plugin logs +- All the resources in the group `velero.io` that are created such as: + - Backup + - Restore + - BackupStorageLocation + - PodVolumeBackup + - PodVolumeRestore + - *etc ...* +- Log of the backup and restore, if specified in the param + +A project called `crash-diagnostics` (or `crashd`) (https://github.com/vmware-tanzu/crash-diagnostics) implements the Kubernetes API queries and provides Starlark scripting language to abstract details, and collect the information into a local copy. It can be used as a standalone CLI executing a Starlark script file. +With the capabilities of embedding files in Go 1.16, we can define a Starlark script gathering the necessary information, embed the script at build time, then the velero debug command will invoke `crashd`, passing in the script’s text contents. + +## Detailed Design +### Triggering the script +The Starlark script to be called by crashd: + +```python +def capture_backup_logs(cmd, namespace): + if args.backup: + log("Collecting log and information for backup: {}".format(args.backup)) + backupDescCmd = "{} --namespace={} backup describe {} --details".format(cmd, namespace, args.backup) + capture_local(cmd=backupDescCmd, file_name="backup_describe_{}.txt".format(args.backup)) + backupLogsCmd = "{} --namespace={} backup logs {}".format(cmd, namespace, args.backup) + capture_local(cmd=backupLogsCmd, file_name="backup_{}.log".format(args.backup)) +def capture_restore_logs(cmd, namespace): + if args.restore: + log("Collecting log and information for restore: {}".format(args.restore)) + restoreDescCmd = "{} --namespace={} restore describe {} --details".format(cmd, namespace, args.restore) + capture_local(cmd=restoreDescCmd, file_name="restore_describe_{}.txt".format(args.restore)) + restoreLogsCmd = "{} --namespace={} restore logs {}".format(cmd, namespace, args.restore) + capture_local(cmd=restoreLogsCmd, file_name="restore_{}.log".format(args.restore)) + +ns = args.namespace if args.namespace else "velero" +output = args.output if args.output else "bundle.tar.gz" +cmd = args.cmd if args.cmd else "velero" +# Working dir for writing during script execution +crshd = crashd_config(workdir="./velero-bundle") +set_defaults(kube_config(path=args.kubeconfig, cluster_context=args.kubecontext)) +log("Collecting velero resources in namespace: {}". format(ns)) +kube_capture(what="objects", namespaces=[ns], groups=['velero.io']) +capture_local(cmd="{} version -n {}".format(cmd, ns), file_name="version.txt") +log("Collecting velero deployment logs in namespace: {}". format(ns)) +kube_capture(what="logs", namespaces=[ns]) +capture_backup_logs(cmd, ns) +capture_restore_logs(cmd, ns) +archive(output_file=output, source_paths=[crshd.workdir]) +log("Generated debug information bundle: {}".format(output)) +``` +The sample command to trigger the script via crashd: +```shell +./crashd run ./velero.cshd --args +'backup=harbor-backup-2nd,namespace=velero,basedir=,restore=,kubeconfig=/home/.kube/minikube-250-224/config,output=' +``` +To trigger the script in `velero debug`, in the package `pkg/cmd/cli/debug` a struct `option` will be introduced +```go +type option struct { + // currCmd the velero command + currCmd string + // workdir for crashd will be $baseDir/velero-debug + baseDir string + // the namespace where velero server is installed + namespace string + // the absolute path for the log bundle to be generated + outputPath string + // the absolute path for the kubeconfig file that will be read by crashd for calling K8S API + kubeconfigPath string + // the kubecontext to be used for calling K8S API + kubeContext string + // optional, the name of the backup resource whose log will be packaged into the debug bundle + backup string + // optional, the name of the restore resource whose log will be packaged into the debug bundle + restore string + // optional, it controls whether to print the debug log messages when calling crashd + verbose bool +} +``` +The code will consolidate the input parameters and execution context of the `velero` CLI to form the option struct, which can be transformed into the `argsMap` that can be used when calling the func `exec.Execute` in `crashd`: +https://github.com/vmware-tanzu/crash-diagnostics/blob/v0.3.4/exec/executor.go#L17 + +## Alternatives Considered +The collection could be done via the kubernetes client-go API, but such integration is not necessarily trivial to implement, therefore, `crashd` is preferred approach + +## Security Considerations +- The starlark script will be embedded into the velero binary, and the byte slice will be passed to the `exec.Execute` func directly, so there’s little risk that the script will be modified before being executed. + +## Compatibility +As the `crashd` project evolves the behavior of the internal functions used in the Starlark script may change. We’ll ensure the correctness of the script via regular E2E tests. + + +## Implementation +1. Bump up to use Go v1.16 to compile velero +2. Embed the starlark script +3. Implement the `velero debug` sub-command to call the script +4. Add E2E test case + +## Open Questions +- **Command dependencies:** In the Starlark script, for collecting version info and backup logs, it calls the `velero backup logs` and `velero version`, which makes the call stack like velero debug -> crashd -> velero xxx. We need to make sure this works under different PATH settings. +- **Progress and error handling:** The log collection may take a relatively long time, log messages should be printed to indicate the progress when different items are being downloaded and packaged. Additionally, when an error happens, `crashd` may omit some errors, so before the script is executed we'll do some validation and make sure the `debug` command fail early if some parameters are incorrect. diff --git a/design/graph-manifest.md b/design/graph-manifest.md new file mode 100644 index 000000000..e469b2a61 --- /dev/null +++ b/design/graph-manifest.md @@ -0,0 +1,219 @@ +# Object Graph Manifest for Velero + +## Abstract + +One to two sentences that describes the goal of this proposal and the problem being solved by the proposed change. +The reader should be able to tell by the title, and the opening paragraph, if this document is relevant to them. + +Currently, Velero does not have a complete manifest of everything in the backup, aside from the backup tarball itself. +This change introduces a new data structure to be stored with a backup in object storage which will allow for more efficient operations in reporting of what a backup contains. +Additionally, this manifest should enable advancements in Velero's features and architecture, enabling dry-run support, concurrent backup and restore operations, and reliable restoration of complex applications. + +## Background + +Right now, Velero backs up items one at a time, sorted by API Group and namespace. +It also restores items one at a time, using the restoreResourcePriorities flag to indicate which order API Groups should have their objects restored first. +While this does work currently, it presents challenges for more complex applications that have their dependencies in the form of a graph rather than strictly linear. + +For example, Cluster API clusters are a set of complex Kubernetes objects that require that the "root" objects are restored first, before their "leaf" objects. +If a Cluster that a ClusterResourceSetBinding refers to does not exist, then a restore of the CAPI cluster will fail. + +Additionally, Velero does not have a reliable way to communicate what objects will be affected in a backup or restore operation without actually performing the operation. +This complicates dry-run tasks, because a user must simply perform the action without knowing what will be touched. +It also complicates allowing backups and restores to run in parallel, because there is currently no way to know if a single Kubernetes object is included in multiple backups or restores, which can lead to unreliability, deadlocking, and race conditions were Velero made to be more concurrent today. + +## Goals + +- Introduce a manifest data structure that defines the contents of a backup. +- Store the manifest data into object storage alongside existing backup data. + +## Non Goals + +This proposal seeks to enable, but not define, the following. + +- Implementing concurrency beyond what already exists in Velero. +- Implementing a dry-run feature. +- Implementing a new restore ordering procedure. + +While the data structure should take these scenarios into account, they will not be implemented alongside it. + +## High-Level Design + +To uniquely identify a Kubernetes object within a cluster or backup, the following fields are sufficient: + +- API Group and Version (example: backup.velero.io/v1) +- Namespace +- Name +- Labels + +This criteria covers the majority of Velero's inclusion or exclusion logic. +However, some additional fields enable further use cases. + +- Owners, which are other Kubernetes objects that have some relationship to this object. They may be strict or soft dependencies. +- Annotations, which provide extra metadata about the object that might be useful for other programs to consume. +- UUID generated by Kubernetes. This is useful in defining Owner relationships, providing a single, immutable key to find an object. This is _not_ considered at restore time, only internally for defining links. + +All of this information already exists within a Velero backup's tarball of resources, but extracting such data is inefficient. +The entire tarball must be downloaded and extracted, and then JSON within parsed to read labels, owners, annotations, and a UUID. +The rest of the information is encoded in the file system structure within the Velero backup tarball. +While doable, this is heavyweight in terms of time and potentially memory. + +Instead, this proposal suggests adding a new manifest structure that is kept alongside the backup tarball. +This structure would contain the above fields only, and could be used to perform inclusion/exclusion logic on a backup, select a resource from within a backup, and do set operations over backup or restore contents to identify overlapping resources. + +Here are some use cases that this data structure should enable, that have been difficult to implement prior to its existence: + +- A dry-run operation on backup, informing the user what would be selected if they were to perform the operation. + A manifest could be created and saved, allowing for a user to do a dry-run, then accept it to perform the backup. + Restore operations can be treated similarly. +- Efficient, non-overlapping parallelization of backup and restore operations. + By building or reading a manifest before performing a backup or restore, Velero can determine if there are overlapping resources. + If there are no overlaps, the operations can proceed in parallel. + If there are overlaps, the operations can proveed serially. +- Graph-based restores for non-linear dependencies. + Not all resources in a Kubernetes cluster can be defined in a strict, linear way. + They may have multiple owners, and writing BackupItemActions or RestoreItemActions to simply return a chain of owners is not an efficient way to support the many Kubernetes operators/controllers being written. + Instead, by having a manifest with enough information, Velero can build a discrete list that ensures dependencies are restored before their dependents, with less input from plugin authors. + +## Detailed Design + +The Manifest data structure would look like this, in Go type structure: + +```golang +// NamespacedItems maps a given namespace to all of its contained items. +type NamespacedItems map[string]*Item + +// APIGroupNamespaces maps an API group/version to a map of namespaces and their items. +type KindNamespaces map[string]NamespacedItems + +type Manifest struct { + // Kinds holds the top level map of all resources in a manifest. + Kinds KindNamespaces + + // Index is used to look up an individual item quickly based on UUID. + // This enables fetching owners out of the maps more efficiently at the cost of memory space. + Index map[string]*Item +} + + +// Item represents a Kubernetes resource within a backup based on it's selectable criteria. +// It is not the whole Kubernetes resource as retrieved from the API server, but rather a collection of important fields needed for filtering. +type Item struct { + // Kubernetes API group which this Item belongs to. + // Could be a core resource, or a CustomResourceDefinition. + APIGroup string + + // Version of the APIGroup that the Item belongs to. + APIVersion string + + // Kubernetes namespace which contains this item. + // Empty string for cluster-level resource. + Namespace string + + // Item's given name. + Name string + + // Map of labels that the Item had at backup time. + Labels map[string]string + + // Map of annotations that the Item had at Backup time. + // Useful for plugins that may decide to process only Items with specific annotations. + Annotations map[string]string + + // Owners is a list of UUIDs to other items that own or refer to this item. + Owners []string + + // Manifest is a pointer to the Manifest in which this object is contained. + // Useful for getting access to things like the Manifest.Index map. + Manifest *Manifest +} +``` + +In addition to the new types, the following Go interfaces would be provided for convenience. + +```golang +type Itermer interface { + // Returns the Item as a string, following the current Velero backup version 1.1.0 tarball structure format. + // ///.json + String() string + + // Owners returns a slice of realized Items that own or refer to the current Item. + // Useful for building out a full graph of Items to restore. + // Will use the UUIDs in Item.Owners to look up the owner Items in the Manifest. + Owners() []*Item + + // Kind returns the Kind of an object, which is a combination of the APIGroup and APIVersion. + // Useful for verifying the needed CustomResourceDefinition exists before actually restoring this Item. + Kind() *Item + + // Children returns a slice of all Items that refer to this item as an Owner. + Children() []*Items +} + +// This error type is being created in order to make reliable sentinel errors. +// See https://dave.cheney.net/2019/06/10/constant-time for more details. +type ManifestError string + +func (e ManifestError) Error() string { + return string(e) +} + +const ItemAlreadyExists = ManifestError("item already exists in manifest") + +type Manifester interface { + // Set returns the entire list of resources as a set of strings (using Itemer.String). + // This is useful for comparing two manifests and determining if they have any overlapping resources. + // In the future, when implementing concurrent operations, this can be used as a sanity check to ensure resources aren't being backed up or restored by two operations at once. + Set() sets.String + + // Adds an item to the appropriate APIGroup and Namespace within a Manifest + // Returns (true, nil) if the Item is successfully added to the Manifest, + // Returns (false, ItemAlreadyExists) if the Item is already in the Manifest. + Add(*Item) (bool, error) +} +``` + +### Serialization + +The entire `Manifest` should be serialized into the `manifest.json` file within the object storage for a single backup. +It is possible that this file could also be compressed for space efficiency. + +### Memory Concerns + +Because the `Manifest` is holding a minimal amount of data, memory sizes should not be a concern for most clusters. +TODO: Document known limits on API group name, resource name, and kind name character limits. + +## Security Considerations + +Introducing this manifest does not increase the attack surface of Velero, as this data is already present in the existing backups. +Storing the manifest.json file next to the existing backup data in the object storage does not change access patterns. + +## Compatibility + +The introduction of this file should trigger Velero backup version 1.2.0, but it will not interfere with Velero versions that do not support the `Manifest` as the file will be additive. +In time, this file will replace the `-resource-list.json.gz` file, but for compatibility the two will appear side by side. + +When first implemented, Velero should simply build the `Manifest` as it backs up items, and serialize it at the end. +Any logic changes that rely on the `Manifest` file must be introduced with their own design document, with their own compatibility concerns. + +## Implementation + +The `Manifest` object will _not_ be implemented as a Kubernetes CustomResourceDefinition, but rather one of Velero's own internal constructs. + +Implementation for the data structure alone should be minimal - the types will need to be defined in a `manifest` package. +Then, the backup process should create a `Manifest`, passing it to the various `*Backuppers` in the `backup` package. +These methods will insert individual `Items` into the `Manifest`. +Finally, logic should be added to the `persistence` package to ensure that the new `manifest.json` file is uploadable and allowed. + +## Alternatives Considered + +None so far. + +## Open Issues + +- When should compatibility with the `-resource-list.json.gz` file be dropped? +- What are some good test case Kubernetes resources and controllers to try this out with? +Cluster API seems like an obvious choice, but are there others? +- Since it is not implemented as a CustomResourceDefinition, how can a `Manifest` be retained so that users could issue a dry-run command, then perform their actual desire operation? +Could it be stored in Velero's temp directories? +Note that this is making Velero itself more stateful. diff --git a/design/new-prepost-backuprestore-plugin-hooks.md b/design/new-prepost-backuprestore-plugin-hooks.md new file mode 100644 index 000000000..65e305147 --- /dev/null +++ b/design/new-prepost-backuprestore-plugin-hooks.md @@ -0,0 +1,735 @@ +# Pre-Backup, Post-Backup, Pre-Restore, and Post-Restore Action Plugin Hooks + +## Abstract + +Velero should provide a way to trigger actions before and after each backup and restore. +**Important**: These proposed plugin hooks are fundamentally different from the existing plugin hooks, BackupItemAction and RestoreItemAction, which are triggered per resource item during backup and restore, respectively. +The proposed plugin hooks are to be executed only once: pre-backup (before backup starts), post-backup (after the backup is completed and uploaded to object storage, including volumes snapshots), pre-restore (before restore starts) and post-restore (after the restore is completed, including volumes are restored). + +### PreBackup and PostBackup Actions + +For the backup, the sequence of events of Velero backup are the following (these sequence depicted is prior upcoming changes for [upload progress #3533](https://github.com/vmware-tanzu/velero/issues/3533) ): + +``` +New Backup Request + |--> Validation of the request + |--> Set Backup Phase "In Progress" + | --> Start Backup + | --> Discover all Plugins + |--> Check if Backup Exists + |--> Backup all K8s Resource Items + |--> Perform all Volumes Snapshots + |--> Final Backup Phase is determined + |--> Persist Backup and Logs on Object Storage +``` +We propose the pre-backup and post-backup plugin hooks to be executed in this sequence: + +``` +New Backup Request + |--> Validation of the request + |--> Set Backup Phase "In Progress" + | --> Start Backup + | --> Discover all Plugins + |--> Check if Backup Exists + |--> **PreBackupActions** are executed, logging actions on existent backup log file + |--> Backup all K8s Resource Items + |--> Perform all Volumes Snapshots + |--> Final Backup Phase is determined + |--> Persist Backup and logs on Object Storage + |--> **PostBackupActions** are executed, logging to its own file +``` +These plugin hooks will be invoked: + +- PreBackupAction: plugin actions are executed after the backup object is created and validated but before the backup is being processed, more precisely _before_ function [c.backupper.Backup](https://github.com/vmware-tanzu/velero/blob/74476db9d791fa91bba0147eac8ec189820adb3d/pkg/controller/backup_controller.go#L590). If the PreBackupActions return an err, the backup object is not processed and the Backup phase will be set as `FailedPreBackupActions`. + +- PostBackupAction: plugin actions are executed after the backup is finished and persisted, more precisely _after_ function [c.runBackup](https://github.com/vmware-tanzu/velero/blob/74476db9d791fa91bba0147eac8ec189820adb3d/pkg/controller/backup_controller.go#L274). + +The proposed plugin hooks will execute actions that will have statuses on their own: +`Backup.Status.PreBackupActionsStatuses` and `Backup.Status.PostBackupActionsStatuses` which will be an array of a proposed struct `ActionStatus` with PluginName, StartTimestamp, CompletionTimestamp and Phase. + +### PreRestore and PostRestore Actions + +For the restore, the sequence of events of Velero restore are the following (these sequence depicted is prior upcoming changes for [upload progress #3533](https://github.com/vmware-tanzu/velero/issues/3533) ): +``` +New Restore Request + |--> Validation of the request + |--> Checks if restore is from a backup or a schedule + |--> Fetches backup + |--> Set Restore Phase "In Progress" + |--> Start Restore + |--> Discover all Plugins + |--> Download backup file to temp + |--> Fetch list of volumes snapshots + |--> Restore K8s items, including PVs + |--> Final Restore Phase is determined + |--> Persist Restore logs on Object Storage +``` +We propose the pre-restore and post-restore plugin hooks to be executed in this sequence: +``` +New Restore Request + |--> Validation of the request + |--> Checks if restore is from a backup or a schedule + |--> Fetches backup + |--> Set Restore Phase "In Progress" + |--> Start Restore + |--> Discover all Plugins + |--> Download backup file to temp + |--> Fetch list of volumes snapshots + |--> **PreRestoreActions** are executed, logging actions on existent backup log file + |--> Restore K8s items, including PVs + |--> Final Restore Phase is determined + |--> Persist Restore logs on Object Storage + |--> **PostRestoreActions** are executed, logging to its own file +``` + +These plugin hooks will be invoked: + +- PreRestoreAction: plugin actions are executed after the restore object is created and validated and before the backup object is fetched, more precisely in function `runValidatedRestore` _after_ function [info.backupStore.GetBackupVolumeSnapshots](https://github.com/vmware-tanzu/velero/blob/7c75cd6cf854064c9a454e53ba22cc5881d3f1f0/pkg/controller/restore_controller.go#L460). If the PreRestoreActions return an err, the restore object is not processed and the Restore phase will be set a `FailedPreRestoreActions`. + +- PostRestoreAction: plugin actions are executed after the restore finishes processing all items and volumes snapshots are restored and logs persisted, more precisely in function `processRestore` _after_ setting [`restore.Status.CompletionTimestamp`](https://github.com/vmware-tanzu/velero/blob/7c75cd6cf854064c9a454e53ba22cc5881d3f1f0/pkg/controller/restore_controller.go#L273). + +The proposed plugin hooks will execute actions that will have statuses on their own: +`Restore.Status.PreRestoreActionsStatuses` and `Restore.Status.PostRestoreActionsStatuses` which will be an array of a proposed struct `ActionStatus` with PluginName, StartTimestamp, CompletionTimestamp and Phase. + +## Background + +Increasingly, Velero is employed for workload migrations across different Kubernetes clusters. +Using Velero for migrations requires an atomic operation involving a Velero backup on a source cluster followed by a Velero restore on a destination cluster. + +It is common during these migrations to perform many actions inside and outside Kubernetes clusters. +**Attention**: these actions are not per resource item, but they are actions to be executed _once_ before and/or after the migration itself (remember, migration in this context is Velero Backup + Velero Restore). + +One important use case driving this proposal is migrating stateful workloads at scale across different clusters/storage backends. +Today, Velero's Restic integration is the response for such use cases, but there are some limitations: + +- Quiesce/unquiesce workloads: Pod hooks are useful for quiescing/unquiescing workloads, but platform engineers often do not have the luxury/visibility/time/knowledge to go through each pod in order to add specific commands to quiesce/unquiesce workloads. +- Orphan PVC/PV pairs: PVCs/PVs that do not have associated running pods are not backed up and consequently, are not migrated. + +Aiming to address these two limitations, and separate from this proposal, we would like to write a Velero plugin that takes advantage of the proposed Pre-Backup plugin hook. This plugin will be executed _once_ (not per resource item) prior backup. It will scale down the applications setting `.spec.replicas=0` to all deployments, statefulsets, daemonsets, replicasets, etc. and will start a small-footprint staging pod that will mount all PVC/PV pairs. Similarly, we would like to write another plugin that will utilize the proposed Post-Restore plugin hook. This plugin will unquiesce migrated applications by killing the staging pod and reinstating original `.spec.replicas` values after the Velero restore is completed. + +Other examples of plugins that can use the proposed plugin hooks are: + +- PostBackupAction: trigger a Velero Restore after a successful Velero backup (and complete the migration operation). +- PreRestoreAction: pre-expand the cluster's capacity via Cluster API to avoid starvation of cluster resources before the restore. +- PostRestoreAction: call actions to be performed outside Kubernetes clusters, such as configure a global load balancer (GLB) that enables the new cluster. + +The post backup actions will be executed after the backup is uploaded (persisted) on the disk. The logs of post-backup actions will be uploaded on the disk once the actions are completed. + +The post restore actions will be executed after the restore is uploaded (persisted) on the disk. The logs of post-restore actions will be uploaded on the disk once the actions are completed. + +This design seeks to provide missing extension points. This proposal's scope is to only add the new plugin hooks, not the plugins themselves. + +## Goals + +- Provide PreBackupAction, PostBackupAction, PreRestoreAction, and PostRestoreAction APIs for plugins to implement. +- Update Velero backup and restore creation logic to invoke registered PreBackupAction and PreRestoreAction plugins before processing the backup and restore respectively. +- Update Velero backup and restore complete logic to invoke registered PostBackupAction and PostRestoreAction plugins the objects are uploaded on disk. +- Create one `ActionStatus` struct to keep track of execution of the plugin hooks. This struct has PluginName, StartTimestamp, CompletionTimestamp and Phase. +- Add sub statuses for the plugins on Backup object: `Backup.Status.PreBackupActionsStatuses` and `Backup.Status.PostBackupActionsStatuses`. They will be flagged as optional and nullable. They will be populated only each plugin registered for the PreBackup and PostBackup hooks, respectively. +- Add sub statuses for the plugins on Restore object: `Backup.Status.PreRestoreActionsStatuses` and `Backup.Status.PostRestoreActionsStatuses`. They will be flagged as optional and nullable. They will be populated only each plugin registered for the PreRestore and PostRestore hooks, respectively. +- that will be populated optionally if Pre/Post Backup/Restore. + +## Non-Goals + +- Specific implementations of the PreBackupAction, PostBackupAction, PreRestoreAction and PostRestoreAction API beyond test cases. +- For migration specific actions (Velero Backup + Velero Restore), add disk synchronization during the validation of the Restore (making sure the newly created backup will show during restore) + +## High-Level Design + +The Velero backup controller package will be modified for `PreBackupAction` and `PostBackupAction`. + +The PreBackupAction plugin API will resemble the BackupItemAction plugin hook design, but with the fundamental difference that it will receive only as input the Velero `Backup` object created. +It will not receive any resource list items because the backup is not yet running at that stage. +In addition, the `PreBackupAction` interface will only have an `Execute()` method since the plugin will be executed once per Backup creation, not per item. + +The Velero backup controller will be modified so that if there are any PreBackupAction plugins registered, they will be + +The PostBackupAction plugin API will resemble the BackupItemAction plugin design, but with the fundamental difference that it will receive only as input the Velero `Backup` object without any resource list items. +By this stage, the backup has already been executed, with items backed up and volumes snapshots processed and persisted. +The `PostBackupAction` interface will only have an `Execute()` method since the plugin will be executed only once per Backup, not per item. + +If there are any PostBackupAction plugins registered, they will be executed after the backup is finished and persisted, more precisely _after_ function [c.runBackup](https://github.com/vmware-tanzu/velero/blob/74476db9d791fa91bba0147eac8ec189820adb3d/pkg/controller/backup_controller.go#L274). + +The Velero restore controller package will be modified for `PreRestoreAction` and `PostRestoreAction`. + +The PreRestoreAction plugin API will resemble the RestoreItemAction plugin design, but with the fundamental difference that it will receive only as input the Velero `Restore` object created. +It will not receive any resource list items because the restore has not yet been running at that stage. +In addition, the `PreRestoreAction` interface will only have an `Execute()` method since the plugin will be executed only once per Restore creation, not per item. + +The Velero restore controller will be modified so that if there are any PreRestoreAction plugins registered, they will be executed after the restore object is created and validated and before the backup object is fetched, more precisely in function `runValidatedRestore` _after_ function [info.backupStore.GetBackupVolumeSnapshots](https://github.com/vmware-tanzu/velero/blob/7c75cd6cf854064c9a454e53ba22cc5881d3f1f0/pkg/controller/restore_controller.go#L460). If the PreRestoreActions return an err, the restore object is not processed and the Restore phase will be set a `FailedPreRestoreActions`. + +The PostRestoreAction plugin API will resemble the RestoreItemAction plugin design, but with the fundamental difference that it will receive only as input the Velero `Restore` object without any resource list items. +At this stage, the restore has already been executed. +The `PostRestoreAction` interface will only have an `Execute()` method since the plugin will be executed only once per Restore, not per item. + +If any PostRestoreAction plugins are registered, they will be executed after the restore finishes processing all items and volumes snapshots are restored and logs persisted, more precisely in function `processRestore` _after_ setting [`restore.Status.CompletionTimestamp`](https://github.com/vmware-tanzu/velero/blob/7c75cd6cf854064c9a454e53ba22cc5881d3f1f0/pkg/controller/restore_controller.go#L273). + +## Detailed Design + +### New Status struct + +To keep the status of the plugins, we propose the following struct: + +```go +type ActionStatus struct { + // PluginName is the name of the registered plugin + // retrieved by the PluginManager as id.Name + // +optional + // +nullable + PluginName string `json:"pluginName,omitempty"` + + // StartTimestamp records the time the plugin started. + // +optional + // +nullable + StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"` + + // CompletionTimestamp records the time the plugin was completed. + // +optional + // +nullable + CompletionTimestamp *metav1.Time `json:"completionTimestamp,omitempty"` + + // Phase is the current state of the Action. + // +optional + // +nullable + Phase ActionPhase `json:"phase,omitempty"` +} + +// ActionPhase is a string representation of the lifecycle phase of an action being executed by a plugin +// of a Velero backup. +// +kubebuilder:validation:Enum=InProgress;Completed;Failed +type ActionPhase string + +const ( + // ActionPhaseInProgress means the action has being executed + ActionPhaseInProgress ActionPhase = "InProgress" + + // ActionPhaseCompleted means the action finished successfully + ActionPhaseCompleted ActionPhase = "Completed" + + // ActionPhaseFailed means the action failed + ActionPhaseFailed ActionPhase = "Failed" +) + +``` + +### Backup Status of the Plugins + +The `Backup` Status section will have the follow: + +```go +type BackupStatus struct { + (...) + // PreBackupActionsStatuses contains information about the pre backup plugins's execution. + // Note that this information is will be only populated if there are prebackup plugins actions + // registered + // +optional + // +nullable + PreBackupActionsStatuses *[]ActionStatus `json:"preBackupActionsStatuses,omitempty"` + + // PostBackupActionsStatuses contains information about the post backup plugins's execution. + // Note that this information is will be only populated if there are postbackup plugins actions + // registered + // +optional + // +nullable + PostBackupActionsStatuses *[]ActionStatus `json:"postBackupActionsStatuses,omitempty"` + +} +``` + +### Restore Status of the Plugins + +The `Restore` Status section will have the follow: + +```go +type RestoreStatus struct { + (...) + // PreRestoreActionsStatuses contains information about the pre Restore plugins's execution. + // Note that this information is will be only populated if there are preRestore plugins actions + // registered + // +optional + // +nullable + PreRestoreActionsStatuses *[]ActionStatus `json:"preRestoreActionsStatuses,omitempty"` + + // PostRestoreActionsStatuses contains information about the post restore plugins's execution. + // Note that this information is will be only populated if there are postrestore plugins actions + // registered + // +optional + // +nullable + PostRestoreActionsStatuses *[]ActionStatus `json:"postRestoreActionsStatuses,omitempty"` + +} +``` + +### New Backup and Restore Phases + +#### New Backup Phase: FailedPreBackupActions + +In case the PreBackupActionsStatuses has at least one `ActionPhase` = `Failed`, it means al least one of the plugins returned an error and consequently, the backup will not move forward. The final status of the Backup object will be set as `FailedPreBackupActions`: + +```go + +// BackupPhase is a string representation of the lifecycle phase +// of a Velero backup. +// +kubebuilder:validation:Enum=New;FailedValidation;FailedPreBackupActions;InProgress;Uploading;UploadingPartialFailure;Completed;PartiallyFailed;Failed;Deleting +type BackupPhase string + +const ( + + (...) + + // BackupPhaseFailedPreBackupActions means one or more the Pre Backup Actions has failed + // and therefore backup will not run. + BackupPhaseFailedPreBackupActions BackupPhase = "FailedPreBackupActions" + + (...) +) + +``` + +#### New Restore Phase FailedPreRestoreActions + +In case the PreRestoreActionsStatuses has at least one `ActionPhase` = `Failed`, it means al least one of the plugins returned an error and consequently, the restore will not move forward. The final status of the Restore object will be set as `FailedPreRestoreActions`: + +```go + +// RestorePhase is a string representation of the lifecycle phase +// of a Velero restore +// +kubebuilder:validation:Enum=New;FailedValidation;FailedPreRestoreActions;InProgress;Completed;PartiallyFailed;Failed +type RestorePhase string + +const ( + + (...) + + // RestorePhaseFailedPreRestoreActions means one or more the Pre Restore Actions has failed + // and therefore restore will not run. + RestorePhaseFailedPreRestoreActions BackupPhase = "FailedPreRestoreActions" + + (...) +) + +``` + +### New Interface types + +#### PreBackupAction + +The `PreBackupAction` interface is as follows: + +```go +// PreBackupAction provides a hook into the backup process before it begins. +type PreBackupAction interface { + // Execute the PreBackupAction plugin providing it access to the Backup that + // is being executed + Execute(backup *api.Backup) error +} +``` + +`PreBackupAction` will be defined in `pkg/plugin/velero/pre_backup_action.go`. + +#### PostBackupAction + +The `PostBackupAction` interface is as follows: + +```go +// PostBackupAction provides a hook into the backup process after it completes. +type PostBackupAction interface { + // Execute the PostBackupAction plugin providing it access to the Backup that + // has been completed + Execute(backup *api.Backup) error +} +``` + +`PostBackupAction` will be defined in `pkg/plugin/velero/post_backup_action.go`. + +#### PreRestoreAction + +The `PreRestoreAction` interface is as follows: + +```go +// PreRestoreAction provides a hook into the restore process before it begins. +type PreRestoreAction interface { + // Execute the PreRestoreAction plugin providing it access to the Restore that + // is being executed + Execute(restore *api.Restore) error +} +``` + +`PreRestoreAction` will be defined in `pkg/plugin/velero/pre_restore_action.go`. + +#### PostRestoreAction + +The `PostRestoreAction` interface is as follows: + +```go +// PostRestoreAction provides a hook into the restore process after it completes. +type PostRestoreAction interface { + // Execute the PostRestoreAction plugin providing it access to the Restore that + // has been completed + Execute(restore *api.Restore) error +} +``` + +`PostRestoreAction` will be defined in `pkg/plugin/velero/post_restore_action.go`. + +### New BackupStore Interface Methods + +For the persistence of the logs originated from the PostBackup and PostRestore plugins, create two additional methods on `BackupStore` interface: + +```go +type BackupStore interface { + (...) + PutPostBackuplog(backup string, log io.Reader) error + PutPostRestoreLog(backup, restore string, log io.Reader) error + (...) +``` + +The implementation of these new two methods will go hand-in-hand with the changes of uploading phases rebase. + + +### Generate Protobuf Definitions and Client/Servers + +In `pkg/plugin/proto`, add the following: + +1. Protobuf definitions will be necessary for PreBackupAction in `pkg/plugin/proto/PreBackupAction.proto`. + +```protobuf +message PreBackupActionExecuteRequest { + ... +} + +service PreBackupAction { + rpc Execute(PreBackupActionExecuteRequest) returns (Empty) +} +``` + +Once these are written, then a client and server implementation can be written in `pkg/plugin/framework/pre_backup_action_client.go` and `pkg/plugin/framework/pre_backup_action_server.go`, respectively. + +2. Protobuf definitions will be necessary for PostBackupAction in `pkg/plugin/proto/PostBackupAction.proto`. + +```protobuf +message PostBackupActionExecuteRequest { + ... +} + +service PostBackupAction { + rpc Execute(PostBackupActionExecuteRequest) returns (Empty) +} +``` + +Once these are written, then a client and server implementation can be written in `pkg/plugin/framework/post_backup_action_client.go` and `pkg/plugin/framework/post_backup_action_server.go`, respectively. + +3. Protobuf definitions will be necessary for PreRestoreAction in `pkg/plugin/proto/PreRestoreAction.proto`. + +```protobuf +message PreRestoreActionExecuteRequest { + ... +} + +service PreRestoreAction { + rpc Execute(PreRestoreActionExecuteRequest) returns (Empty) +} +``` + +Once these are written, then a client and server implementation can be written in `pkg/plugin/framework/pre_restore_action_client.go` and `pkg/plugin/framework/pre_restore_action_server.go`, respectively. + +4. Protobuf definitions will be necessary for PostRestoreAction in `pkg/plugin/proto/PostRestoreAction.proto`. + +```protobuf +message PostRestoreActionExecuteRequest { + ... +} + +service PostRestoreAction { + rpc Execute(PostRestoreActionExecuteRequest) returns (Empty) +} +``` + +Once these are written, then a client and server implementation can be written in `pkg/plugin/framework/post_restore_action_client.go` and `pkg/plugin/framework/post_restore_action_server.go`, respectively. + +### Restartable Delete Plugins + +Similar to the `RestoreItemAction` and `BackupItemAction` plugins, restartable processes will need to be implemented (with the difference that there is no `AppliedTo()` method). + +In `pkg/plugin/clientmgmt/`, add + +1. `restartable_pre_backup_action.go`, creating the following unexported type: + +```go +type restartablePreBackupAction struct { + key kindAndName + sharedPluginProcess RestartableProcess +} + +func newRestartablePreBackupAction(name string, sharedPluginProcess RestartableProcess) *restartablePreBackupAction { + // ... +} + +func (r *restartablePreBackupAction) getPreBackupAction() (velero.PreBackupAction, error) { + // ... +} + +func (r *restartablePreBackupAction) getDelegate() (velero.PreBackupAction, error) { + // ... +} + +// Execute restarts the plugin's process if needed, then delegates the call. +func (r *restartablePreBackupAction) Execute(input *velero.PreBackupActionInput) (error) { + // ... +} +``` + +2. `restartable_post_backup_action.go`, creating the following unexported type: + +```go +type restartablePostBackupAction struct { + key kindAndName + sharedPluginProcess RestartableProcess +} + +func newRestartablePostBackupAction(name string, sharedPluginProcess RestartableProcess) *restartablePostBackupAction { + // ... +} + +func (r *restartablePostBackupAction) getPostBackupAction() (velero.PostBackupAction, error) { + // ... +} + +func (r *restartablePostBackupAction) getDelegate() (velero.PostBackupAction, error) { + // ... +} + +// Execute restarts the plugin's process if needed, then delegates the call. +func (r *restartablePostBackupAction) Execute(input *velero.PostBackupActionInput) (error) { + // ... +} +``` + +3. `restartable_pre_restore_action.go`, creating the following unexported type: + +```go +type restartablePreRestoreAction struct { + key kindAndName + sharedPluginProcess RestartableProcess +} + +func newRestartablePreRestoreAction(name string, sharedPluginProcess RestartableProcess) *restartablePreRestoreAction { + // ... +} + +func (r *restartablePreRestoreAction) getPreRestoreAction() (velero.PreRestoreAction, error) { + // ... +} + +func (r *restartablePreRestoreAction) getDelegate() (velero.PreRestoreAction, error) { + // ... +} + +// Execute restarts the plugin's process if needed, then delegates the call. +func (r *restartablePreRestoreAction) Execute(input *velero.PreRestoreActionInput) (error) { + // ... +} +``` + +4. `restartable_post_restore_action.go`, creating the following unexported type: + +```go +type restartablePostRestoreAction struct { + key kindAndName + sharedPluginProcess RestartableProcess +} + +func newRestartablePostRestoreAction(name string, sharedPluginProcess RestartableProcess) *restartablePostRestoreAction { + // ... +} + +func (r *restartablePostRestoreAction) getPostRestoreAction() (velero.PostRestoreAction, error) { + // ... +} + +func (r *restartablePostRestoreAction) getDelegate() (velero.PostRestoreAction, error) { + // ... +} + +// Execute restarts the plugin's process if needed, then delegates the call. +func (r *restartablePostRestoreAction) Execute(input *velero.PostRestoreActionInput) (error) { + // ... +} +``` + +### Plugin Manager Changes + +Add the following methods to the `Manager` interface in `pkg/plugin/clientmgmt/manager.go`: + +```go +type Manager interface { + ... + // Get PreBackupAction returns a PreBackupAction plugin for name. + GetPreBackupAction(name string) (PreBackupAction, error) + + // Get PreBackupActions returns the all PreBackupAction plugins. + GetPreBackupActions() ([]PreBackupAction, error) + + // Get PostBackupAction returns a PostBackupAction plugin for name. + GetPostBackupAction(name string) (PostBackupAction, error) + + // GetPostBackupActions returns the all PostBackupAction plugins. + GetPostBackupActions() ([]PostBackupAction, error) + + // Get PreRestoreAction returns a PreRestoreAction plugin for name. + GetPreRestoreAction(name string) (PreRestoreAction, error) + + // Get PreRestoreActions returns the all PreRestoreAction plugins. + GetPreRestoreActions() ([]PreRestoreAction, error) + + // Get PostRestoreAction returns a PostRestoreAction plugin for name. + GetPostRestoreAction(name string) (PostRestoreAction, error) + + // GetPostRestoreActions returns the all PostRestoreAction plugins. + GetPostRestoreActions() ([]PostRestoreAction, error) + +} +``` + +`GetPreBackupAction` and `GetPreBackupActions` will invoke the `restartablePreBackupAction` implementations. +`GetPostBackupAction` and `GetPostBackupActions` will invoke the `restartablePostBackupAction` implementations. +`GetPreRestoreAction` and `GetPreRestoreActions` will invoke the `restartablePreRestoreAction` implementations. +`GetPostRestoreAction` and `GetPostRestoreActions` will invoke the `restartablePostRestoreAction` implementations. + +### How to invoke the Plugins + +#### Getting Pre/Post Backup Actions + +Getting Actions on `backup_controller.go` in `runBackup`: + +```go + + backupLog.Info("Getting PreBackup actions") + preBackupActions, err := pluginManager.GetPreBackupActions() + if err != nil { + return err + } + + backupLog.Info("Getting PostBackup actions") + postBackupActions, err := pluginManager.GetPostBackupActions() + if err != nil { + return err + } +``` + +#### Pre Backup Actions Plugins + +Calling the Pre Backup actions: + +```go + for _, preBackupAction := range preBackupActions { + err := preBackupAction.Execute(backup.Backup) + if err != nil { + backup.Backup.Status.Phase = velerov1api.BackupPhaseFailedPreBackupActions + return err + } + } +``` + +#### Post Backup Actions Plugins + +Calling the Post Backup actions: + +```go + for _, postBackupAction := range postBackupActions { + err := postBackupAction.Execute(backup.Backup) + if err != nil { + postBackupLog.Error(err) + } + } +``` + +#### Getting Pre/Post Restore Actions + +Getting Actions on `restore_controller.go` in `runValidatedRestore`: + +```go + + restoreLog.Info("Getting PreRestore actions") + preRestoreActions, err := pluginManager.GetPreRestoreActions() + if err != nil { + return errors.Wrap(err, "error getting pre-restore actions") + } + + restoreLog.Info("Getting PostRestore actions") + postRestoreActions, err := pluginManager.GetPostRestoreActions() + if err != nil { + return errors.Wrap(err, "error getting post-restore actions") + } +``` + +#### Pre Restore Actions Plugins + +Calling the Pre Restore actions: + +```go + for _, preRestoreAction := range preRestoreActions { + err := preRestoreAction.Execute(restoreReq.Restore) + if err != nil { + restoreReq.Restore.Status.Phase = velerov1api.RestorePhaseFailedPreRestoreActions + return errors.Wrap(err, "error executing pre-restore action") + } + } +``` + +#### Post Restore Actions Plugins + +Calling the Post Restore actions: + +```go + for _, postRestoreAction := range postRestoreActions { + err := postRestoreAction.Execute(restoreReq.Restore) + if err != nil { + postRestoreLog.Error(err.Error()) + } + } +``` + +### Giving the User the Option to Skip the Execution of the Plugins + +Velero plugins are loaded as init containers. If plugins are unloaded, they trigger a restart of the Velero controller. +Not mentioning if one plugin does get loaded for any reason (i.e., docker hub image pace limit), Velero does not start. +In other words, the constant load/unload of plugins can disrupt the Velero controller, and they cannot be the only method to run the actions from these plugins selectively. +As part of this proposal, we want to give the velero user the ability to skip the execution of the plugins via annotations on the Velero CR backup and restore objects. +If one of these exists, the given plugin, referenced below as `plugin-name`, will be skipped. + +Backup Object Annotations: + +``` + /prebackup=skip + /postbackup=skip +``` + +Restore Object Annotations: + +``` + /prerestore=skip + /postrestore=skip +``` + +## Alternatives Considered + +An alternative to these plugin hooks is to implement all the pre/post backup/restore logic _outside_ Velero. +In this case, one would need to write an external controller that works similar to what [Konveyor Crane](https://github.com/konveyor/mig-controller/blob/master/pkg/controller/migmigration/quiesce.go) does today when quiescing applications. +We find this a viable way, but we think that Velero users can benefit from Velero having greater embedded capabilities, which will allow users to write or load plugins extensions without relying on an external components. + +## Security Considerations + +The plugins will only be invoked if loaded per a user's discretion. +It is recommended to check security vulnerabilities before execution. + +## Compatibility + +In terms of backward compatibility, this design should stay compatible with most Velero installations that are upgrading. +If plugins are not present, then the backup/restore process should proceed the same way it worked before their inclusion. + +## Implementation + +The implementation dependencies are roughly in the order as they are described in the [Detailed Design](#detailed-design) section. + +## Open Issues diff --git a/design/velero-debug.md b/design/velero-debug.md deleted file mode 100644 index 6b2454c22..000000000 --- a/design/velero-debug.md +++ /dev/null @@ -1,120 +0,0 @@ -# `velero debug` command for gathering troubleshooting information - -## Abstract -To simplify the communication between velero users and developers, this document proposes the `velero debug` command to generate a tarball including the logs needed for debugging. - -Github issue: https://github.com/vmware-tanzu/velero/issues/675 - -## Background -Gathering information to troubleshoot a Velero deployment is currently spread across multiple commands, and is not very efficient. Logs for the Velero server itself are accessed via a kubectl logs command, while information on specific backups or restores are accessed via a Velero subcommand. Restic logs are even more complicated to retrieve, since one must gather logs for every instance of the daemonset, and there’s currently no good mechanism to locate which node a particular restic backup ran against. -A dedicated subcommand can lower this effort and reduce back-and-forth between user and developer for collecting the logs. - - -## Goals -- Enable efficient log collection for Velero and associated components, like plugins and restic. - -## Non Goals -- Collecting logs for components that do not belong to velero such as storage service. -- Automated log analysis. - -## High-Level Design -With the introduction of the new command `velero debug`, the command would download all of the following information: -- velero deployment logs -- restic DaemonSet logs -- Plugin logs - need clarification for vSphere plugin see open quetions -- Resource and log of the backup and restore, if specified in the param -- Resources: - - BackupStorageLocation - - PodVolumeBackups - - PodVolumeRestores - -A project called `crash-diagnostics` (or `crashd`) (https://github.com/vmware-tanzu/crash-diagnostics) implements the Kubernetes API queries and provides Starlark scripting language to abstract details, and collect the information into a local copy. It can be used as a standalone CLI executing a Starlark script file. -With the capabilities of embedding files in Go 1.16, we can define a Starlark script gathering the necessary information, embed the script at build time, then the velero debug command will invoke `crashd`, passing in the script’s text contents. - -## Detailed Design -### Triggering the script -The Starlark script to be called by crashd: - -```python -def capture_backup_logs(): - if args.backup: - kube_capture(what="objects", kinds=['backups'], names=[args.backup]) - backupLogsCmd = "velero backup logs {}".format(args.backup) - capture_local(cmd=backupLogsCmd) -def capture_restore_logs(): - if args.restore: - kube_capture(what="objects", kinds=['restores'], names=[args.restore]) - restoreLogsCmd = "velero restore logs {}".format(args.restore) - capture_local(cmd=restoreLogsCmd) - -ns = args.namespace if args.namespace else "velero" -basedir = args.basedir if args.basedir else os.home -output = args.output if args.output else "bundle.tar.gz" -# Working dir for writing during script execution -crshd = crashd_config(workdir="{0}/velero-bundle".format(basedir)) -set_defaults(kube_config(path=args.kubeconfig)) -capture_local(cmd="velero version -n {}".format(ns)) -capture_backup_logs() -capture_restore_logs() -kube_capture(what="logs", namespaces=[ns]) -kube_capture(what="objects", namespaces=[ns], kinds=['backupstoragelocations', 'podvolumebackups', 'podvolumerestores']) -archive(output_file=output, source_paths=[crshd.workdir]) -``` -The sample command to trigger the script via crashd: -```shell -./crashd run ./velero.cshd --args -'backup=harbor-backup-2nd,namespace=velero,basedir=,restore=,kubeconfig=/home/.kube/minikube-250-224/config,output=' -``` -To trigger the script in `velero debug`, in the package `pkg/cmd/cli/debug` a struct `option` will be introduced -```go -type option struct { - // workdir for crashd will be $baseDir/tmp/crashd - baseDir string - // the namespace where velero server is installed - namespace string - // the absolute path for the log bundle to be generated - outputPath string - // the absolute path for the kubeconfig file that will be read by crashd for calling K8S API - kubeconfigPath string - // optional, the name of the backup resource whose log will be packaged into the debug bundle - backup string - // optional, the name of the restore resource whose log will be packaged into the debug bundle - restore string -} -``` -The code will consolidate the input parameters and execution context of the `velero` CLI to form the option struct, which can be transformed into the `args` string for `crashd` -### kubeconfig -When it comes to accessing the API of k8s, `crashd` has a limitation that it can only accept a path of kubeconfig file, without customizing the `context`, and it does not honor the environment variables such as `KUBECONFIG`. `velero` does honor the environment variables and allow user to customize the path to kubeconfig and the `context` -There are two ways to make crashd have consistent behavior as velero in terms of getting the kube configuration: -1. Modify crashd to make it honor the environment variable and allow user to set context while calling k8s APIs. This is a preferred approach and it does make `crashd` better, but it may take longer time because we need to convince the maintainers of `crashd`, and double check the change will not break their current use cases. - There are 2 issues opened: - https://github.com/vmware-tanzu/crash-diagnostics/issues/208 - https://github.com/vmware-tanzu/crash-diagnostics/issues/122 - I'll try to contact the maintainers of `crashd` to see the feasibility for velero v1.7 -2. Before calling the `crashd` script velero CLI will use `client-go` to generate a temp `kubeconfig` file honoring the environment variable and global flags, and pass it to crashd. Although there’s no permission elevation and the temp file will be removed, there’s still some security concern because the temp file is accessible by other programs before it’s deleted, or it may not be deleted if an error happens. - -Therefore, we should consider `option 1` the better choice, and see `option 2` as the backup. - -## Alternatives Considered -The collection could be done via the kubernetes client-go API, but such integration is not necessarily trivial to implement, therefore, `crashd` is preferred approach - - -## Security Considerations -- The current released version of `crashd` depends on `client-go v0.19.0` which has a known CVE, we need to make sure that when it’s compiled into velero it uses the version that has the CVE fixed. We should write a PR or push crashd maintainer to fix the CVE-2021-3121 in 0.19.0 -- The starlark script will be embedded into the velero binary, so there’s little risk that the script will be modified before being called. -- There may be minor security issues if we choose to create a temp `kubeconfig` file for `crashd` and remove it afterwards. If we have to choose this option, we need to review it with security experts to better understand the risks. - -## Compatibility -As the `crashd` project evolves the behavior of the internal functions used in the Starlark script may change. We’ll ensure the correctness of the script via regular E2E tests. - - -## Implementation -1. Bump up to use Go v1.16 to compile velero -2. Embed the starlark script -3. Implement the `velero debug` sub-command to call the script -4. Add E2E test case - -## Open Questions -- **Log collection for vsphere plugin:** Per the design of vsphere plugin: https://github.com/vmware-tanzu/velero-plugin-for-vsphere#architecture when user backup resource on a guest cluster the code in component in the supervisor cluster may be called. Per discussion in v1.7 we will only support collecting logs of process running in one k8s cluster. In terms of implementation, we will do investigate the possibility to call extra script in crashd and ask vsphere plugin developer to provide a script to do the log collection, but the details remain TBD. -- **Command dependencies:** In the Starlark script, for collecting version info and backup logs, it calls the `velero backup logs` and `velero version`, which makes the call stack like velero debug -> crashd -> velero xxx. We need to make sure this works under different PATH settings. -- **Progress and error handling:** The log collection may take a relatively long time, log messages should be printed to indicate the progress when different items are being downloaded and packaged. Additionally, when an error happens, we need to double check if it’s omitted by crashd. diff --git a/go.mod b/go.mod index 7b5e196db..72a629712 100644 --- a/go.mod +++ b/go.mod @@ -1,49 +1,126 @@ module github.com/vmware-tanzu/velero -go 1.16 +go 1.17 require ( + cloud.google.com/go/storage v1.10.0 + github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-sdk-for-go v42.0.0+incompatible - github.com/Azure/go-autorest/autorest v0.11.1 - github.com/Azure/go-autorest/autorest/azure/auth v0.4.2 + github.com/Azure/azure-storage-blob-go v0.14.0 + github.com/Azure/go-autorest/autorest v0.11.21 + github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 github.com/Azure/go-autorest/autorest/to v0.3.0 - github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect github.com/aws/aws-sdk-go v1.28.2 - github.com/docker/spdystream v0.0.0-20170912183627-bc6354cbbc29 // indirect - github.com/evanphx/json-patch v4.9.0+incompatible - github.com/fatih/color v1.10.0 + github.com/bombsimon/logrusr v1.1.0 + github.com/evanphx/json-patch v4.11.0+incompatible + github.com/fatih/color v1.13.0 github.com/gobwas/glob v0.2.3 github.com/gofrs/uuid v3.2.0+incompatible - github.com/golang/protobuf v1.4.3 - github.com/google/uuid v1.1.2 - github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd + github.com/golang/protobuf v1.5.2 + github.com/google/uuid v1.2.0 + github.com/hashicorp/go-hclog v0.12.0 github.com/hashicorp/go-plugin v0.0.0-20190610192547-a1bc61569a26 github.com/joho/godotenv v1.3.0 github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0 - github.com/onsi/ginkgo v1.16.4 - github.com/onsi/gomega v1.10.2 + github.com/onsi/ginkgo v1.16.5 + github.com/onsi/gomega v1.16.0 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.7.1 + github.com/prometheus/client_golang v1.11.0 github.com/robfig/cron v1.1.0 - github.com/sirupsen/logrus v1.7.0 - github.com/spf13/afero v1.2.2 - github.com/spf13/cobra v1.1.1 + github.com/sirupsen/logrus v1.8.1 + github.com/spf13/afero v1.6.0 + github.com/spf13/cobra v1.2.1 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.6.1 - github.com/vmware-tanzu/crash-diagnostics v0.3.4 - golang.org/x/mod v0.3.0 - golang.org/x/net v0.0.0-20201110031124-69a78807bb2b - google.golang.org/grpc v1.31.0 - k8s.io/api v0.20.9 - k8s.io/apiextensions-apiserver v0.19.12 - k8s.io/apimachinery v0.20.9 - k8s.io/cli-runtime v0.20.9 - k8s.io/client-go v0.20.9 + github.com/stretchr/testify v1.7.0 + github.com/vmware-tanzu/crash-diagnostics v0.3.7 + golang.org/x/mod v0.4.2 + golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 + google.golang.org/api v0.56.0 + google.golang.org/grpc v1.40.0 + k8s.io/api v0.22.2 + k8s.io/apiextensions-apiserver v0.22.2 + k8s.io/apimachinery v0.22.2 + k8s.io/cli-runtime v0.22.2 + k8s.io/client-go v0.22.2 k8s.io/klog v1.0.0 k8s.io/kube-aggregator v0.19.12 - sigs.k8s.io/cluster-api v0.3.11-0.20210106212952-b6c1b5b3db3d - sigs.k8s.io/controller-runtime v0.7.1-0.20201215171748-096b2e07c091 - sigs.k8s.io/yaml v1.2.0 + sigs.k8s.io/cluster-api v1.0.0 + sigs.k8s.io/controller-runtime v0.10.2 + sigs.k8s.io/yaml v1.3.0 +) + +require ( + cloud.google.com/go v0.93.3 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.14 // indirect + github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver v3.5.1+incompatible // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dimchansky/utfbom v1.1.1 // indirect + github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect + github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/go-logr/logr v0.4.0 // indirect + github.com/go-logr/zapr v0.4.0 // indirect + github.com/gobuffalo/flect v0.2.3 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/go-cmp v0.5.6 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/googleapis/gax-go/v2 v2.1.0 // indirect + github.com/googleapis/gnostic v0.5.5 // indirect + github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect + github.com/imdario/mergo v0.3.12 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect + github.com/json-iterator/go v1.1.11 // indirect + github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect + github.com/mattn/go-colorable v0.1.9 // indirect + github.com/mattn/go-ieproxy v0.0.1 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/go-testing-interface v1.0.0 // indirect + github.com/moby/spdystream v0.2.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/nxadm/tail v1.4.8 // indirect + github.com/oklog/run v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.26.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect + github.com/stretchr/objx v0.2.0 // indirect + github.com/vladimirvivien/gexe v0.1.1 // indirect + go.opencensus.io v0.23.0 // indirect + go.starlark.net v0.0.0-20201006213952-227f4aabceb5 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + go.uber.org/zap v1.19.0 // indirect + golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect + golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect + golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 // indirect + golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect + golang.org/x/text v0.3.7 // indirect + golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect + gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71 // indirect + google.golang.org/protobuf v1.27.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + k8s.io/component-base v0.22.2 // indirect + k8s.io/klog/v2 v2.9.0 // indirect + k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e // indirect + k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect ) replace github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2 diff --git a/go.sum b/go.sum index c4c9fdffb..525808f44 100644 --- a/go.sum +++ b/go.sum @@ -13,8 +13,18 @@ cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bP cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0 h1:Dg9iHVQfrhq82rUNu9ZxUDrJLaxFUe/HlCVaLyRruq8= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3 h1:wPBktZFzYBcCZVARvwVKqH1uEj+aLXofJEtrb4oOsio= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -24,6 +34,7 @@ cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM7 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/firestore v1.6.0/go.mod h1:afJwI0vaXwAG54kI7A//lP/lSPDkQORQuMkv56TxEPU= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -32,29 +43,37 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= +github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= github.com/Azure/azure-sdk-for-go v42.0.0+incompatible h1:yz6sFf5bHZ+gEOQVuK5JhPqTTAmv+OvSLSaqgzqaCwY= github.com/Azure/azure-sdk-for-go v42.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= +github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0= github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= -github.com/Azure/go-autorest/autorest v0.11.1 h1:eVvIXUKiTgv++6YnWb42DUA1YL7qDugnKP0HljexdnQ= -github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest v0.11.21 h1:w77zY/9RnUAWcIQyDC0Fc89mCvwftR8F+zsR/OH6enk= +github.com/Azure/go-autorest/autorest v0.11.21/go.mod h1:Do/yuMSW/13ayUkcVREpsMHGG+MvV81uzSCFgYPj4tM= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= -github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= -github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= -github.com/Azure/go-autorest/autorest/adal v0.9.5 h1:Y3bBUV4rTuxenJJs41HU3qmqsb+auo+a3Lz+PlJPpL0= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/azure/auth v0.4.2 h1:iM6UAvjR97ZIeR93qTcwpKNMpV+/FTWjwEbuPD495Tk= -github.com/Azure/go-autorest/autorest/azure/auth v0.4.2/go.mod h1:90gmfKdlmKgfjUpnCEpOJzsUEjrWDSLwHIG73tSXddM= -github.com/Azure/go-autorest/autorest/azure/cli v0.3.1 h1:LXl088ZQlP0SBppGFsRZonW6hSvwgL5gRByMbvUbx8U= -github.com/Azure/go-autorest/autorest/azure/cli v0.3.1/go.mod h1:ZG5p860J94/0kI9mNJVoIoLgXcirM2gF5i2kWloofxw= +github.com/Azure/go-autorest/autorest/adal v0.9.11/go.mod h1:nBKAnTomx8gDtl+3ZCJv2v0KACFHWTB2drffI1B68Pk= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/adal v0.9.14 h1:G8hexQdV5D4khOXrWG2YuLCFKhWYmWD8bHYaXN5ophk= +github.com/Azure/go-autorest/autorest/adal v0.9.14/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 h1:TzPg6B6fTZ0G1zBf3T54aI7p3cAT6u//TOXGPmFMOXg= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.8/go.mod h1:kxyKZTSfKh8OVFWPAgOgQ/frrJgeYQJPyR5fLFmXko4= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 h1:dMOmEJfkLKW/7JsokJqkyoYSgmR08hi9KrhjZb+JALY= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM= github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= @@ -62,7 +81,6 @@ github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSY github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= -github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/to v0.3.0 h1:zebkZaadz7+wIQYgC7GXaz3Wb28yKYfVkkBKwc38VF8= @@ -70,66 +88,77 @@ github.com/Azure/go-autorest/autorest/to v0.3.0/go.mod h1:MgwOyqaIuKdG4TL/2ywSsI github.com/Azure/go-autorest/autorest/validation v0.2.0 h1:15vMO4y76dehZSq7pAaOLQxC6dZYsSrj2GQpflyM/L4= github.com/Azure/go-autorest/autorest/validation v0.2.0/go.mod h1:3EEqHnBxQGHXRYq3HT1WyXAvT7LLY3tl70hw6tQIbjI= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= -github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= +github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alessio/shellescape v1.2.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= -github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.28.2 h1:j5IXG9CdyLfcVfICqo1PXVv+rua+QQHbkXuvuU/JF+8= github.com/aws/aws-sdk-go v1.28.2/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bifurcation/mint v0.0.0-20180715133206-93c51c6ce115/go.mod h1:zVt7zX3K/aDCk9Tj+VM7YymsX66ERvzCJzw8rFCX2JU= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/caddyserver/caddy v1.0.3 h1:i9gRhBgvc5ifchwWtSe7pDpsdS9+Q0Rw9oYQmYUTw1w= -github.com/caddyserver/caddy v1.0.3/go.mod h1:G+ouvOY32gENkJC+jhgl62TyhvqEsFaDiZ4uw0RzP1E= -github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/bombsimon/logrusr v1.1.0 h1:Y03FI4Z/Shyrc9jF26vuaUbnPxC5NMJnTtJA/3Lihq8= +github.com/bombsimon/logrusr v1.1.0/go.mod h1:Jq0nHtvxabKE5EMwAAdgTaz7dfWE8C4i11NOltxGQpc= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= -github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/coredns/corefile-migration v1.0.11 h1:ptBYGW2ADXIB7ZEBPrhhTvNwJLQfxE3Q9IUMBhJCEeI= -github.com/coredns/corefile-migration v1.0.11/go.mod h1:RMy/mXdeDlYwzt0vdMEJvT2hGJ2I86/eO0UdXmH9XNI= +github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= +github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/coredns/caddy v1.1.0 h1:ezvsPrT/tA/7pYDBZxu0cT0VmWk75AfIaf6GSYCNMf0= +github.com/coredns/caddy v1.1.0/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4= +github.com/coredns/corefile-migration v1.0.13 h1:ld5RswmH1xjqBUEukw4QxC1PakLNNoVlsZEV8FGwoV8= +github.com/coredns/corefile-migration v1.0.13/go.mod h1:XnhgULOEouimnzgn0t4WPuFDN2/PJQcTxdWKC5eXNGE= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -138,27 +167,27 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docker/spdystream v0.0.0-20170912183627-bc6354cbbc29 h1:llBx5m8Gk0lrAaiLud2wktkX/e8haX7Ru0oVfQqtZQ4= -github.com/docker/spdystream v0.0.0-20170912183627-bc6354cbbc29/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/drone/envsubst v1.0.3-0.20200709223903-efdb65b94e5a/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g= +github.com/drone/envsubst/v2 v2.0.0-20210615175204-7bf45dbf5372/go.mod h1:esf2rsHFNlZlxsqsZDojNBcnNs5REqIvRrWRHqX0vEU= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= @@ -168,91 +197,71 @@ github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v0.0.0-20200808040245-162e5629780b/go.mod h1:NAJj0yf/KaRKURN6nyi7A9IZydMivZEm9oQLWNjfKDc= -github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.1.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs= +github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= -github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/go-acme/lego v2.5.0+incompatible/go.mod h1:yzMNe9CasVUhkquNvti5nAtPmG94USbYxYrZfTkIn0M= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.2.1/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.3.0 h1:q4c+kbcR0d5rSurhBR8dIgieOaYpXtsdTYfx22Cu6rs= -github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/zapr v0.2.0 h1:v6Ji8yBW77pva6NkJKQdHLAJKrIJKRHz0RXwPqCHSR4= -github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= -github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= -github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= -github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= -github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM= +github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= -github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= -github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= -github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= -github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= -github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= -github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= -github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= -github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= -github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= -github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= -github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/gobuffalo/flect v0.2.2 h1:PAVD7sp0KOdfswjAw9BpLCU9hXo7wFSzgpQ+zNeks/A= -github.com/gobuffalo/flect v0.2.2/go.mod h1:vmkQwuZYhN5Pc4ljYQZzP+1sq+NEkK+lh20jmEmX3jc= +github.com/gobuffalo/flect v0.2.3 h1:f/ZukRnSNA/DUpSNDadko7Qc0PhGvsew35p/2tu+CRY= +github.com/gobuffalo/flect v0.2.3/go.mod h1:vmkQwuZYhN5Pc4ljYQZzP+1sq+NEkK+lh20jmEmX3jc= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -262,8 +271,9 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -271,7 +281,8 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -285,13 +296,16 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= -github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -299,16 +313,24 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github/v33 v33.0.0/go.mod h1:GMdDnVZY/2TsWgp/lkYnpSAh6TrzhANBBwm6k6TTEXg= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -316,129 +338,157 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gax-go/v2 v2.1.0 h1:6DWmvNpomjL1+3liNSZbVns3zsYzzCjm6pRBO1tLeso= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/googleapis/gnostic v0.5.2 h1:YVUUPsHTL1setd3iy+OvO3cToQBzLmYat6N9iIh++Gc= -github.com/googleapis/gnostic v0.5.2/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= +github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd h1:rNuUHR+CvK1IS89MMtcF0EpcVMZtjKfPRp4MEmt/aTs= github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= +github.com/hashicorp/go-hclog v0.12.0 h1:d4QkX8FRTYaKaCZBoXYY8zJX2BXjWxurN/GA2tkrmZM= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-plugin v0.0.0-20190610192547-a1bc61569a26 h1:sADP8l/FAtMyWJ9GIcQT/04Ae80ZZ75ogOrtW0DIZhc= github.com/hashicorp/go-plugin v0.0.0-20190610192547-a1bc61569a26/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0 h1:ipLtV9ubLEYx42YvwDa12eVPQvjuGZoPdbCozGzVNRc= github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys= -github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= -github.com/lucas-clemente/aes12 v0.0.0-20171027163421-cd47fb39b79f/go.mod h1:JpH9J1c9oX6otFSgdUHwUBUizmKlrMjxWnIAjff4m04= -github.com/lucas-clemente/quic-clients v0.1.0/go.mod h1:y5xVIEoObKqULIKivu+gD/LU90pL73bTdtQjPBvtCBk= -github.com/lucas-clemente/quic-go v0.10.2/go.mod h1:hvaRS9IHjFLMq76puFJeWNfmn+H70QZ/CXoxqw9bzao= -github.com/lucas-clemente/quic-go-certificates v0.0.0-20160823095156-d2f86524cced/go.mod h1:NCcRLrOTZbzhZvixZLlERbJtDtYsmMw8Jc4vS8Z0g58= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mholt/certmagic v0.6.2-0.20190624175158-6a42ef9fe8c2/go.mod h1:g4cOPxcjV0oFq3qwpjSA30LReKD8AoIfwAY9VvG35NY= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.3/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -450,19 +500,25 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= -github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -470,42 +526,46 @@ github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs= -github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= +github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= +github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -514,51 +574,63 @@ github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6T github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/robfig/cron v1.1.0 h1:jk4/Hud3TTdcrJgUOBgsqrZBarcxl6ADIjSC2iniwLY= github.com/robfig/cron v1.1.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday v0.0.0-20170610170232-067529f716f4/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= -github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= +github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= +github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -568,71 +640,91 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= -github.com/vladimirvivien/echo v0.0.1-alpha.6 h1:L1elSMyiiqia7+5ikH24xKIkYAlecRXP6i4YmAF1tkc= -github.com/vladimirvivien/echo v0.0.1-alpha.6/go.mod h1:64h/A7+5GmiBaeztyIr8BVf/07B7knV6OAP06jX+oyE= -github.com/vmware-tanzu/crash-diagnostics v0.3.4 h1:ytpBTTDg9oBErLnwBsMAc+AcE8W3oIJhD7PagJ9xIAM= -github.com/vmware-tanzu/crash-diagnostics v0.3.4/go.mod h1:2+zhcACaLIZesMLGJrBV4GJwMel0KGid6dCFgt2TdP0= +github.com/vladimirvivien/gexe v0.1.1 h1:2A0SBaOSKH+cwLVdt6H+KkHZotZWRNLlWygANGw5DxE= +github.com/vladimirvivien/gexe v0.1.1/go.mod h1:LHQL00w/7gDUKIak24n801ABp8C+ni6eBht9vGVst8w= +github.com/vmware-tanzu/crash-diagnostics v0.3.7 h1:6gbv/3o1FzyRLS7Dz/+yVg1Lk1oRBQLyI3d1YTtlTT8= +github.com/vmware-tanzu/crash-diagnostics v0.3.7/go.mod h1:gO8670rd+qdjnJVol674snT/A46GQ27u085kKhZznlM= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= +github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= -go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= +go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= +go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= +go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= +go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= +go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= +go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= +go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.starlark.net v0.0.0-20201006213952-227f4aabceb5 h1:ApvY/1gw+Yiqb/FKeks3KnVPWpkR3xzij82XPKLjJVw= go.starlark.net v0.0.0-20201006213952-227f4aabceb5/go.mod h1:f0znQkUKRrkk36XxWbGjMqQM8wGv/xHBVE2qc3B5oFU= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= -go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee h1:4yd7jl+vXjalO5ztz6Vc1VADv+S/80LGJmyl1ROJ2AI= -golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -654,8 +746,10 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -663,23 +757,21 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -691,7 +783,8 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -708,15 +801,35 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 h1:ADo5wSpq2gqaCGQWzk7S5vd//0iyyLeAratkEoG5dLE= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 h1:ld7aEMNHoBnnDAX15v1T6z31v8HwR2A9FYOuAhWqkwc= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f h1:Qmd2pbz05z7z6lm0DrgQVVPuBm92jqujBKMHMOlOQEw= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -726,7 +839,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -734,12 +848,10 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -750,10 +862,14 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -761,6 +877,7 @@ golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -776,30 +893,62 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091 h1:DMyOG0U+gKfu8JZzg2UQe9MeaC1X+xQWlAKcRnjxjCw= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -810,15 +959,13 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -840,6 +987,7 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -848,16 +996,27 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.1.0 h1:Phva6wqu+xR//Njw6iorylFFgn/z547tw5Ne3HZPQ+k= -gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= +gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= +gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -874,6 +1033,19 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.56.0 h1:08F9XVYTLOGeSQb3xI9C0gXMuQanhdGed0cWFhDozbI= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -903,15 +1075,40 @@ google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987 h1:PDIOdWxZ8eRizhKa1AAvY53xsvLB1cWorMjslvY3VA8= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71 h1:z+ErRPu0+KS02Td3fOAgdX+lnPDh/VyaABEJPD4JRQs= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -925,8 +1122,22 @@ google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0 h1:T7P4R73V3SSDPhH7WW7ATbfViLtmamH0DKrP3f9AuDI= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -936,20 +1147,26 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/mcuadros/go-syslog.v2 v2.2.1/go.mod h1:l5LPIyOOyIdQquNg+oU6Z3524YwrcqEm0aKH+5zpt2U= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= @@ -958,98 +1175,94 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.19.0/go.mod h1:I1K45XlvTrDjmj5LoM5LuP/KYrhWbjUKT/SoPG0qTjw= -k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= k8s.io/api v0.19.12/go.mod h1:EK+KvSq2urA6+CjVdZyAHEphXoLq2K2eW6lxOzTKSaY= -k8s.io/api v0.20.9 h1:BKGV+rmHIyEVaJvWDYfqyfvHqdL/PhZalNr4mM/932Q= -k8s.io/api v0.20.9/go.mod h1:wTKbf3LIlu+vuXqOk4Bi5drnSUtB10ou5XYrlgOuCdQ= -k8s.io/apiextensions-apiserver v0.19.2/go.mod h1:EYNjpqIAvNZe+svXVx9j4uBaVhTB4C94HkY3w058qcg= -k8s.io/apiextensions-apiserver v0.19.12 h1:wY1liMH0Q2XozYj/qFQp+MJR0BbuMBn/L8/IMDvDByo= -k8s.io/apiextensions-apiserver v0.19.12/go.mod h1:LXHRkG+7v33P+IzTKYVuqffGlB7Xg7K0VgfObTacbQg= -k8s.io/apimachinery v0.18.8/go.mod h1:6sQd+iHEqmOtALqOFjSWp2KZ9F0wlU/nWm0ZgsYWMig= +k8s.io/api v0.22.2 h1:M8ZzAD0V6725Fjg53fKeTJxGsJvRbk4TEm/fexHMtfw= +k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8= +k8s.io/apiextensions-apiserver v0.22.2 h1:zK7qI8Ery7j2CaN23UCFaC1hj7dMiI87n01+nKuewd4= +k8s.io/apiextensions-apiserver v0.22.2/go.mod h1:2E0Ve/isxNl7tWLSUDgi6+cmwHi5fQRdwGVCxbC+KFA= k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= -k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/apimachinery v0.19.12/go.mod h1:9eb44nUQSsz9QZiilFRuMj3ZbTmoWolU8S2gnXoRMjo= -k8s.io/apimachinery v0.20.9 h1:3e+0NczVYKqsEDiuxA+0a50t27fCyhyI7vSrmfaMpLg= -k8s.io/apimachinery v0.20.9/go.mod h1:kQa//VOAwyVwJ2+L9kOREbsnryfsGSkSM1przND4+mw= -k8s.io/apiserver v0.19.2/go.mod h1:FreAq0bJ2vtZFj9Ago/X0oNGC51GfubKK/ViOKfVAOA= -k8s.io/apiserver v0.19.12 h1:xQjt/jLqdYszJTRTDDnHUnA0S+a1TZ/8rgWr0/xoa6I= +k8s.io/apimachinery v0.22.2 h1:ejz6y/zNma8clPVfNDLnPbleBo6MpoFy/HBiBqCouVk= +k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= k8s.io/apiserver v0.19.12/go.mod h1:ldZAZTNIKfMMv/UUEhk6UyTXC0/34iRdNFHo+MJOPc4= -k8s.io/cli-runtime v0.19.2/go.mod h1:CMynmJM4Yf02TlkbhKxoSzi4Zf518PukJ5xep/NaNeY= -k8s.io/cli-runtime v0.20.9 h1:KJ/z9I9r0Oyb7mjfsO4ZdasGCihIiTO7llXz+oZ0rgA= -k8s.io/cli-runtime v0.20.9/go.mod h1:MjEpWNIcmJL+gANS1+SVfvWpRSQRxOvTcx+vzvDTtTY= +k8s.io/apiserver v0.22.2 h1:TdIfZJc6YNhu2WxeAOWq1TvukHF0Sfx0+ln4XK9qnL4= +k8s.io/apiserver v0.22.2/go.mod h1:vrpMmbyjWrgdyOvZTSpsusQq5iigKNWv9o9KlDAbBHI= +k8s.io/cli-runtime v0.22.2 h1:fsd9rFk9FSaVq4SUq1fM27c8CFGsYZUJ/3BkgmjYWuY= +k8s.io/cli-runtime v0.22.2/go.mod h1:tkm2YeORFpbgQHEK/igqttvPTRIHFRz5kATlw53zlMI= k8s.io/client-go v0.19.0/go.mod h1:H9E/VT95blcFQnlyShFgnFT9ZnJOAceiUHM3MlRC+mU= -k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA= k8s.io/client-go v0.19.12/go.mod h1:BAGKQraZ6fDmXhT46pGXWZQQqN7P4E0BJux0+9O6Gt0= -k8s.io/client-go v0.20.9 h1:Th0Ccrpq8nEuVOzvgn7GhdjUSLCGB92AUxSfUURstDU= -k8s.io/client-go v0.20.9/go.mod h1:SjslwSB3f2wb/RwvGMfPIwsiBTPnD/Hp1xBGlz6U3t8= -k8s.io/cluster-bootstrap v0.19.2 h1:6/LI5EnKCcB0QiDKIsTxoCOdKZtsSwr8Xm/tEhiMv78= -k8s.io/cluster-bootstrap v0.19.2/go.mod h1:bzngsppPfdt9vAHUnDIEoMNsxD2b6XArVVH/W9PDDFk= +k8s.io/client-go v0.22.2 h1:DaSQgs02aCC1QcwUdkKZWOeaVsQjYvWv8ZazcZ6JcHc= +k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U= +k8s.io/cluster-bootstrap v0.22.2 h1:jP6Nkp3CdSfr50cAn/7WGsNS52zrwMhvr0V+E3Vkh/w= +k8s.io/cluster-bootstrap v0.22.2/go.mod h1:ZkmQKprEqvrUccMnbRHISsMscA1dsQ8SffM9nHq6CgE= k8s.io/code-generator v0.19.0/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= -k8s.io/code-generator v0.19.2/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= k8s.io/code-generator v0.19.12/go.mod h1:ADrDvaUQWGn4a8lX0ONtzb7uFmDRQOMSYIMk1qWIAx8= -k8s.io/component-base v0.19.2/go.mod h1:g5LrsiTiabMLZ40AR6Hl45f088DevyGY+cCE2agEIVo= -k8s.io/component-base v0.19.12 h1:pEsTceFc3Hs9qJX4h2XYXQDzRMCNzTO8MX34bhjq0Io= +k8s.io/code-generator v0.22.2/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= k8s.io/component-base v0.19.12/go.mod h1:tpwExE0sY3A7CwtlxGL7SnQOdQfUlnFybT6GmAD+z/s= -k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/component-base v0.22.2 h1:vNIvE0AIrLhjX8drH0BgCNJcR4QZxMXcJzBsDplDx9M= +k8s.io/component-base v0.22.2/go.mod h1:5Br2QhI9OTe79p+TzPe9JKNQYvEKbq9rTJDWllunGug= +k8s.io/component-helpers v0.22.2/go.mod h1:+N61JAR9aKYSWbnLA88YcFr9K/6ISYvRNybX7QW7Rs8= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= +k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-aggregator v0.19.12 h1:OwyNUe/7/gxzEnaLd3sC9Yrpx0fZAERzvFslX5Qq5g8= k8s.io/kube-aggregator v0.19.12/go.mod h1:K76wPd03pSHEmS1FgJOcpryac5C3va4cbCvSu+4EmE0= -k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/kubectl v0.19.2/go.mod h1:4ib3oj5ma6gF95QukTvC7ZBMxp60+UEAhDPjLuBIrV4= -k8s.io/metrics v0.19.2/go.mod h1:IlLaAGXN0q7yrtB+SV0q3JIraf6VtlDr+iuTcX21fCU= +k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e h1:KLHHjkdQFomZy8+06csTWZ0m1343QqxZhR2LJ1OxCYM= +k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= +k8s.io/kubectl v0.22.2/go.mod h1:BApg2j0edxLArCOfO0ievI27EeTQqBDMNU9VQH734iQ= +k8s.io/metrics v0.22.2/go.mod h1:GUcsBtpsqQD1tKFS/2wCKu4ZBowwRncLOJH1rgWs3uw= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b h1:wxEMGetGMur3J1xuGLQY7GEQYg9bZxKn3tKo5k/eYcs= +k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/cluster-api v0.3.11-0.20210106212952-b6c1b5b3db3d h1:OwD6b5QiyY4JXWRu7bK0gxKv/x51akqGLPmD1kRJmmI= -sigs.k8s.io/cluster-api v0.3.11-0.20210106212952-b6c1b5b3db3d/go.mod h1:HwDMTKYusSK0SnTAr/P3htrxdiykBADqffkamkiUXhk= -sigs.k8s.io/controller-runtime v0.7.1-0.20201215171748-096b2e07c091 h1:tqrTDj7mJmM6TdpoM1rN2PzBRH9yzCReqKGMy4sp+f0= -sigs.k8s.io/controller-runtime v0.7.1-0.20201215171748-096b2e07c091/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU= -sigs.k8s.io/kind v0.9.0/go.mod h1:cxKQWwmbtRDzQ+RNKnR6gZG6fjbeTtItp5cGf+ww+1Y= -sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= -sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= -sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/cluster-api v1.0.0 h1:GcVA2ObQTXo/+jzSLWPy4Bd3NeiwJyAB8n19kyJIotA= +sigs.k8s.io/cluster-api v1.0.0/go.mod h1:V230kMSaYENTUcx1QRkoRCklb3vfphQGV3/z4ODNGWo= +sigs.k8s.io/controller-runtime v0.10.2 h1:jW8qiY+yMnnPx6O9hu63tgcwaKzd1yLYui+mpvClOOc= +sigs.k8s.io/controller-runtime v0.10.2/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= +sigs.k8s.io/kustomize/api v0.8.11/go.mod h1:a77Ls36JdfCWojpUqR6m60pdGY1AYFix4AH83nJtY1g= +sigs.k8s.io/kustomize/cmd/config v0.9.13/go.mod h1:7547FLF8W/lTaDf0BDqFTbZxM9zqwEJqCKN9sSR0xSs= +sigs.k8s.io/kustomize/kustomize/v4 v4.2.0/go.mod h1:MOkR6fmhwG7hEDRXBYELTi5GSFcLwfqwzTRHW3kv5go= +sigs.k8s.io/kustomize/kyaml v0.11.0/go.mod h1:GNMwjim4Ypgp/MueD3zXHLRJEjz7RvtPae0AwlvEMFM= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/hack/build-image/Dockerfile b/hack/build-image/Dockerfile index 395822ffc..5f3564631 100644 --- a/hack/build-image/Dockerfile +++ b/hack/build-image/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.16 +FROM golang:1.17 ARG GOPROXY @@ -24,17 +24,19 @@ ENV GOPROXY=${GOPROXY} RUN mkdir -p /go/src/k8s.io WORKDIR /go/src/k8s.io RUN git config --global advice.detachedHead false -RUN git clone -b v0.18.4 https://github.com/kubernetes/code-generator +RUN git clone -b v0.22.2 https://github.com/kubernetes/code-generator -RUN wget --quiet https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.1/kubebuilder_2.3.1_linux_amd64.tar.gz && \ - tar -zxvf kubebuilder_2.3.1_linux_amd64.tar.gz && \ - mv kubebuilder_2.3.1_linux_amd64 /usr/local/kubebuilder && \ - chmod +x /usr/local/kubebuilder && \ - export PATH=$PATH:/usr/local/kubebuilder/bin && \ - rm kubebuilder_2.3.1_linux_amd64.tar.gz +# kubebuilder test bundle is separated from kubebuilder. Need to setup it for CI test. +RUN curl -sSLo envtest-bins.tar.gz https://go.kubebuilder.io/test-tools/1.22.1/linux/amd64 && \ + mkdir /usr/local/kubebuilder && \ + tar -C /usr/local/kubebuilder --strip-components=1 -zvxf envtest-bins.tar.gz + +RUN wget --quiet https://github.com/kubernetes-sigs/kubebuilder/releases/download/v3.2.0/kubebuilder_linux_amd64 && \ + mv kubebuilder_linux_amd64 /usr/local/kubebuilder/bin/kubebuilder && \ + chmod +x /usr/local/kubebuilder/bin/kubebuilder # get controller-tools -RUN go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.3.0 +RUN go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.7.0 # get goimports (the revision is pinned so we don't indiscriminately update, but the particular commit # is not important) diff --git a/hack/crd-gen/v1beta1/main.go b/hack/crd-gen/v1beta1/main.go deleted file mode 100644 index 2f204a0d2..000000000 --- a/hack/crd-gen/v1beta1/main.go +++ /dev/null @@ -1,136 +0,0 @@ -/* -Copyright the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This code embeds the CRD manifests in config/crd/v1beta1/bases in -// config/crd/v1beta1/crds/crds.go. - -package main - -import ( - "bytes" - "compress/gzip" - "fmt" - "io" - "io/ioutil" - "log" - "os" - "text/template" -) - -// This is relative to config/crd/crds -const goHeaderFile = "../../../../hack/boilerplate.go.txt" - -const tpl = `{{.GoHeader}} -// Code generated by crds_generate.go; DO NOT EDIT. - -package crds - -import ( - "bytes" - "compress/gzip" - "io/ioutil" - - apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" - apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - "k8s.io/client-go/kubernetes/scheme" -) - -var rawCRDs = [][]byte{ -{{- range .RawCRDs }} - []byte({{ . }}), -{{- end }} -} - -var CRDs = crds() - -func crds() []*apiextv1beta1.CustomResourceDefinition { - apiextinstall.Install(scheme.Scheme) - decode := scheme.Codecs.UniversalDeserializer().Decode - var objs []*apiextv1beta1.CustomResourceDefinition - for _, crd := range rawCRDs { - gzr, err := gzip.NewReader(bytes.NewReader(crd)) - if err != nil { - panic(err) - } - bytes, err := ioutil.ReadAll(gzr) - if err != nil { - panic(err) - } - gzr.Close() - - obj, _, err := decode(bytes, nil, nil) - if err != nil { - panic(err) - } - objs = append(objs, obj.(*apiextv1beta1.CustomResourceDefinition)) - } - return objs -} -` - -type templateData struct { - GoHeader string - RawCRDs []string -} - -func main() { - headerBytes, err := ioutil.ReadFile(goHeaderFile) - if err != nil { - log.Fatalln(err) - } - - data := templateData{ - GoHeader: string(headerBytes), - } - - // This is relative to config/crd/crds - manifests, err := ioutil.ReadDir("../bases") - if err != nil { - log.Fatalln(err) - } - - for _, crd := range manifests { - file, err := os.Open("../bases/" + crd.Name()) - if err != nil { - log.Fatalln(err) - } - - // gzip compress manifest - var buf bytes.Buffer - gzw := gzip.NewWriter(&buf) - if _, err := io.Copy(gzw, file); err != nil { - log.Fatalln(err) - } - file.Close() - gzw.Close() - - data.RawCRDs = append(data.RawCRDs, fmt.Sprintf("%q", buf.Bytes())) - } - - t, err := template.New("crd").Parse(tpl) - if err != nil { - log.Fatalln(err) - } - - out, err := os.Create("crds.go") - if err != nil { - log.Fatalln(err) - } - - if err := t.Execute(out, data); err != nil { - log.Fatalln(err) - } -} diff --git a/hack/release-tools/tag-release.sh b/hack/release-tools/tag-release.sh index 5301dd1de..a8591a899 100755 --- a/hack/release-tools/tag-release.sh +++ b/hack/release-tools/tag-release.sh @@ -38,6 +38,9 @@ # This script is meant to be a combination of documentation and executable. # If you have questions at any point, please stop and ask! +# Fail on any error. +set -eo pipefail + # Directory in which the script itself resides, so we can use it for calling programs that are in the same directory. DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" @@ -89,18 +92,31 @@ fi # Since we're past the validation of the VELERO_VERSION, parse the version's individual components. eval $(go run $DIR/chk_version.go) - printf "To clarify, you've provided a version string of $VELERO_VERSION.\n" printf "Based on this, the following assumptions have been made: \n" -[[ "$VELERO_PATCH" != 0 ]] && printf "*\t This is a patch release.\n" +# $VELERO_PATCH gets populated by the chk_version.go scrip that parses and verifies the given version format +# If we've got a patch release, we assume the tag is on release branch. +if [[ "$VELERO_PATCH" != 0 ]]; then + printf "*\t This is a patch release.\n" + ON_RELEASE_BRANCH=TRUE +fi -# $VELERO_PRERELEASE gets populated by the chk_version.go script that parses and verifies the given version format +# $VELERO_PRERELEASE gets populated by the chk_version.go script that parses and verifies the given version format +# If we've got a GA release, we assume the tag is on release branch. # -n is "string is non-empty" [[ -n $VELERO_PRERELEASE ]] && printf "*\t This is a pre-release.\n" # -z is "string is empty" -[[ -z $VELERO_PRERELEASE ]] && printf "*\t This is a GA release.\n" +if [[ -z $VELERO_PRERELEASE ]]; then + printf "*\t This is a GA release.\n" + ON_RELEASE_BRANCH=TRUE +fi + +if [[ "$ON_RELEASE_BRANCH" == "TRUE" ]]; then + release_branch_name=release-$VELERO_MAJOR.$VELERO_MINOR + printf "*\t The commit to tag is on branch: %s. Please make sure this branch has been created.\n" $release_branch_name +fi if [[ $publish == "TRUE" ]]; then echo "If this is all correct, press enter/return to proceed to TAG THE RELEASE and UPLOAD THE TAG TO GITHUB." @@ -117,55 +133,29 @@ echo "Alright, let's go." echo "Pulling down all git tags and branches before doing any work." git fetch "$remote" --tags -# $VELERO_PATCH gets populated by the chk_version.go scrip that parses and verifies the given version format -# If we've got a patch release, we'll need to create a release branch for it. -if [[ "$VELERO_PATCH" > 0 ]]; then - release_branch_name=release-$VELERO_MAJOR.$VELERO_MINOR +if [[ -n $release_branch_name ]]; then + # Tag on release branch remote_release_branch_name="$remote/$release_branch_name" # Determine whether the local and remote release branches already exist local_branch=$(git branch | grep "$release_branch_name") remote_branch=$(git branch -r | grep "$remote_release_branch_name") - - if [[ -n $remote_branch ]]; then - if [[ -z $local_branch ]]; then + if [[ -z $remote_branch ]]; then + echo "The branch $remote_release_branch_name must be created before you tag the release." + exit 1 + fi + if [[ -z $local_branch ]]; then # Remote branch exists, but does not exist locally. Checkout and track the remote branch. git checkout --track "$remote_release_branch_name" - else + else # Checkout the local release branch and ensure it is up to date with the remote git checkout "$release_branch_name" git pull --set-upstream "$remote" "$release_branch_name" - fi - else - if [[ -z $local_branch ]]; then - # Neither the remote or local release branch exists, create it - git checkout -b $release_branch_name - else - # The local branch exists so check it out. - git checkout $release_branch_name - fi fi - - echo "Now you'll need to cherry-pick any relevant git commits into this release branch." - echo "Either pause this script with ctrl-z, or open a new terminal window and do the cherry-picking." - if [[ $publish == "TRUE" ]]; then - read -p "Press enter when you're done cherry-picking. THIS WILL MAKE A TAG PUSH THE BRANCH TO $remote" - else - read -p "Press enter when you're done cherry-picking." - fi - - # TODO can/should we add a way to review the cherry-picked commits before the push? - - if [[ $publish == "TRUE" ]]; then - echo "Pushing $release_branch_name to \"$remote\" remote" - git push --set-upstream "$remote" $release_branch_name - fi - tag_and_push else echo "Checking out $remote/main." git checkout "$remote"/main - tag_and_push fi diff --git a/hack/test.sh b/hack/test.sh index 7814d2753..bab638319 100755 --- a/hack/test.sh +++ b/hack/test.sh @@ -38,5 +38,10 @@ if [[ -n "${GOFLAGS:-}" ]]; then echo "GOFLAGS: ${GOFLAGS}" fi -go test -installsuffix "static" -short -timeout 60s "${TARGETS[@]}" +# After bumping up "sigs.k8s.io/controller-runtime" to v0.10.2, get the error "panic: mkdir /.cache/kubebuilder-envtest: permission denied" +# when running this script with "make test" command. This is caused by that "make test" runs inside a container with user and group specified, +# but the user and group don't exist inside the container, when the code(https://github.com/kubernetes-sigs/controller-runtime/blob/v0.10.2/pkg/internal/testing/addr/manager.go#L44) +# tries to get the cache directory, it gets the directory "/" and then get the permission error when trying to create directory under "/". +# Specifying the cache directory by environment variable "XDG_CACHE_HOME" to workaround it +XDG_CACHE_HOME=/tmp/ go test -installsuffix "static" -short -timeout 60s "${TARGETS[@]}" echo "Success!" diff --git a/hack/update-generated-crd-code.sh b/hack/update-generated-crd-code.sh index 35bd93927..e3355be48 100755 --- a/hack/update-generated-crd-code.sh +++ b/hack/update-generated-crd-code.sh @@ -44,24 +44,21 @@ ${GOPATH}/src/k8s.io/code-generator/generate-groups.sh \ --output-base ../../.. \ $@ -# Generate both apiextensions.k8s.io/v1beta1 and apiextensions.k8s.io/v1 -for version in v1beta1 v1 -do - # Generate manifests e.g. CRD, RBAC etc. - controller-gen \ - crd:crdVersions=$version,preserveUnknownFields=false,trivialVersions=true \ - paths=./pkg/apis/velero/v1/... \ - paths=./pkg/controller/... \ - output:crd:artifacts:config=config/crd/$version/bases +# Generate apiextensions.k8s.io/v1 +# Generate manifests e.g. CRD, RBAC etc. +controller-gen \ + crd:crdVersions=v1\ + paths=./pkg/apis/velero/v1/... \ + paths=./pkg/controller/... \ + output:crd:artifacts:config=config/crd/v1/bases - # this is a super hacky workaround for https://github.com/kubernetes/kubernetes/issues/91395 - # which a result of fixing the validation on CRD objects. The validation ensures the fields that are list map keys, are either marked - # as required or have default values to ensure merging of list map items work as expected. - # With "containerPort" and "protocol" being considered as x-kubernetes-list-map-keys in the container ports, and "protocol" was not - # a required field, the CRD would fail validation with errors similar to the one reported in https://github.com/kubernetes/kubernetes/issues/91395. - # once controller-gen (above) is able to generate CRDs with `protocol` as a required field, this hack can be removed. - kubectl patch -f config/crd/$version/bases/velero.io_restores.yaml -p "$(cat hack/restore-crd-patch-$version.json)" --type=json --local=true -o yaml > /tmp/velero.io_restores-yaml.patched - mv /tmp/velero.io_restores-yaml.patched config/crd/$version/bases/velero.io_restores.yaml +# this is a super hacky workaround for https://github.com/kubernetes/kubernetes/issues/91395 +# which a result of fixing the validation on CRD objects. The validation ensures the fields that are list map keys, are either marked +# as required or have default values to ensure merging of list map items work as expected. +# With "containerPort" and "protocol" being considered as x-kubernetes-list-map-keys in the container ports, and "protocol" was not +# a required field, the CRD would fail validation with errors similar to the one reported in https://github.com/kubernetes/kubernetes/issues/91395. +# once controller-gen (above) is able to generate CRDs with `protocol` as a required field, this hack can be removed. +kubectl patch -f config/crd/v1/bases/velero.io_restores.yaml -p "$(cat hack/restore-crd-patch-v1.json)" --type=json --local=true -o yaml > /tmp/velero.io_restores-yaml.patched +mv /tmp/velero.io_restores-yaml.patched config/crd/v1/bases/velero.io_restores.yaml - go generate ./config/crd/$version/crds -done +go generate ./config/crd/v1/crds diff --git a/hack/verify-generated-crd-code.sh b/hack/verify-generated-crd-code.sh index 90d1f04b9..ef7490c83 100755 --- a/hack/verify-generated-crd-code.sh +++ b/hack/verify-generated-crd-code.sh @@ -19,14 +19,11 @@ HACK_DIR=$(dirname "${BASH_SOURCE}") ${HACK_DIR}/update-generated-crd-code.sh --verify-only # ensure no changes to generated CRDs -for version in v1beta1 v1 -do - if ! git diff --exit-code config/crd/$version/crds/crds.go >/dev/null; then - # revert changes to state before running CRD generation to stay consistent - # with code-generator `--verify-only` option which discards generated changes - git checkout config/crd +if ! git diff --exit-code config/crd/v1/crds/crds.go >/dev/null; then + # revert changes to state before running CRD generation to stay consistent + # with code-generator `--verify-only` option which discards generated changes + git checkout config/crd - echo "CRD verification - failed! Generated CRDs are out-of-date, please run 'make update' and 'git add' the generated file(s)." - exit 1 - fi -done + echo "CRD verification - failed! Generated CRDs are out-of-date, please run 'make update' and 'git add' the generated file(s)." + exit 1 +fi diff --git a/internal/delete/delete_item_action_handler.go b/internal/delete/delete_item_action_handler.go index c8545c345..609f75914 100644 --- a/internal/delete/delete_item_action_handler.go +++ b/internal/delete/delete_item_action_handler.go @@ -19,6 +19,8 @@ package delete import ( "io" + "github.com/vmware-tanzu/velero/pkg/plugin/framework" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/labels" @@ -29,7 +31,6 @@ import ( "github.com/vmware-tanzu/velero/pkg/archive" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/plugin/velero" - "github.com/vmware-tanzu/velero/pkg/util/collections" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) @@ -41,14 +42,13 @@ type Context struct { Filesystem filesystem.Interface Log logrus.FieldLogger DiscoveryHelper discovery.Helper - - resolvedActions []resolvedAction + resolvedActions []framework.DeleteItemResolvedAction } func InvokeDeleteActions(ctx *Context) error { var err error - ctx.resolvedActions, err = resolveActions(ctx.Actions, ctx.DiscoveryHelper) - + resolver := framework.NewDeleteItemActionResolver(ctx.Actions) + ctx.resolvedActions, err = resolver.ResolveActions(ctx.DiscoveryHelper) // No actions installed and no error means we don't have to continue; // just do the backup deletion without worrying about plugins. if len(ctx.resolvedActions) == 0 && err == nil { @@ -74,15 +74,8 @@ func InvokeDeleteActions(ctx *Context) error { processdResources := sets.NewString() - ctx.Log.Debugf("Trying to reconcile resource names with Kube API server.") - // Transform resource names based on what's canonical in the API server. for resource := range backupResources { - gvr, _, err := ctx.DiscoveryHelper.ResourceFor(schema.ParseGroupResource(resource).WithVersion("")) - if err != nil { - return errors.Wrapf(err, "failed to resolve resource into complete group/version/resource: %v", resource) - } - - groupResource := gvr.GroupResource() + groupResource := schema.ParseGroupResource(resource) // We've already seen this group/resource, so don't process it again. if processdResources.Has(groupResource.String()) { @@ -92,8 +85,6 @@ func InvokeDeleteActions(ctx *Context) error { // Get a list of all items that exist for this resource resourceList := backupResources[groupResource.String()] if resourceList == nil { - // After canonicalization from the API server, the resources may not exist in the tarball - // Skip them if that's the case. continue } @@ -119,10 +110,10 @@ func InvokeDeleteActions(ctx *Context) error { itemLog.Infof("invoking DeleteItemAction plugins") for _, action := range actions { - if !action.selector.Matches(labels.Set(obj.GetLabels())) { + if !action.Selector.Matches(labels.Set(obj.GetLabels())) { continue } - err = action.Execute(&velero.DeleteItemActionExecuteInput{ + err = action.DeleteItemAction.Execute(&velero.DeleteItemActionExecuteInput{ Item: obj, Backup: ctx.Backup, }) @@ -139,65 +130,12 @@ func InvokeDeleteActions(ctx *Context) error { } // getApplicableActions takes resolved DeleteItemActions and filters them for a given group/resource and namespace. -func (ctx *Context) getApplicableActions(groupResource schema.GroupResource, namespace string) []resolvedAction { - var actions []resolvedAction - +func (ctx *Context) getApplicableActions(groupResource schema.GroupResource, namespace string) []framework.DeleteItemResolvedAction { + var actions []framework.DeleteItemResolvedAction for _, action := range ctx.resolvedActions { - if !action.resourceIncludesExcludes.ShouldInclude(groupResource.String()) { - continue + if action.ShouldUse(groupResource, namespace, nil, ctx.Log) { + actions = append(actions, action) } - - if namespace != "" && !action.namespaceIncludesExcludes.ShouldInclude(namespace) { - continue - } - - if namespace == "" && !action.namespaceIncludesExcludes.IncludeEverything() { - continue - } - - actions = append(actions, action) } - return actions } - -// resolvedActions are DeleteItemActions decorated with resource/namespace include/exclude collections, as well as label selectors for easy comparison. -type resolvedAction struct { - velero.DeleteItemAction - - resourceIncludesExcludes *collections.IncludesExcludes - namespaceIncludesExcludes *collections.IncludesExcludes - selector labels.Selector -} - -// resolveActions resolves the AppliesTo ResourceSelectors of DeleteItemActions plugins against the Kubernetes discovery API for fully-qualified names. -func resolveActions(actions []velero.DeleteItemAction, helper discovery.Helper) ([]resolvedAction, error) { - var resolved []resolvedAction - - for _, action := range actions { - resourceSelector, err := action.AppliesTo() - if err != nil { - return nil, err - } - - resources := collections.GetResourceIncludesExcludes(helper, resourceSelector.IncludedResources, resourceSelector.ExcludedResources) - namespaces := collections.NewIncludesExcludes().Includes(resourceSelector.IncludedNamespaces...).Excludes(resourceSelector.ExcludedNamespaces...) - - selector := labels.Everything() - if resourceSelector.LabelSelector != "" { - if selector, err = labels.Parse(resourceSelector.LabelSelector); err != nil { - return nil, err - } - } - - res := resolvedAction{ - DeleteItemAction: action, - resourceIncludesExcludes: resources, - namespaceIncludesExcludes: namespaces, - selector: selector, - } - resolved = append(resolved, res) - } - - return resolved, nil -} diff --git a/pkg/apis/velero/v1/constants.go b/pkg/apis/velero/v1/constants.go index a7292d568..4d06be19f 100644 --- a/pkg/apis/velero/v1/constants.go +++ b/pkg/apis/velero/v1/constants.go @@ -46,4 +46,8 @@ const ( // APIGroupVersionsFeatureFlag is the feature flag string that defines whether or not to handle multiple API Group Versions APIGroupVersionsFeatureFlag = "EnableAPIGroupVersions" + + // UploadProgressFeatureFlag is the feature flag string that defines whether or not upload progress monitoring is enabled + // and whether or not ItemSnapshotters should be invoked + UploadProgressFeatureFlag = "EnableUploadProgress" ) diff --git a/pkg/apis/velero/v1/download_request_types.go b/pkg/apis/velero/v1/download_request_types.go index 945cd8257..e773f64df 100644 --- a/pkg/apis/velero/v1/download_request_types.go +++ b/pkg/apis/velero/v1/download_request_types.go @@ -25,13 +25,14 @@ type DownloadRequestSpec struct { } // DownloadTargetKind represents what type of file to download. -// +kubebuilder:validation:Enum=BackupLog;BackupContents;BackupVolumeSnapshots;BackupResourceList;RestoreLog;RestoreResults +// +kubebuilder:validation:Enum=BackupLog;BackupContents;BackupVolumeSnapshots;BackupItemSnapshots;BackupResourceList;RestoreLog;RestoreResults type DownloadTargetKind string const ( DownloadTargetKindBackupLog DownloadTargetKind = "BackupLog" DownloadTargetKindBackupContents DownloadTargetKind = "BackupContents" DownloadTargetKindBackupVolumeSnapshots DownloadTargetKind = "BackupVolumeSnapshots" + DownloadTargetKindBackupItemSnapshots DownloadTargetKind = "BackupItemSnapshots" DownloadTargetKindBackupResourceList DownloadTargetKind = "BackupResourceList" DownloadTargetKindRestoreLog DownloadTargetKind = "RestoreLog" DownloadTargetKindRestoreResults DownloadTargetKind = "RestoreResults" diff --git a/pkg/apis/velero/v1/zz_generated.deepcopy.go b/pkg/apis/velero/v1/zz_generated.deepcopy.go index 021ee11ff..27e687336 100644 --- a/pkg/apis/velero/v1/zz_generated.deepcopy.go +++ b/pkg/apis/velero/v1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/pkg/backup/backup.go b/pkg/backup/backup.go index 746695e00..697be8500 100644 --- a/pkg/backup/backup.go +++ b/pkg/backup/backup.go @@ -1,5 +1,5 @@ /* -Copyright the Velero contributors. +Copyright the Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -33,7 +33,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" kubeerrs "k8s.io/apimachinery/pkg/util/errors" @@ -44,9 +43,11 @@ import ( "github.com/vmware-tanzu/velero/pkg/discovery" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/podexec" "github.com/vmware-tanzu/velero/pkg/restic" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/collections" ) @@ -62,6 +63,9 @@ type Backupper interface { // Backup takes a backup using the specification in the velerov1api.Backup and writes backup and log data // to the given writers. Backup(logger logrus.FieldLogger, backup *Request, backupFile io.Writer, actions []velero.BackupItemAction, volumeSnapshotterGetter VolumeSnapshotterGetter) error + BackupWithResolvers(log logrus.FieldLogger, backupRequest *Request, backupFile io.Writer, + backupItemActionResolver framework.BackupItemActionResolver, itemSnapshotterResolver framework.ItemSnapshotterResolver, + volumeSnapshotterGetter VolumeSnapshotterGetter) error } // kubernetesBackupper implements Backupper. @@ -76,14 +80,6 @@ type kubernetesBackupper struct { clientPageSize int } -type resolvedAction struct { - velero.BackupItemAction - - resourceIncludesExcludes *collections.IncludesExcludes - namespaceIncludesExcludes *collections.IncludesExcludes - selector labels.Selector -} - func (i *itemKey) String() string { return fmt.Sprintf("resource=%s,namespace=%s,name=%s", i.resource, i.namespace, i.name) } @@ -121,38 +117,6 @@ func NewKubernetesBackupper( }, nil } -func resolveActions(actions []velero.BackupItemAction, helper discovery.Helper) ([]resolvedAction, error) { - var resolved []resolvedAction - - for _, action := range actions { - resourceSelector, err := action.AppliesTo() - if err != nil { - return nil, err - } - - resources := collections.GetResourceIncludesExcludes(helper, resourceSelector.IncludedResources, resourceSelector.ExcludedResources) - namespaces := collections.NewIncludesExcludes().Includes(resourceSelector.IncludedNamespaces...).Excludes(resourceSelector.ExcludedNamespaces...) - - selector := labels.Everything() - if resourceSelector.LabelSelector != "" { - if selector, err = labels.Parse(resourceSelector.LabelSelector); err != nil { - return nil, err - } - } - - res := resolvedAction{ - BackupItemAction: action, - resourceIncludesExcludes: resources, - namespaceIncludesExcludes: namespaces, - selector: selector, - } - - resolved = append(resolved, res) - } - - return resolved, nil -} - // getNamespaceIncludesExcludes returns an IncludesExcludes list containing which namespaces to // include and exclude from the backup. func getNamespaceIncludesExcludes(backup *velerov1api.Backup) *collections.IncludesExcludes { @@ -205,7 +169,20 @@ type VolumeSnapshotterGetter interface { // a complete backup failure is returned. Errors that constitute partial failures (i.e. failures to // back up individual resources that don't prevent the backup from continuing to be processed) are logged // to the backup log. -func (kb *kubernetesBackupper) Backup(log logrus.FieldLogger, backupRequest *Request, backupFile io.Writer, actions []velero.BackupItemAction, volumeSnapshotterGetter VolumeSnapshotterGetter) error { +func (kb *kubernetesBackupper) Backup(log logrus.FieldLogger, backupRequest *Request, backupFile io.Writer, + actions []velero.BackupItemAction, volumeSnapshotterGetter VolumeSnapshotterGetter) error { + backupItemActions := framework.NewBackupItemActionResolver(actions) + itemSnapshotters := framework.NewItemSnapshotterResolver(nil) + return kb.BackupWithResolvers(log, backupRequest, backupFile, backupItemActions, itemSnapshotters, + volumeSnapshotterGetter) +} + +func (kb *kubernetesBackupper) BackupWithResolvers(log logrus.FieldLogger, + backupRequest *Request, + backupFile io.Writer, + backupItemActionResolver framework.BackupItemActionResolver, + itemSnapshotterResolver framework.ItemSnapshotterResolver, + volumeSnapshotterGetter VolumeSnapshotterGetter) error { gzippedData := gzip.NewWriter(backupFile) defer gzippedData.Close() @@ -224,7 +201,7 @@ func (kb *kubernetesBackupper) Backup(log logrus.FieldLogger, backupRequest *Req backupRequest.ResourceIncludesExcludes = collections.GetResourceIncludesExcludes(kb.discoveryHelper, backupRequest.Spec.IncludedResources, backupRequest.Spec.ExcludedResources) log.Infof("Including resources: %s", backupRequest.ResourceIncludesExcludes.IncludesString()) log.Infof("Excluding resources: %s", backupRequest.ResourceIncludesExcludes.ExcludesString()) - log.Infof("Backing up all pod volumes using restic: %t", *backupRequest.Backup.Spec.DefaultVolumesToRestic) + log.Infof("Backing up all pod volumes using Restic: %t", boolptr.IsSetToTrue(backupRequest.Backup.Spec.DefaultVolumesToRestic)) var err error backupRequest.ResourceHooks, err = getResourceHooks(backupRequest.Spec.Hooks.Resources, kb.discoveryHelper) @@ -232,7 +209,12 @@ func (kb *kubernetesBackupper) Backup(log logrus.FieldLogger, backupRequest *Req return err } - backupRequest.ResolvedActions, err = resolveActions(actions, kb.discoveryHelper) + backupRequest.ResolvedActions, err = backupItemActionResolver.ResolveActions(kb.discoveryHelper) + if err != nil { + return err + } + + backupRequest.ResolvedItemSnapshotters, err = itemSnapshotterResolver.ResolveActions(kb.discoveryHelper) if err != nil { return err } diff --git a/pkg/backup/backup_test.go b/pkg/backup/backup_test.go index 351c60429..0747409bb 100644 --- a/pkg/backup/backup_test.go +++ b/pkg/backup/backup_test.go @@ -970,6 +970,30 @@ func TestBackupResourceCohabitation(t *testing.T) { "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", }, }, + { + name: "when deployments exist that are not in the cohabitating groups those are backed up along with apps/deployments", + backup: defaultBackup().Result(), + apiResources: []*test.APIResource{ + test.VeleroDeployments( + builder.ForTestCR("Deployment", "foo", "bar").Result(), + builder.ForTestCR("Deployment", "zoo", "raz").Result(), + ), + test.Deployments( + builder.ForDeployment("foo", "bar").Result(), + builder.ForDeployment("zoo", "raz").Result(), + ), + }, + want: []string{ + "resources/deployments.apps/namespaces/foo/bar.json", + "resources/deployments.apps/namespaces/zoo/raz.json", + "resources/deployments.apps/v1-preferredversion/namespaces/foo/bar.json", + "resources/deployments.apps/v1-preferredversion/namespaces/zoo/raz.json", + "resources/deployments.velero.io/namespaces/foo/bar.json", + "resources/deployments.velero.io/namespaces/zoo/raz.json", + "resources/deployments.velero.io/v1-preferredversion/namespaces/foo/bar.json", + "resources/deployments.velero.io/v1-preferredversion/namespaces/zoo/raz.json", + }, + }, } for _, tc := range tests { diff --git a/pkg/backup/item_backupper.go b/pkg/backup/item_backupper.go index 4f5c7ca8b..32a9f5504 100644 --- a/pkg/backup/item_backupper.go +++ b/pkg/backup/item_backupper.go @@ -1,5 +1,5 @@ /* -Copyright 2020 the Velero contributors. +Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "path/filepath" + "strings" "time" "github.com/pkg/errors" @@ -29,10 +30,10 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" kubeerrs "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/sets" "github.com/vmware-tanzu/velero/internal/hook" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -304,26 +305,9 @@ func (ib *itemBackupper) executeActions( metadata metav1.Object, ) (runtime.Unstructured, error) { for _, action := range ib.backupRequest.ResolvedActions { - if !action.resourceIncludesExcludes.ShouldInclude(groupResource.String()) { - log.Debug("Skipping action because it does not apply to this resource") + if !action.ShouldUse(groupResource, namespace, metadata, log) { continue } - - if namespace != "" && !action.namespaceIncludesExcludes.ShouldInclude(namespace) { - log.Debug("Skipping action because it does not apply to this namespace") - continue - } - - if namespace == "" && !action.namespaceIncludesExcludes.IncludeEverything() { - log.Debug("Skipping action because resource is cluster-scoped and action only applies to specific namespaces") - continue - } - - if !action.selector.Matches(labels.Set(metadata.GetLabels())) { - log.Debug("Skipping action because label selector does not match") - continue - } - log.Info("Executing custom action") updatedItem, additionalItemIdentifiers, err := action.Execute(obj, ib.backupRequest.Backup) @@ -395,7 +379,13 @@ func (ib *itemBackupper) volumeSnapshotter(snapshotLocation *velerov1api.VolumeS // on PVs const ( zoneLabelDeprecated = "failure-domain.beta.kubernetes.io/zone" - zoneLabel = "topology.kubernetes.io/zone" + // this is reused for nodeAffinity requirements + zoneLabel = "topology.kubernetes.io/zone" + + awsEbsCsiZoneKey = "topology.ebs.csi.aws.com/zone" + azureCsiZoneKey = "topology.disk.csi.azure.com/zone" + gkeCsiZoneKey = "topology.gke.io/zone" + gkeZoneSeparator = "__" ) // takePVSnapshot triggers a snapshot for the volume/disk underlying a PersistentVolume if the provided @@ -432,7 +422,14 @@ func (ib *itemBackupper) takePVSnapshot(obj runtime.Unstructured, log logrus.Fie log.Infof("label %q is not present on PersistentVolume, checking deprecated label...", zoneLabel) pvFailureDomainZone, labelFound = pv.Labels[zoneLabelDeprecated] if !labelFound { + var k string log.Infof("label %q is not present on PersistentVolume", zoneLabelDeprecated) + k, pvFailureDomainZone = zoneFromPVNodeAffinity(pv, awsEbsCsiZoneKey, azureCsiZoneKey, gkeCsiZoneKey, zoneLabel, zoneLabelDeprecated) + if pvFailureDomainZone != "" { + log.Infof("zone info from nodeAffinity requirements: %s, key: %s", pvFailureDomainZone, k) + } else { + log.Infof("zone info not available in nodeAffinity requirements") + } } } @@ -535,3 +532,36 @@ func resourceVersion(obj runtime.Unstructured) string { gvk := obj.GetObjectKind().GroupVersionKind() return gvk.Version } + +// zoneFromPVNodeAffinity iterates the node affinity requirement of a PV to +// get its availability zone, it returns the key merely for logging. +func zoneFromPVNodeAffinity(res *corev1api.PersistentVolume, topologyKeys ...string) (string, string) { + nodeAffinity := res.Spec.NodeAffinity + if nodeAffinity == nil { + return "", "" + } + keySet := sets.NewString(topologyKeys...) + providerGke := false + zones := make([]string, 0) + for _, term := range nodeAffinity.Required.NodeSelectorTerms { + if term.MatchExpressions == nil { + continue + } + for _, exp := range term.MatchExpressions { + if keySet.Has(exp.Key) && exp.Operator == "In" && len(exp.Values) > 0 { + if exp.Key == gkeCsiZoneKey { + providerGke = true + zones = append(zones, exp.Values[0]) + } else { + return exp.Key, exp.Values[0] + } + } + } + } + + if providerGke { + return gkeCsiZoneKey, strings.Join(zones, gkeZoneSeparator) + } + + return "", "" +} diff --git a/pkg/backup/item_backupper_test.go b/pkg/backup/item_backupper_test.go index 192b0c585..2152a5301 100644 --- a/pkg/backup/item_backupper_test.go +++ b/pkg/backup/item_backupper_test.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -45,3 +46,127 @@ func Test_resourceKey(t *testing.T) { }) } } + +func Test_zoneFromPVNodeAffinity(t *testing.T) { + keys := []string{ + awsEbsCsiZoneKey, + azureCsiZoneKey, + gkeCsiZoneKey, + zoneLabel, + zoneLabelDeprecated, + } + tests := []struct { + name string + pv *corev1api.PersistentVolume + wantKey string + wantValue string + }{ + { + name: "AWS CSI Volume", + pv: builder.ForPersistentVolume("awscsi").NodeAffinityRequired( + builder.ForNodeSelector( + *builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.ebs.csi.aws.com/zone", + "In", "us-east-2c").Result(), + ).Result(), + ).Result(), + wantKey: "topology.ebs.csi.aws.com/zone", + wantValue: "us-east-2c", + }, + { + name: "Azure CSI Volume", + pv: builder.ForPersistentVolume("azurecsi").NodeAffinityRequired( + builder.ForNodeSelector( + *builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.disk.csi.azure.com/zone", + "In", "us-central").Result(), + ).Result(), + ).Result(), + wantKey: "topology.disk.csi.azure.com/zone", + wantValue: "us-central", + }, + { + name: "GCP CSI Volume", + pv: builder.ForPersistentVolume("gcpcsi").NodeAffinityRequired( + builder.ForNodeSelector( + *builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.gke.io/zone", + "In", "us-west1-a").Result(), + ).Result(), + ).Result(), + wantKey: "topology.gke.io/zone", + wantValue: "us-west1-a", + }, + { + name: "AWS CSI Volume with multiple zone value, returns the first", + pv: builder.ForPersistentVolume("awscsi").NodeAffinityRequired( + builder.ForNodeSelector( + *builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.ebs.csi.aws.com/zone", + "In", "us-east-2c", "us-west").Result(), + ).Result(), + ).Result(), + wantKey: "topology.ebs.csi.aws.com/zone", + wantValue: "us-east-2c", + }, + { + name: "Volume with no matching key", + pv: builder.ForPersistentVolume("no-matching-pv").NodeAffinityRequired( + builder.ForNodeSelector( + *builder.NewNodeSelectorTermBuilder().WithMatchExpression("some-key", + "In", "us-west").Result(), + ).Result(), + ).Result(), + wantKey: "", + wantValue: "", + }, + { + name: "Volume with multiple valid keys, returns the first match", // it should never happen + pv: builder.ForPersistentVolume("multi-matching-pv").NodeAffinityRequired( + builder.ForNodeSelector( + *builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.disk.csi.azure.com/zone", + "In", "us-central").Result(), + *builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.ebs.csi.aws.com/zone", + "In", "us-east-2c", "us-west").Result(), + *builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.ebs.csi.aws.com/zone", + "In", "unknown").Result(), + ).Result(), + ).Result(), + wantKey: "topology.disk.csi.azure.com/zone", + wantValue: "us-central", + }, + { + /* an valid example of node affinity in a GKE's regional PV + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: topology.gke.io/zone + operator: In + values: + - us-central1-a + - matchExpressions: + - key: topology.gke.io/zone + operator: In + values: + - us-central1-c + */ + name: "Volume with multiple valid keys, and provider is gke, returns all valid entries's first zone value", + pv: builder.ForPersistentVolume("multi-matching-pv").NodeAffinityRequired( + builder.ForNodeSelector( + *builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.gke.io/zone", + "In", "us-central1-c").Result(), + *builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.gke.io/zone", + "In", "us-east-2c", "us-east-2b").Result(), + *builder.NewNodeSelectorTermBuilder().WithMatchExpression("topology.gke.io/zone", + "In", "europe-north1-a").Result(), + ).Result(), + ).Result(), + wantKey: "topology.gke.io/zone", + wantValue: "us-central1-c__us-east-2c__europe-north1-a", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + k, v := zoneFromPVNodeAffinity(tt.pv, keys...) + assert.Equal(t, tt.wantKey, k) + assert.Equal(t, tt.wantValue, v) + }) + } +} diff --git a/pkg/backup/item_collector.go b/pkg/backup/item_collector.go index 6029d8f97..b3ebb9c0d 100644 --- a/pkg/backup/item_collector.go +++ b/pkg/backup/item_collector.go @@ -26,7 +26,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" - apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" @@ -209,16 +209,18 @@ func (r *itemCollector) getResourceItems(log logrus.FieldLogger, gv schema.Group } if cohabitator, found := r.cohabitatingResources[resource.Name]; found { - if cohabitator.seen { - log.WithFields( - logrus.Fields{ - "cohabitatingResource1": cohabitator.groupResource1.String(), - "cohabitatingResource2": cohabitator.groupResource2.String(), - }, - ).Infof("Skipping resource because it cohabitates and we've already processed it") - return nil, nil + if gv.Group == cohabitator.groupResource1.Group || gv.Group == cohabitator.groupResource2.Group { + if cohabitator.seen { + log.WithFields( + logrus.Fields{ + "cohabitatingResource1": cohabitator.groupResource1.String(), + "cohabitatingResource2": cohabitator.groupResource2.String(), + }, + ).Infof("Skipping resource because it cohabitates and we've already processed it") + return nil, nil + } + cohabitator.seen = true } - cohabitator.seen = true } namespacesToList := getNamespacesToList(r.backupRequest.NamespaceIncludesExcludes) @@ -293,7 +295,6 @@ func (r *itemCollector) getResourceItems(log logrus.FieldLogger, gv schema.Group if selector := r.backupRequest.Spec.LabelSelector; selector != nil { labelSelector = metav1.FormatLabelSelector(selector) } - listOptions := metav1.ListOptions{LabelSelector: labelSelector} log.Info("Listing items") unstructuredItems := make([]unstructured.Unstructured, 0) @@ -301,50 +302,42 @@ func (r *itemCollector) getResourceItems(log logrus.FieldLogger, gv schema.Group if r.pageSize > 0 { // If limit is positive, use a pager to split list over multiple requests // Use Velero's dynamic list function instead of the default - listFunc := pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) { - list, err := resourceClient.List(listOptions) - if err != nil { - return nil, err - } - return list, nil - }) - listPager := pager.New(listFunc) + listPager := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) { + return resourceClient.List(opts) + })) // Use the page size defined in the server config // TODO allow configuration of page buffer size listPager.PageSize = int64(r.pageSize) // Add each item to temporary slice - var items []unstructured.Unstructured - err := listPager.EachListItem(context.Background(), listOptions, func(object runtime.Object) error { - item, isUnstructured := object.(*unstructured.Unstructured) - if !isUnstructured { - // We should never hit this - log.Error("Got type other than Unstructured from pager func") - return nil + list, paginated, err := listPager.List(context.Background(), metav1.ListOptions{LabelSelector: labelSelector}) + if err != nil { + log.WithError(errors.WithStack(err)).Error("Error listing resources") + continue + } + if !paginated { + log.Infof("list for groupResource %s was not paginated", gr) + } + err = meta.EachListItem(list, func(object runtime.Object) error { + u, ok := object.(*unstructured.Unstructured) + if !ok { + log.WithError(errors.WithStack(fmt.Errorf("expected *unstructured.Unstructured but got %T", u))).Error("unable to understand entry in the list") + return fmt.Errorf("expected *unstructured.Unstructured but got %T", u) } - items = append(items, *item) + unstructuredItems = append(unstructuredItems, *u) return nil }) - if statusError, isStatusError := err.(*apierrors.StatusError); isStatusError && statusError.Status().Reason == metav1.StatusReasonExpired { - log.WithError(errors.WithStack(err)).Error("Error paging item list. Falling back on unpaginated list") - unstructuredList, err := resourceClient.List(listOptions) - if err != nil { - log.WithError(errors.WithStack(err)).Error("Error listing items") - continue - } - items = unstructuredList.Items - } else if err != nil { - log.WithError(errors.WithStack(err)).Error("Error paging item list") + if err != nil { + log.WithError(errors.WithStack(err)).Error("unable to understand paginated list") continue } - unstructuredItems = append(unstructuredItems, items...) } else { // If limit is not positive, do not use paging. Instead, request all items at once unstructuredList, err := resourceClient.List(metav1.ListOptions{LabelSelector: labelSelector}) - unstructuredItems = append(unstructuredItems, unstructuredList.Items...) if err != nil { log.WithError(errors.WithStack(err)).Error("Error listing items") continue } + unstructuredItems = append(unstructuredItems, unstructuredList.Items...) } log.Infof("Retrieved %d items", len(unstructuredItems)) diff --git a/pkg/backup/request.go b/pkg/backup/request.go index 0b4a63611..bc483ef97 100644 --- a/pkg/backup/request.go +++ b/pkg/backup/request.go @@ -22,6 +22,7 @@ import ( "github.com/vmware-tanzu/velero/internal/hook" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/util/collections" "github.com/vmware-tanzu/velero/pkg/volume" ) @@ -42,11 +43,11 @@ type Request struct { NamespaceIncludesExcludes *collections.IncludesExcludes ResourceIncludesExcludes *collections.IncludesExcludes ResourceHooks []hook.ResourceHook - ResolvedActions []resolvedAction - - VolumeSnapshots []*volume.Snapshot - PodVolumeBackups []*velerov1api.PodVolumeBackup - BackedUpItems map[itemKey]struct{} + ResolvedActions []framework.BackupItemResolvedAction + ResolvedItemSnapshotters []framework.ItemSnapshotterResolvedAction + VolumeSnapshots []*volume.Snapshot + PodVolumeBackups []*velerov1api.PodVolumeBackup + BackedUpItems map[itemKey]struct{} } // BackupResourceList returns the list of backed up resources grouped by the API diff --git a/pkg/builder/node_selector_builder.go b/pkg/builder/node_selector_builder.go new file mode 100644 index 000000000..11ce306b5 --- /dev/null +++ b/pkg/builder/node_selector_builder.go @@ -0,0 +1,64 @@ +package builder + +import corev1api "k8s.io/api/core/v1" + +// NodeSelectorBuilder builds NodeSelector objects +type NodeSelectorBuilder struct { + object *corev1api.NodeSelector +} + +// ForNodeSelector returns the NodeSelectorBuilder instance with given terms +func ForNodeSelector(term ...corev1api.NodeSelectorTerm) *NodeSelectorBuilder { + return &NodeSelectorBuilder{ + object: &corev1api.NodeSelector{ + NodeSelectorTerms: term, + }, + } +} + +// Result returns the built NodeSelector +func (b *NodeSelectorBuilder) Result() *corev1api.NodeSelector { + return b.object +} + +// NodeSelectorTermBuilder builds NodeSelectorTerm objects. +type NodeSelectorTermBuilder struct { + object *corev1api.NodeSelectorTerm +} + +// NewNodeSelectorTermBuilder initializes an instance of NodeSelectorTermBuilder +func NewNodeSelectorTermBuilder() *NodeSelectorTermBuilder { + return &NodeSelectorTermBuilder{ + object: &corev1api.NodeSelectorTerm{ + MatchExpressions: make([]corev1api.NodeSelectorRequirement, 0), + MatchFields: make([]corev1api.NodeSelectorRequirement, 0), + }, + } +} + +// WithMatchExpression appends the MatchExpression to the NodeSelectorTerm +func (ntb *NodeSelectorTermBuilder) WithMatchExpression(key string, op string, values ...string) *NodeSelectorTermBuilder { + req := corev1api.NodeSelectorRequirement{ + Key: key, + Operator: corev1api.NodeSelectorOperator(op), + Values: values, + } + ntb.object.MatchExpressions = append(ntb.object.MatchExpressions, req) + return ntb +} + +// WithMatchField appends the MatchField to the NodeSelectorTerm +func (ntb *NodeSelectorTermBuilder) WithMatchField(key string, op string, values ...string) *NodeSelectorTermBuilder { + req := corev1api.NodeSelectorRequirement{ + Key: key, + Operator: corev1api.NodeSelectorOperator(op), + Values: values, + } + ntb.object.MatchFields = append(ntb.object.MatchFields, req) + return ntb +} + +// Result returns the built NodeSelectorTerm +func (ntb *NodeSelectorTermBuilder) Result() *corev1api.NodeSelectorTerm { + return ntb.object +} diff --git a/pkg/builder/persistent_volume_builder.go b/pkg/builder/persistent_volume_builder.go index 648778bae..5fee88c19 100644 --- a/pkg/builder/persistent_volume_builder.go +++ b/pkg/builder/persistent_volume_builder.go @@ -94,3 +94,11 @@ func (b *PersistentVolumeBuilder) StorageClass(name string) *PersistentVolumeBui b.object.Spec.StorageClassName = name return b } + +// NodeAffinityRequired sets the PersistentVolume's NodeAffinity Requirement. +func (b *PersistentVolumeBuilder) NodeAffinityRequired(req *corev1api.NodeSelector) *PersistentVolumeBuilder { + b.object.Spec.NodeAffinity = &corev1api.VolumeNodeAffinity{ + Required: req, + } + return b +} diff --git a/pkg/builder/service_builder.go b/pkg/builder/service_builder.go new file mode 100644 index 000000000..059d2980a --- /dev/null +++ b/pkg/builder/service_builder.go @@ -0,0 +1,57 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + corev1api "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ServiceBuilder builds Service objects. +type ServiceBuilder struct { + object *corev1api.Service +} + +// ForService is the constructor for a ServiceBuilder. +func ForService(ns, name string) *ServiceBuilder { + return &ServiceBuilder{ + object: &corev1api.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1api.SchemeGroupVersion.String(), + Kind: "Service", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: name, + }, + }, + } +} + +// Result returns the built Service. +func (s *ServiceBuilder) Result() *corev1api.Service { + return s.object +} + +// ObjectMeta applies functional options to the Service's ObjectMeta. +func (s *ServiceBuilder) ObjectMeta(opts ...ObjectMetaOpt) *ServiceBuilder { + for _, opt := range opts { + opt(s.object) + } + + return s +} diff --git a/pkg/builder/statefulset_builder.go b/pkg/builder/statefulset_builder.go new file mode 100644 index 000000000..0edcd0455 --- /dev/null +++ b/pkg/builder/statefulset_builder.go @@ -0,0 +1,62 @@ +/* +Copyright 2021 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// StatefulSetBuilder builds StatefulSet objects. +type StatefulSetBuilder struct { + object *appsv1.StatefulSet +} + +// ForStatefulSet is the constructor for a StatefulSetBuilder. +func ForStatefulSet(ns, name string) *StatefulSetBuilder { + return &StatefulSetBuilder{ + object: &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: appsv1.SchemeGroupVersion.String(), + Kind: "StatefulSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: name, + }, + Spec: appsv1.StatefulSetSpec{ + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{}, + }, + }, + } +} + +// Result returns the built StatefulSet. +func (b *StatefulSetBuilder) Result() *appsv1.StatefulSet { + return b.object +} + +// StorageClass sets the StatefulSet's VolumeClaimTemplates storage class name. +func (b *StatefulSetBuilder) StorageClass(names ...string) *StatefulSetBuilder { + for _, name := range names { + nameTmp := name + b.object.Spec.VolumeClaimTemplates = append(b.object.Spec.VolumeClaimTemplates, + corev1.PersistentVolumeClaim{Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: &nameTmp}}) + } + return b +} diff --git a/pkg/builder/storage_class_builder.go b/pkg/builder/storage_class_builder.go index 994543332..8ffe4afd1 100644 --- a/pkg/builder/storage_class_builder.go +++ b/pkg/builder/storage_class_builder.go @@ -23,7 +23,8 @@ import ( // StorageClassBuilder builds StorageClass objects. type StorageClassBuilder struct { - object *storagev1api.StorageClass + object *storagev1api.StorageClass + objectSlice []*storagev1api.StorageClass } // ForStorageClass is the constructor for a StorageClassBuilder. @@ -54,3 +55,29 @@ func (b *StorageClassBuilder) ObjectMeta(opts ...ObjectMetaOpt) *StorageClassBui return b } + +// ForStorageClassSlice is the constructor for a storageClassSlice in StorageClassBuilder. +func ForStorageClassSlice(names ...string) *StorageClassBuilder { + var storageClassSlice []*storagev1api.StorageClass + for _, name := range names { + storageClass := &storagev1api.StorageClass{ + TypeMeta: metav1.TypeMeta{ + APIVersion: storagev1api.SchemeGroupVersion.String(), + Kind: "StorageClass", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + storageClassSlice = append(storageClassSlice, storageClass) + } + + return &StorageClassBuilder{ + objectSlice: storageClassSlice, + } +} + +// SliceResult returns the built StorageClass slice. +func (b *StorageClassBuilder) SliceResult() []*storagev1api.StorageClass { + return b.objectSlice +} diff --git a/pkg/builder/testcr_builder.go b/pkg/builder/testcr_builder.go new file mode 100644 index 000000000..85bc83b1c --- /dev/null +++ b/pkg/builder/testcr_builder.go @@ -0,0 +1,77 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" +) + +// CustomResourceBuilder builds objects based on velero APIVersion CRDs. +type TestCRBuilder struct { + object *TestCR +} + +// ForTestCR is the constructor for a TestCRBuilder. +func ForTestCR(crdKind, ns, name string) *TestCRBuilder { + return &TestCRBuilder{ + object: &TestCR{ + TypeMeta: metav1.TypeMeta{ + APIVersion: velerov1api.SchemeGroupVersion.String(), + Kind: crdKind, + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: name, + }, + }, + } +} + +// Result returns the built TestCR. +func (b *TestCRBuilder) Result() *TestCR { + return b.object +} + +// ObjectMeta applies functional options to the TestCR's ObjectMeta. +func (b *TestCRBuilder) ObjectMeta(opts ...ObjectMetaOpt) *TestCRBuilder { + for _, opt := range opts { + opt(b.object) + } + + return b +} + +type TestCR struct { + metav1.TypeMeta `json:",inline"` + + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // +optional + Spec TestCRSpec `json:"spec,omitempty"` + + // +optional + Status TestCRStatus `json:"status,omitempty"` +} + +type TestCRSpec struct { +} + +type TestCRStatus struct { +} diff --git a/pkg/cmd/cli/backup/create.go b/pkg/cmd/cli/backup/create.go index b4e391314..f2d165df5 100644 --- a/pkg/cmd/cli/backup/create.go +++ b/pkg/cmd/cli/backup/create.go @@ -1,5 +1,5 @@ /* -Copyright 2020 the Velero contributors. +Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -27,6 +27,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeerrs "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/tools/cache" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -37,6 +38,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/cmd/util/output" veleroclient "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned" v1 "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions/velero/v1" + "github.com/vmware-tanzu/velero/pkg/util/collections" ) const DefaultBackupTTL time.Duration = 30 * 24 * time.Hour @@ -162,6 +164,11 @@ func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Facto return fmt.Errorf("A backup name is required, unless you are creating based on a schedule.") } + errs := collections.ValidateNamespaceIncludesExcludes(o.IncludeNamespaces, o.ExcludeNamespaces) + if len(errs) > 0 { + return kubeerrs.NewAggregate(errs) + } + if o.StorageLocation != "" { location := &velerov1api.BackupStorageLocation{} if err := client.Get(context.Background(), kbclient.ObjectKey{ @@ -284,11 +291,11 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { return nil } -// parseOrderedResources converts to map of Kinds to an ordered list of specific resources of that Kind. +// ParseOrderedResources converts to map of Kinds to an ordered list of specific resources of that Kind. // Resource names in the list are in format 'namespace/resourcename' and separated by commas. // Key-value pairs in the mapping are separated by semi-colon. // Ex: 'pods=ns1/pod1,ns1/pod2;persistentvolumeclaims=ns1/pvc4,ns1/pvc8'. -func parseOrderedResources(orderMapStr string) (map[string]string, error) { +func ParseOrderedResources(orderMapStr string) (map[string]string, error) { entries := strings.Split(orderMapStr, ";") if len(entries) == 0 { return nil, fmt.Errorf("Invalid OrderedResources '%s'.", orderMapStr) @@ -315,7 +322,7 @@ func (o *CreateOptions) BuildBackup(namespace string) (*velerov1api.Backup, erro return nil, err } if o.Name == "" { - o.Name = schedule.TimestampedName(time.Now()) + o.Name = schedule.TimestampedName(time.Now().UTC()) } backupBuilder = builder.ForBackup(namespace, o.Name). FromSchedule(schedule) @@ -330,7 +337,7 @@ func (o *CreateOptions) BuildBackup(namespace string) (*velerov1api.Backup, erro StorageLocation(o.StorageLocation). VolumeSnapshotLocations(o.SnapshotLocations...) if len(o.OrderedResources) > 0 { - orders, err := parseOrderedResources(o.OrderedResources) + orders, err := ParseOrderedResources(o.OrderedResources) if err != nil { return nil, err } diff --git a/pkg/cmd/cli/backup/create_test.go b/pkg/cmd/cli/backup/create_test.go index 09bffd81e..401e93e47 100644 --- a/pkg/cmd/cli/backup/create_test.go +++ b/pkg/cmd/cli/backup/create_test.go @@ -1,5 +1,5 @@ /* -Copyright 2020 the Velero contributors. +Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -34,7 +34,7 @@ func TestCreateOptions_BuildBackup(t *testing.T) { o := NewCreateOptions() o.Labels.Set("velero.io/test=true") o.OrderedResources = "pods=p1,p2;persistentvolumeclaims=pvc1,pvc2" - orders, err := parseOrderedResources(o.OrderedResources) + orders, err := ParseOrderedResources(o.OrderedResources) assert.NoError(t, err) backup, err := o.BuildBackup(testNamespace) @@ -100,10 +100,10 @@ func TestCreateOptions_BuildBackupFromSchedule(t *testing.T) { } func TestCreateOptions_OrderedResources(t *testing.T) { - orderedResources, err := parseOrderedResources("pods= ns1/p1; ns1/p2; persistentvolumeclaims=ns2/pvc1, ns2/pvc2") + orderedResources, err := ParseOrderedResources("pods= ns1/p1; ns1/p2; persistentvolumeclaims=ns2/pvc1, ns2/pvc2") assert.NotNil(t, err) - orderedResources, err = parseOrderedResources("pods= ns1/p1,ns1/p2 ; persistentvolumeclaims=ns2/pvc1,ns2/pvc2") + orderedResources, err = ParseOrderedResources("pods= ns1/p1,ns1/p2 ; persistentvolumeclaims=ns2/pvc1,ns2/pvc2") assert.NoError(t, err) expectedResources := map[string]string{ @@ -112,7 +112,7 @@ func TestCreateOptions_OrderedResources(t *testing.T) { } assert.Equal(t, orderedResources, expectedResources) - orderedResources, err = parseOrderedResources("pods= ns1/p1,ns1/p2 ; persistentvolumes=pv1,pv2") + orderedResources, err = ParseOrderedResources("pods= ns1/p1,ns1/p2 ; persistentvolumes=pv1,pv2") assert.NoError(t, err) expectedMixedResources := map[string]string{ diff --git a/pkg/cmd/cli/backuplocation/create.go b/pkg/cmd/cli/backuplocation/create.go index a541d7589..a90564be3 100644 --- a/pkg/cmd/cli/backuplocation/create.go +++ b/pkg/cmd/cli/backuplocation/create.go @@ -1,5 +1,5 @@ /* -Copyright 2020 the Velero contributors. +Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -78,6 +78,7 @@ func NewCreateOptions() *CreateOptions { return &CreateOptions{ Credential: flag.NewMap(), Config: flag.NewMap(), + Labels: flag.NewMap(), AccessMode: flag.NewEnum( string(velerov1api.BackupStorageLocationAccessModeReadWrite), string(velerov1api.BackupStorageLocationAccessModeReadWrite), @@ -133,39 +134,22 @@ func (o *CreateOptions) Complete(args []string, f client.Factory) error { return nil } -func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { - var backupSyncPeriod, validationFrequency *metav1.Duration - +func (o *CreateOptions) BuildBackupStorageLocation(namespace string, setBackupSyncPeriod, setValidationFrequency bool) (*velerov1api.BackupStorageLocation, error) { var caCertData []byte if o.CACertFile != "" { realPath, err := filepath.Abs(o.CACertFile) if err != nil { - return err + return nil, err } caCertData, err = ioutil.ReadFile(realPath) if err != nil { - return err + return nil, err } } - if c.Flags().Changed("backup-sync-period") { - backupSyncPeriod = &metav1.Duration{Duration: o.BackupSyncPeriod} - } - - if c.Flags().Changed("validation-frequency") { - validationFrequency = &metav1.Duration{Duration: o.ValidationFrequency} - } - - var secretName, secretKey string - for k, v := range o.Credential.Data() { - secretName = k - secretKey = v - break - } - backupStorageLocation := &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ - Namespace: f.Namespace(), + Namespace: namespace, Name: o.Name, Labels: o.Labels.Data(), }, @@ -178,15 +162,37 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { CACert: caCertData, }, }, - Config: o.Config.Data(), - Credential: builder.ForSecretKeySelector(secretName, secretKey).Result(), - Default: o.DefaultBackupStorageLocation, - AccessMode: velerov1api.BackupStorageLocationAccessMode(o.AccessMode.String()), - BackupSyncPeriod: backupSyncPeriod, - ValidationFrequency: validationFrequency, + Config: o.Config.Data(), + Default: o.DefaultBackupStorageLocation, + AccessMode: velerov1api.BackupStorageLocationAccessMode(o.AccessMode.String()), }, } + if setBackupSyncPeriod { + backupStorageLocation.Spec.BackupSyncPeriod = &metav1.Duration{Duration: o.BackupSyncPeriod} + } + + if setValidationFrequency { + backupStorageLocation.Spec.ValidationFrequency = &metav1.Duration{Duration: o.ValidationFrequency} + } + + for secretName, secretKey := range o.Credential.Data() { + backupStorageLocation.Spec.Credential = builder.ForSecretKeySelector(secretName, secretKey).Result() + break + } + + return backupStorageLocation, nil +} + +func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { + setBackupSyncPeriod := c.Flags().Changed("backup-sync-period") + setValidationFrequency := c.Flags().Changed("validation-frequency") + + backupStorageLocation, err := o.BuildBackupStorageLocation(f.Namespace(), setBackupSyncPeriod, setValidationFrequency) + if err != nil { + return err + } + if printed, err := output.PrintWithFormat(c, backupStorageLocation); printed || err != nil { return err } diff --git a/pkg/cmd/cli/backuplocation/create_test.go b/pkg/cmd/cli/backuplocation/create_test.go new file mode 100644 index 000000000..e09d45271 --- /dev/null +++ b/pkg/cmd/cli/backuplocation/create_test.go @@ -0,0 +1,89 @@ +/* +Copyright the Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package backuplocation + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestBuildBackupStorageLocationSetsNamespace(t *testing.T) { + o := NewCreateOptions() + + bsl, err := o.BuildBackupStorageLocation("velero-test-ns", false, false) + assert.NoError(t, err) + assert.Equal(t, "velero-test-ns", bsl.Namespace) +} + +func TestBuildBackupStorageLocationSetsSyncPeriod(t *testing.T) { + o := NewCreateOptions() + o.BackupSyncPeriod = 2 * time.Minute + + bsl, err := o.BuildBackupStorageLocation("velero-test-ns", false, false) + assert.NoError(t, err) + assert.Nil(t, bsl.Spec.BackupSyncPeriod) + + bsl, err = o.BuildBackupStorageLocation("velero-test-ns", true, false) + assert.NoError(t, err) + assert.Equal(t, &metav1.Duration{Duration: 2 * time.Minute}, bsl.Spec.BackupSyncPeriod) +} + +func TestBuildBackupStorageLocationSetsValidationFrequency(t *testing.T) { + o := NewCreateOptions() + o.ValidationFrequency = 2 * time.Minute + + bsl, err := o.BuildBackupStorageLocation("velero-test-ns", false, false) + assert.NoError(t, err) + assert.Nil(t, bsl.Spec.ValidationFrequency) + + bsl, err = o.BuildBackupStorageLocation("velero-test-ns", false, true) + assert.NoError(t, err) + assert.Equal(t, &metav1.Duration{Duration: 2 * time.Minute}, bsl.Spec.ValidationFrequency) +} + +func TestBuildBackupStorageLocationSetsCredential(t *testing.T) { + o := NewCreateOptions() + + bsl, err := o.BuildBackupStorageLocation("velero-test-ns", false, false) + assert.NoError(t, err) + assert.Nil(t, bsl.Spec.Credential) + + setErr := o.Credential.Set("my-secret=key-from-secret") + assert.NoError(t, setErr) + + bsl, err = o.BuildBackupStorageLocation("velero-test-ns", false, true) + assert.NoError(t, err) + assert.Equal(t, &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{Name: "my-secret"}, + Key: "key-from-secret", + }, bsl.Spec.Credential) +} + +func TestBuildBackupStorageLocationSetsLabels(t *testing.T) { + o := NewCreateOptions() + + err := o.Labels.Set("key=value") + assert.NoError(t, err) + + bsl, err := o.BuildBackupStorageLocation("velero-test-ns", false, false) + assert.NoError(t, err) + assert.Equal(t, map[string]string{"key": "value"}, bsl.Labels) +} diff --git a/pkg/cmd/cli/backuplocation/delete.go b/pkg/cmd/cli/backuplocation/delete.go index eecd08f47..daedf77cd 100644 --- a/pkg/cmd/cli/backuplocation/delete.go +++ b/pkg/cmd/cli/backuplocation/delete.go @@ -1,5 +1,5 @@ /* -Copyright 2020 the Velero contributors. +Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,7 +22,6 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" kubeerrs "k8s.io/apimachinery/pkg/util/errors" @@ -34,6 +33,8 @@ import ( "github.com/vmware-tanzu/velero/pkg/cmd/cli" ) +const bslLabelKey = "velero.io/storage-location" + // NewDeleteCommand creates and returns a new cobra command for deleting backup-locations. func NewDeleteCommand(f client.Factory, use string) *cobra.Command { o := cli.NewDeleteOptions("backup-location") @@ -120,7 +121,65 @@ func Run(f client.Factory, o *cli.DeleteOptions) error { continue } fmt.Printf("Backup storage location %q deleted successfully.\n", location.Name) + + // Delete backups associated with the deleted BSL. + backupList, err := findAssociatedBackups(kbClient, location.Name, f.Namespace()) + if err != nil { + errs = append(errs, fmt.Errorf("find backups associated with BSL %q: %w", location.Name, err)) + } else if deleteErrs := deleteBackups(kbClient, backupList); deleteErrs != nil { + errs = append(errs, deleteErrs...) + } + + // Delete Restic repositories associated with the deleted BSL. + resticRepoList, err := findAssociatedResticRepos(kbClient, location.Name, f.Namespace()) + if err != nil { + errs = append(errs, fmt.Errorf("find Restic repositories associated with BSL %q: %w", location.Name, err)) + } else if deleteErrs := deleteResticRepos(kbClient, resticRepoList); deleteErrs != nil { + errs = append(errs, deleteErrs...) + } } return kubeerrs.NewAggregate(errs) } + +func findAssociatedBackups(client kbclient.Client, bslName, ns string) (velerov1api.BackupList, error) { + var backups velerov1api.BackupList + err := client.List(context.Background(), &backups, &kbclient.ListOptions{ + Namespace: ns, + Raw: &metav1.ListOptions{LabelSelector: bslLabelKey + "=" + bslName}, + }) + return backups, err +} + +func findAssociatedResticRepos(client kbclient.Client, bslName, ns string) (velerov1api.ResticRepositoryList, error) { + var repos velerov1api.ResticRepositoryList + err := client.List(context.Background(), &repos, &kbclient.ListOptions{ + Namespace: ns, + Raw: &metav1.ListOptions{LabelSelector: bslLabelKey + "=" + bslName}, + }) + return repos, err +} + +func deleteBackups(client kbclient.Client, backups velerov1api.BackupList) []error { + var errs []error + for _, backup := range backups.Items { + if err := client.Delete(context.Background(), &backup, &kbclient.DeleteOptions{}); err != nil { + errs = append(errs, errors.WithStack(fmt.Errorf("delete backup %q associated with deleted BSL: %w", backup.Name, err))) + continue + } + fmt.Printf("Backup associated with deleted BSL(s) %q deleted successfully.\n", backup.Name) + } + return errs +} + +func deleteResticRepos(client kbclient.Client, repos velerov1api.ResticRepositoryList) []error { + var errs []error + for _, repo := range repos.Items { + if err := client.Delete(context.Background(), &repo, &kbclient.DeleteOptions{}); err != nil { + errs = append(errs, errors.WithStack(fmt.Errorf("delete Restic repository %q associated with deleted BSL: %w", repo.Name, err))) + continue + } + fmt.Printf("Restic repository associated with deleted BSL(s) %q deleted successfully.\n", repo.Name) + } + return errs +} diff --git a/pkg/cmd/cli/bug/bug.go b/pkg/cmd/cli/bug/bug.go index e5d0952aa..ea775ec31 100644 --- a/pkg/cmd/cli/bug/bug.go +++ b/pkg/cmd/cli/bug/bug.go @@ -55,10 +55,15 @@ about: Tell us about a problem you are experiencing **What did you expect to happen:** +**The following information will help us better understand what's going on**: -**The output of the following commands will help us better understand what's going on**: -(Pasting long output into a [GitHub gist](https://gist.github.com) or other pastebin is fine.) +_If you are using velero v1.7.0+:_ +Please use ` + "`velero debug --backup --restore ` " + + `to generate the support bundle, and attach to this issue, more options please refer to ` + + "`velero debug --help` " + ` +_If you are using earlier versions:_ +Please provide the output of the following commands (Pasting long output into a [GitHub gist](https://gist.github.com) or other pastebin is fine.) - ` + "`kubectl logs deployment/velero -n velero`" + ` - ` + "`velero backup describe ` or `kubectl get backup/ -n velero -o yaml`" + ` - ` + "`velero backup logs `" + ` diff --git a/pkg/cmd/cli/debug/cshd-scripts/velero.cshd b/pkg/cmd/cli/debug/cshd-scripts/velero.cshd index c871b27de..29e27f753 100644 --- a/pkg/cmd/cli/debug/cshd-scripts/velero.cshd +++ b/pkg/cmd/cli/debug/cshd-scripts/velero.cshd @@ -1,26 +1,31 @@ -def capture_backup_logs(namespace): +def capture_backup_logs(cmd, namespace): if args.backup: - log("Collecting log for backup: {}".format(args.backup)) - backupLogsCmd = "velero --namespace={} backup logs {}".format(namespace, args.backup) + log("Collecting log and information for backup: {}".format(args.backup)) + backupDescCmd = "{} --namespace={} backup describe {} --details".format(cmd, namespace, args.backup) + capture_local(cmd=backupDescCmd, file_name="backup_describe_{}.txt".format(args.backup)) + backupLogsCmd = "{} --namespace={} backup logs {}".format(cmd, namespace, args.backup) capture_local(cmd=backupLogsCmd, file_name="backup_{}.log".format(args.backup)) -def capture_restore_logs(namespace): +def capture_restore_logs(cmd, namespace): if args.restore: - log("Collecting log for restore: {}".format(args.restore)) - restoreLogsCmd = "velero --namespace={} restore logs {}".format(namespace, args.restore) + log("Collecting log and information for restore: {}".format(args.restore)) + restoreDescCmd = "{} --namespace={} restore describe {} --details".format(cmd, namespace, args.restore) + capture_local(cmd=restoreDescCmd, file_name="restore_describe_{}.txt".format(args.restore)) + restoreLogsCmd = "{} --namespace={} restore logs {}".format(cmd, namespace, args.restore) capture_local(cmd=restoreLogsCmd, file_name="restore_{}.log".format(args.restore)) ns = args.namespace if args.namespace else "velero" output = args.output if args.output else "bundle.tar.gz" +cmd = args.cmd if args.cmd else "velero" # Working dir for writing during script execution crshd = crashd_config(workdir="./velero-bundle") set_defaults(kube_config(path=args.kubeconfig, cluster_context=args.kubecontext)) log("Collecting velero resources in namespace: {}". format(ns)) kube_capture(what="objects", namespaces=[ns], groups=['velero.io']) -capture_local(cmd="velero version -n {}".format(ns), file_name="version.txt") +capture_local(cmd="{} version -n {}".format(cmd, ns), file_name="version.txt") log("Collecting velero deployment logs in namespace: {}". format(ns)) kube_capture(what="logs", namespaces=[ns]) -capture_backup_logs(ns) -capture_restore_logs(ns) +capture_backup_logs(cmd, ns) +capture_restore_logs(cmd, ns) archive(output_file=output, source_paths=[crshd.workdir]) log("Generated debug information bundle: {}".format(output)) diff --git a/pkg/cmd/cli/debug/debug.go b/pkg/cmd/cli/debug/debug.go index 913b0dfa7..e324f2631 100644 --- a/pkg/cmd/cli/debug/debug.go +++ b/pkg/cmd/cli/debug/debug.go @@ -42,6 +42,8 @@ import ( var scriptBytes []byte type option struct { + // currCmd the velero command + currCmd string // workdir for crashd will be $baseDir/velero-debug baseDir string // the namespace where velero server is installed @@ -74,6 +76,7 @@ func (o *option) asCrashdArgs() string { func (o *option) asCrashdArgMap() exec.ArgMap { return exec.ArgMap{ + "cmd": o.currCmd, "output": o.outputPath, "namespace": o.namespace, "basedir": o.baseDir, @@ -100,6 +103,10 @@ func (o *option) complete(f client.Factory, fs *pflag.FlagSet) error { o.baseDir = tmpDir o.namespace = f.Namespace() kp, kc := kubeconfigAndContext(fs) + o.currCmd, err = os.Executable() + if err != nil { + return err + } o.kubeconfigPath, err = filepath.Abs(kp) if err != nil { return fmt.Errorf("invalid kubeconfig path: %s, %v", kp, err) diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index f14c8725c..5b4941dff 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -25,11 +25,9 @@ import ( "time" "github.com/pkg/errors" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" "github.com/vmware-tanzu/velero/internal/velero" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -37,7 +35,6 @@ import ( "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" - velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/install" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" ) @@ -72,7 +69,6 @@ type InstallOptions struct { Plugins flag.StringArray NoDefaultBackupLocation bool CRDsOnly bool - CRDsVersion string CACertFile string Features string DefaultVolumesToRestic bool @@ -107,7 +103,6 @@ func (o *InstallOptions) BindFlags(flags *pflag.FlagSet) { flags.DurationVar(&o.DefaultResticMaintenanceFrequency, "default-restic-prune-frequency", o.DefaultResticMaintenanceFrequency, "How often 'restic prune' is run for restic repositories by default. Optional.") flags.Var(&o.Plugins, "plugins", "Plugin container images to install into the Velero Deployment") flags.BoolVar(&o.CRDsOnly, "crds-only", o.CRDsOnly, "Only generate CustomResourceDefinition resources. Useful for updating CRDs for an existing Velero install.") - flags.StringVar(&o.CRDsVersion, "crds-version", o.CRDsVersion, "The version to generate CustomResourceDefinition resources if Velero can't discover the Kubernetes preferred CRD API version. Optional.") flags.StringVar(&o.CACertFile, "cacert", o.CACertFile, "File containing a certificate bundle to use when verifying TLS connections to the object store. Optional.") flags.StringVar(&o.Features, "features", o.Features, "Comma separated list of Velero feature flags to be set on the Velero deployment and the restic daemonset, if restic is enabled") flags.BoolVar(&o.DefaultVolumesToRestic, "default-volumes-to-restic", o.DefaultVolumesToRestic, "Bool flag to configure Velero server to use restic by default to backup all pod volumes on all backups. Optional.") @@ -134,7 +129,6 @@ func NewInstallOptions() *InstallOptions { UseVolumeSnapshots: true, NoDefaultBackupLocation: false, CRDsOnly: false, - CRDsVersion: "v1", DefaultVolumesToRestic: false, } } @@ -193,7 +187,6 @@ func (o *InstallOptions) AsVeleroOptions() (*install.VeleroOptions, error) { NoDefaultBackupLocation: o.NoDefaultBackupLocation, CACertData: caCertData, Features: strings.Split(o.Features, ","), - CRDsVersion: o.CRDsVersion, DefaultVolumesToRestic: o.DefaultVolumesToRestic, }, nil } @@ -254,30 +247,9 @@ This is useful as a starting point for more customized installations. // Run executes a command in the context of the provided arguments. func (o *InstallOptions) Run(c *cobra.Command, f client.Factory) error { - // Find the kube-apiserver group apiextensions.k8s.io preferred API version - clientset, err := f.KubeClient() - if err == nil { - // kubeconfig available - discoveryHelper, err := velerodiscovery.NewHelper(clientset.Discovery(), &logrus.Logger{}) - if err == nil { - // kubernetes apiserver available - gvr, _, err := discoveryHelper.ResourceFor( - schema.GroupVersionResource{ - Group: "apiextensions.k8s.io", - Resource: "customresourcedefinitions", - }) - if err != nil { - return err - } - - // Update the group apiextensions.k8s.io preferred API version - o.CRDsVersion = gvr.Version - } - } - var resources *unstructured.UnstructuredList if o.CRDsOnly { - resources = install.AllCRDs(o.CRDsVersion) + resources = install.AllCRDs() } else { vo, err := o.AsVeleroOptions() if err != nil { @@ -348,11 +320,6 @@ func (o *InstallOptions) Validate(c *cobra.Command, args []string, f client.Fact return err } - // Check the CRD version is valid. - if o.CRDsVersion != "v1beta1" && o.CRDsVersion != "v1" { - return errors.Errorf("CRD version must be v1beta1 or v1") - } - // If we're only installing CRDs, we can skip the rest of the validation. if o.CRDsOnly { return nil diff --git a/pkg/cmd/cli/restic/server.go b/pkg/cmd/cli/restic/server.go index 4de389bf9..5f8fd526b 100644 --- a/pkg/cmd/cli/restic/server.go +++ b/pkg/cmd/cli/restic/server.go @@ -32,6 +32,7 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" v1 "k8s.io/api/core/v1" + storagev1api "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" @@ -151,6 +152,7 @@ func newResticServer(logger logrus.FieldLogger, factory client.Factory, metricAd velerov1api.AddToScheme(scheme) v1.AddToScheme(scheme) + storagev1api.AddToScheme(scheme) mgr, err := ctrl.NewManager(clientConfig, ctrl.Options{ Scheme: scheme, }) diff --git a/pkg/cmd/cli/schedule/create.go b/pkg/cmd/cli/schedule/create.go index 7a4971aa7..5e15ac6bc 100644 --- a/pkg/cmd/cli/schedule/create.go +++ b/pkg/cmd/cli/schedule/create.go @@ -111,11 +111,20 @@ func (o *CreateOptions) Complete(args []string, f client.Factory) error { } func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { + var orders map[string]string + veleroClient, err := f.Client() if err != nil { return err } + if len(o.BackupOptions.OrderedResources) > 0 { + orders, err = backup.ParseOrderedResources(o.BackupOptions.OrderedResources) + if err != nil { + return err + } + } + schedule := &api.Schedule{ ObjectMeta: metav1.ObjectMeta{ Namespace: f.Namespace(), @@ -135,6 +144,7 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { StorageLocation: o.BackupOptions.StorageLocation, VolumeSnapshotLocations: o.BackupOptions.SnapshotLocations, DefaultVolumesToRestic: o.BackupOptions.DefaultVolumesToRestic.Value, + OrderedResources: orders, }, Schedule: o.Schedule, UseOwnerReferencesInBackup: &o.UseOwnerReferencesInBackup, diff --git a/pkg/cmd/server/plugin/plugin.go b/pkg/cmd/server/plugin/plugin.go index 1a79b6dd6..507200266 100644 --- a/pkg/cmd/server/plugin/plugin.go +++ b/pkg/cmd/server/plugin/plugin.go @@ -55,6 +55,7 @@ func NewCommand(f client.Factory) *cobra.Command { RegisterRestoreItemAction("velero.io/crd-preserve-fields", newCRDV1PreserveUnknownFieldsItemAction). RegisterRestoreItemAction("velero.io/change-pvc-node-selector", newChangePVCNodeSelectorItemAction(f)). RegisterRestoreItemAction("velero.io/apiservice", newAPIServiceRestoreItemAction). + RegisterRestoreItemAction("velero.io/admission-webhook-configuration", newAdmissionWebhookConfigurationAction). Serve() }, } @@ -202,3 +203,7 @@ func newChangePVCNodeSelectorItemAction(f client.Factory) veleroplugin.HandlerIn func newAPIServiceRestoreItemAction(logger logrus.FieldLogger) (interface{}, error) { return restore.NewAPIServiceAction(logger), nil } + +func newAdmissionWebhookConfigurationAction(logger logrus.FieldLogger) (interface{}, error) { + return restore.NewAdmissionWebhookConfigurationAction(logger), nil +} diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index e0ac43a01..5a00648c4 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -27,6 +27,7 @@ import ( "strings" "time" + "github.com/bombsimon/logrusr" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" @@ -302,8 +303,11 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s velerov1api.AddToScheme(scheme) corev1api.AddToScheme(scheme) + ctrl.SetLogger(logrusr.NewLogger(logger)) + mgr, err := ctrl.NewManager(clientConfig, ctrl.Options{ - Scheme: scheme, + Scheme: scheme, + Namespace: f.Namespace(), }) if err != nil { cancelFunc() diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 8fdcb332e..6adfeeec7 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -1,5 +1,5 @@ /* -Copyright the Velero contributors. +Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -53,6 +53,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" + "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/collections" "github.com/vmware-tanzu/velero/pkg/util/encode" @@ -424,7 +425,7 @@ func (c *backupController) prepareBackupRequest(backup *velerov1api.Backup) *pkg } // validate the included/excluded namespaces - for _, err := range collections.ValidateIncludesExcludes(request.Spec.IncludedNamespaces, request.Spec.ExcludedNamespaces) { + for _, err := range collections.ValidateNamespaceIncludesExcludes(request.Spec.IncludedNamespaces, request.Spec.ExcludedNamespaces) { request.Status.ValidationErrors = append(request.Status.ValidationErrors, fmt.Sprintf("Invalid included/excluded namespace lists: %v", err)) } @@ -569,6 +570,10 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { if err != nil { return err } + itemSnapshotters, err := pluginManager.GetItemSnapshotters() + if err != nil { + return err + } backupLog.Info("Setting up backup store to check for backup existence") backupStore, err := c.backupStoreGetter.Get(backup.StorageLocation, pluginManager, backupLog) @@ -586,8 +591,12 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { return errors.Errorf("backup already exists in object storage") } + backupItemActionsResolver := framework.NewBackupItemActionResolver(actions) + itemSnapshottersResolver := framework.NewItemSnapshotterResolver(itemSnapshotters) + var fatalErrs []error - if err := c.backupper.Backup(backupLog, backup, backupFile, actions, pluginManager); err != nil { + if err := c.backupper.BackupWithResolvers(backupLog, backup, backupFile, backupItemActionsResolver, + itemSnapshottersResolver, pluginManager); err != nil { fatalErrs = append(fatalErrs, err) } diff --git a/pkg/controller/backup_controller_test.go b/pkg/controller/backup_controller_test.go index 16fc6c1a0..85c581a81 100644 --- a/pkg/controller/backup_controller_test.go +++ b/pkg/controller/backup_controller_test.go @@ -47,6 +47,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/persistence" persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" + "github.com/vmware-tanzu/velero/pkg/plugin/framework" pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" @@ -63,6 +64,13 @@ func (b *fakeBackupper) Backup(logger logrus.FieldLogger, backup *pkgbackup.Requ return args.Error(0) } +func (b *fakeBackupper) BackupWithResolvers(logger logrus.FieldLogger, backup *pkgbackup.Request, backupFile io.Writer, + backupItemActionResolver framework.BackupItemActionResolver, itemSnapshotterResolver framework.ItemSnapshotterResolver, + volumeSnapshotterGetter pkgbackup.VolumeSnapshotterGetter) error { + args := b.Called(logger, backup, backupFile, backupItemActionResolver, itemSnapshotterResolver, volumeSnapshotterGetter) + return args.Error(0) +} + func defaultBackup() *builder.BackupBuilder { return builder.ForBackup(velerov1api.DefaultNamespace, "backup-1") } @@ -825,7 +833,9 @@ func TestProcessBackupCompletions(t *testing.T) { pluginManager.On("GetBackupItemActions").Return(nil, nil) pluginManager.On("CleanupClients").Return(nil) + pluginManager.On("GetItemSnapshotters").Return(nil, nil) backupper.On("Backup", mock.Anything, mock.Anything, mock.Anything, []velero.BackupItemAction(nil), pluginManager).Return(nil) + backupper.On("BackupWithResolvers", mock.Anything, mock.Anything, mock.Anything, framework.BackupItemActionResolver{}, framework.ItemSnapshotterResolver{}, pluginManager).Return(nil) backupStore.On("BackupExists", test.backupLocation.Spec.StorageType.ObjectStorage.Bucket, test.backup.Name).Return(test.backupExists, test.existenceCheckError) // Ensure we have a CompletionTimestamp when uploading and that the backup name matches the backup in the object store. diff --git a/pkg/controller/backup_deletion_controller.go b/pkg/controller/backup_deletion_controller.go index 3e503e81a..90707be70 100644 --- a/pkg/controller/backup_deletion_controller.go +++ b/pkg/controller/backup_deletion_controller.go @@ -291,7 +291,7 @@ func (c *backupDeletionController) processRequest(req *velerov1api.DeleteBackupR backupStore, err := c.backupStoreGetter.Get(location, pluginManager, log) if err != nil { - errs = append(errs, err.Error()) + return errors.Wrap(err, "error getting the backup store") } actions, err := pluginManager.GetDeleteItemActions() diff --git a/pkg/controller/backup_deletion_controller_test.go b/pkg/controller/backup_deletion_controller_test.go index 388df8ae3..a03e98eb0 100644 --- a/pkg/controller/backup_deletion_controller_test.go +++ b/pkg/controller/backup_deletion_controller_test.go @@ -1,5 +1,5 @@ /* -Copyright the Velero contributors. +Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import ( "context" "fmt" "io/ioutil" + "strings" "testing" "time" @@ -36,19 +37,20 @@ import ( core "k8s.io/client-go/testing" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/vmware-tanzu/velero/pkg/builder" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks" + "github.com/vmware-tanzu/velero/pkg/volume" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" - "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/fake" informers "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions" "github.com/vmware-tanzu/velero/pkg/metrics" persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" - "github.com/vmware-tanzu/velero/pkg/plugin/velero" - "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks" velerotest "github.com/vmware-tanzu/velero/pkg/test" - "github.com/vmware-tanzu/velero/pkg/volume" ) func TestBackupDeletionControllerProcessQueueItem(t *testing.T) { @@ -183,6 +185,29 @@ func setupBackupDeletionControllerTest(t *testing.T, objects ...runtime.Object) } func TestBackupDeletionControllerProcessRequest(t *testing.T) { + t.Run("failed to get backup store", func(t *testing.T) { + backup := builder.ForBackup(velerov1api.DefaultNamespace, "foo").StorageLocation("default").Result() + location := &velerov1api.BackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: backup.Namespace, + Name: backup.Spec.StorageLocation, + }, + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "objStoreProvider", + StorageType: velerov1api.StorageType{ + ObjectStorage: &velerov1api.ObjectStorageLocation{ + Bucket: "bucket", + }, + }, + }, + } + td := setupBackupDeletionControllerTest(t, location, backup) + td.controller.backupStoreGetter = &fakeErrorBackupStoreGetter{} + err := td.controller.processRequest(td.req) + assert.NotNil(t, err) + assert.True(t, strings.HasPrefix(err.Error(), "error getting the backup store")) + }) + t.Run("missing spec.backupName", func(t *testing.T) { td := setupBackupDeletionControllerTest(t) td.req.Spec.BackupName = "" diff --git a/pkg/controller/backup_sync_controller.go b/pkg/controller/backup_sync_controller.go index e4c8ff022..aeb848ac0 100644 --- a/pkg/controller/backup_sync_controller.go +++ b/pkg/controller/backup_sync_controller.go @@ -312,7 +312,7 @@ func (c *backupSyncController) run() { c.deleteOrphanedBackups(location.Name, backupStoreBackups, log) // update the location's last-synced time field - statusPatch := client.MergeFrom(location.DeepCopyObject()) + statusPatch := client.MergeFrom(location.DeepCopy()) location.Status.LastSyncedTime = &metav1.Time{Time: time.Now().UTC()} if err := c.kbClient.Status().Patch(context.Background(), &location, statusPatch); err != nil { log.WithError(errors.WithStack(err)).Error("Error patching backup location's last-synced time") diff --git a/pkg/controller/pod_volume_backup_controller.go b/pkg/controller/pod_volume_backup_controller.go index ef9c8e4c1..7c92ca187 100644 --- a/pkg/controller/pod_volume_backup_controller.go +++ b/pkg/controller/pod_volume_backup_controller.go @@ -208,7 +208,7 @@ func (c *podVolumeBackupController) processBackup(req *velerov1api.PodVolumeBack return c.fail(req, errors.Wrap(err, "error getting pod").Error(), log) } - volumeDir, err := kube.GetVolumeDirectory(pod, req.Spec.Volume, c.pvcLister, c.pvLister) + volumeDir, err := kube.GetVolumeDirectory(log, pod, req.Spec.Volume, c.pvcLister, c.pvLister, c.kbClient) if err != nil { log.WithError(err).Error("Error getting volume directory name") return c.fail(req, errors.Wrap(err, "error getting volume directory name").Error(), log) diff --git a/pkg/controller/pod_volume_restore_controller.go b/pkg/controller/pod_volume_restore_controller.go index 3b13cb8a8..e04de5c65 100644 --- a/pkg/controller/pod_volume_restore_controller.go +++ b/pkg/controller/pod_volume_restore_controller.go @@ -297,7 +297,7 @@ func (c *podVolumeRestoreController) processRestore(req *velerov1api.PodVolumeRe return c.failRestore(req, errors.Wrap(err, "error getting pod").Error(), log) } - volumeDir, err := kube.GetVolumeDirectory(pod, req.Spec.Volume, c.pvcLister, c.pvLister) + volumeDir, err := kube.GetVolumeDirectory(log, pod, req.Spec.Volume, c.pvcLister, c.pvLister, c.kbClient) if err != nil { log.WithError(err).Error("Error getting volume directory name") return c.failRestore(req, errors.Wrap(err, "error getting volume directory name").Error(), log) diff --git a/pkg/controller/restore_controller.go b/pkg/controller/restore_controller.go index 78472df4d..5b83e5a13 100644 --- a/pkg/controller/restore_controller.go +++ b/pkg/controller/restore_controller.go @@ -47,6 +47,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" + "github.com/vmware-tanzu/velero/pkg/plugin/framework" pkgrestore "github.com/vmware-tanzu/velero/pkg/restore" "github.com/vmware-tanzu/velero/pkg/util/collections" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" @@ -443,6 +444,13 @@ func (c *restoreController) runValidatedRestore(restore *api.Restore, info backu if err != nil { return errors.Wrap(err, "error getting restore item actions") } + actionsResolver := framework.NewRestoreItemActionResolver(actions) + + itemSnapshotters, err := pluginManager.GetItemSnapshotters() + if err != nil { + return errors.Wrap(err, "error getting item snapshotters") + } + snapshotItemResolver := framework.NewItemSnapshotterResolver(itemSnapshotters) backupFile, err := downloadToTempFile(restore.Spec.BackupName, info.backupStore, restoreLog) if err != nil { @@ -476,7 +484,8 @@ func (c *restoreController) runValidatedRestore(restore *api.Restore, info backu VolumeSnapshots: volumeSnapshots, BackupReader: backupFile, } - restoreWarnings, restoreErrors := c.restorer.Restore(restoreReq, actions, c.snapshotLocationLister, pluginManager) + restoreWarnings, restoreErrors := c.restorer.RestoreWithResolvers(restoreReq, actionsResolver, snapshotItemResolver, + c.snapshotLocationLister, pluginManager) restoreLog.Info("restore completed") // re-instantiate the backup store because credentials could have changed since the original diff --git a/pkg/controller/restore_controller_test.go b/pkg/controller/restore_controller_test.go index d9f2a181c..1641b4f85 100644 --- a/pkg/controller/restore_controller_test.go +++ b/pkg/controller/restore_controller_test.go @@ -44,8 +44,10 @@ import ( "github.com/vmware-tanzu/velero/pkg/metrics" persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" + "github.com/vmware-tanzu/velero/pkg/plugin/framework" pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/velero" + isv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/item_snapshotter/v1" pkgrestore "github.com/vmware-tanzu/velero/pkg/restore" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/logging" @@ -505,7 +507,8 @@ func TestProcessQueueItem(t *testing.T) { if test.expectedRestorerCall != nil { backupStore.On("GetBackupContents", test.backup.Name).Return(ioutil.NopCloser(bytes.NewReader([]byte("hello world"))), nil) - restorer.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(warnings, errors) + restorer.On("RestoreWithResolvers", mock.Anything, mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(warnings, errors) backupStore.On("PutRestoreLog", test.backup.Name, test.restore.Name, mock.Anything).Return(test.putRestoreLogErr) @@ -545,6 +548,7 @@ func TestProcessQueueItem(t *testing.T) { if test.restore != nil { pluginManager.On("GetRestoreItemActions").Return(nil, nil) + pluginManager.On("GetItemSnapshotters").Return([]isv1.ItemSnapshotter{}, nil) pluginManager.On("CleanupClients") } @@ -858,3 +862,17 @@ func (r *fakeRestorer) Restore( return res.Get(0).(pkgrestore.Result), res.Get(1).(pkgrestore.Result) } + +func (r *fakeRestorer) RestoreWithResolvers(req pkgrestore.Request, + resolver framework.RestoreItemActionResolver, + itemSnapshotterResolver framework.ItemSnapshotterResolver, + snapshotLocationLister listers.VolumeSnapshotLocationLister, + volumeSnapshotterGetter pkgrestore.VolumeSnapshotterGetter, +) (pkgrestore.Result, pkgrestore.Result) { + res := r.Called(req.Log, req.Restore, req.Backup, req.BackupReader, resolver, itemSnapshotterResolver, + snapshotLocationLister, volumeSnapshotterGetter) + + r.calledWithArg = *req.Restore + + return res.Get(0).(pkgrestore.Result), res.Get(1).(pkgrestore.Result) +} diff --git a/pkg/controller/schedule_controller.go b/pkg/controller/schedule_controller.go index 1d114494d..e5f48814f 100644 --- a/pkg/controller/schedule_controller.go +++ b/pkg/controller/schedule_controller.go @@ -276,11 +276,11 @@ func (c *scheduleController) submitBackupIfDue(item *api.Schedule, cronSchedule } func getNextRunTime(schedule *api.Schedule, cronSchedule cron.Schedule, asOf time.Time) (bool, time.Time) { - // get the latest run time (if the schedule hasn't run yet, this will be the zero value which will trigger - // an immediate backup) var lastBackupTime time.Time if schedule.Status.LastBackup != nil { lastBackupTime = schedule.Status.LastBackup.Time + } else { + lastBackupTime = schedule.CreationTimestamp.Time } nextRunTime := cronSchedule.Next(lastBackupTime) diff --git a/pkg/controller/schedule_controller_test.go b/pkg/controller/schedule_controller_test.go index 2dfff7c06..5900e673e 100644 --- a/pkg/controller/schedule_controller_test.go +++ b/pkg/controller/schedule_controller_test.go @@ -274,7 +274,7 @@ func TestGetNextRunTime(t *testing.T) { { name: "first run", schedule: defaultSchedule(), - expectedDue: true, + expectedDue: false, expectedNextRunTimeOffset: "5m", }, { @@ -319,6 +319,9 @@ func TestGetNextRunTime(t *testing.T) { require.NoError(t, err, "unable to parse test.lastRanOffset: %v", err) test.schedule.Status.LastBackup = &metav1.Time{Time: testClock.Now().Add(-offsetDuration)} + test.schedule.CreationTimestamp = *test.schedule.Status.LastBackup + } else { + test.schedule.CreationTimestamp = metav1.Time{Time: testClock.Now()} } nextRunTimeOffset, err := time.ParseDuration(test.expectedNextRunTimeOffset) @@ -326,11 +329,11 @@ func TestGetNextRunTime(t *testing.T) { panic(err) } - // calculate expected next run time (if the schedule hasn't run yet, this - // will be the zero value which will trigger an immediate backup) var baseTime time.Time if test.lastRanOffset != "" { baseTime = test.schedule.Status.LastBackup.Time + } else { + baseTime = test.schedule.CreationTimestamp.Time } expectedNextRunTime := baseTime.Add(nextRunTimeOffset) diff --git a/pkg/controller/suite_test.go b/pkg/controller/suite_test.go index d4c07b995..b5c5b4d08 100644 --- a/pkg/controller/suite_test.go +++ b/pkg/controller/suite_test.go @@ -18,6 +18,7 @@ package controller import ( "context" + "fmt" "path/filepath" "testing" "time" @@ -130,6 +131,13 @@ func (t *testEnvironment) stop() error { return env.Stop() } +type fakeErrorBackupStoreGetter struct { +} + +func (f *fakeErrorBackupStoreGetter) Get(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) { + return nil, fmt.Errorf("some error") +} + type fakeSingleObjectBackupStoreGetter struct { store persistence.BackupStore } diff --git a/pkg/generated/clientset/versioned/fake/clientset_generated.go b/pkg/generated/clientset/versioned/fake/clientset_generated.go index 2b780e794..971ea5491 100644 --- a/pkg/generated/clientset/versioned/fake/clientset_generated.go +++ b/pkg/generated/clientset/versioned/fake/clientset_generated.go @@ -74,7 +74,10 @@ func (c *Clientset) Tracker() testing.ObjectTracker { return c.tracker } -var _ clientset.Interface = &Clientset{} +var ( + _ clientset.Interface = &Clientset{} + _ testing.FakeClient = &Clientset{} +) // VeleroV1 retrieves the VeleroV1Client func (c *Clientset) VeleroV1() velerov1.VeleroV1Interface { diff --git a/pkg/generated/clientset/versioned/fake/register.go b/pkg/generated/clientset/versioned/fake/register.go index 0ee7ffaba..3482b9c35 100644 --- a/pkg/generated/clientset/versioned/fake/register.go +++ b/pkg/generated/clientset/versioned/fake/register.go @@ -29,7 +29,7 @@ import ( var scheme = runtime.NewScheme() var codecs = serializer.NewCodecFactory(scheme) -var parameterCodec = runtime.NewParameterCodec(scheme) + var localSchemeBuilder = runtime.SchemeBuilder{ velerov1.AddToScheme, } diff --git a/pkg/generated/listers/velero/v1/backup.go b/pkg/generated/listers/velero/v1/backup.go index f61b1bab7..fa3f5cb6f 100644 --- a/pkg/generated/listers/velero/v1/backup.go +++ b/pkg/generated/listers/velero/v1/backup.go @@ -26,8 +26,10 @@ import ( ) // BackupLister helps list Backups. +// All objects returned here must be treated as read-only. type BackupLister interface { // List lists all Backups in the indexer. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.Backup, err error) // Backups returns an object that can list and get Backups. Backups(namespace string) BackupNamespaceLister @@ -58,10 +60,13 @@ func (s *backupLister) Backups(namespace string) BackupNamespaceLister { } // BackupNamespaceLister helps list and get Backups. +// All objects returned here must be treated as read-only. type BackupNamespaceLister interface { // List lists all Backups in the indexer for a given namespace. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.Backup, err error) // Get retrieves the Backup from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. Get(name string) (*v1.Backup, error) BackupNamespaceListerExpansion } diff --git a/pkg/generated/listers/velero/v1/backupstoragelocation.go b/pkg/generated/listers/velero/v1/backupstoragelocation.go index 8c6c140bd..74daf16dc 100644 --- a/pkg/generated/listers/velero/v1/backupstoragelocation.go +++ b/pkg/generated/listers/velero/v1/backupstoragelocation.go @@ -26,8 +26,10 @@ import ( ) // BackupStorageLocationLister helps list BackupStorageLocations. +// All objects returned here must be treated as read-only. type BackupStorageLocationLister interface { // List lists all BackupStorageLocations in the indexer. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.BackupStorageLocation, err error) // BackupStorageLocations returns an object that can list and get BackupStorageLocations. BackupStorageLocations(namespace string) BackupStorageLocationNamespaceLister @@ -58,10 +60,13 @@ func (s *backupStorageLocationLister) BackupStorageLocations(namespace string) B } // BackupStorageLocationNamespaceLister helps list and get BackupStorageLocations. +// All objects returned here must be treated as read-only. type BackupStorageLocationNamespaceLister interface { // List lists all BackupStorageLocations in the indexer for a given namespace. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.BackupStorageLocation, err error) // Get retrieves the BackupStorageLocation from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. Get(name string) (*v1.BackupStorageLocation, error) BackupStorageLocationNamespaceListerExpansion } diff --git a/pkg/generated/listers/velero/v1/deletebackuprequest.go b/pkg/generated/listers/velero/v1/deletebackuprequest.go index 45d270213..954e9aaf8 100644 --- a/pkg/generated/listers/velero/v1/deletebackuprequest.go +++ b/pkg/generated/listers/velero/v1/deletebackuprequest.go @@ -26,8 +26,10 @@ import ( ) // DeleteBackupRequestLister helps list DeleteBackupRequests. +// All objects returned here must be treated as read-only. type DeleteBackupRequestLister interface { // List lists all DeleteBackupRequests in the indexer. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.DeleteBackupRequest, err error) // DeleteBackupRequests returns an object that can list and get DeleteBackupRequests. DeleteBackupRequests(namespace string) DeleteBackupRequestNamespaceLister @@ -58,10 +60,13 @@ func (s *deleteBackupRequestLister) DeleteBackupRequests(namespace string) Delet } // DeleteBackupRequestNamespaceLister helps list and get DeleteBackupRequests. +// All objects returned here must be treated as read-only. type DeleteBackupRequestNamespaceLister interface { // List lists all DeleteBackupRequests in the indexer for a given namespace. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.DeleteBackupRequest, err error) // Get retrieves the DeleteBackupRequest from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. Get(name string) (*v1.DeleteBackupRequest, error) DeleteBackupRequestNamespaceListerExpansion } diff --git a/pkg/generated/listers/velero/v1/downloadrequest.go b/pkg/generated/listers/velero/v1/downloadrequest.go index 206c42e25..6552cf02d 100644 --- a/pkg/generated/listers/velero/v1/downloadrequest.go +++ b/pkg/generated/listers/velero/v1/downloadrequest.go @@ -26,8 +26,10 @@ import ( ) // DownloadRequestLister helps list DownloadRequests. +// All objects returned here must be treated as read-only. type DownloadRequestLister interface { // List lists all DownloadRequests in the indexer. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.DownloadRequest, err error) // DownloadRequests returns an object that can list and get DownloadRequests. DownloadRequests(namespace string) DownloadRequestNamespaceLister @@ -58,10 +60,13 @@ func (s *downloadRequestLister) DownloadRequests(namespace string) DownloadReque } // DownloadRequestNamespaceLister helps list and get DownloadRequests. +// All objects returned here must be treated as read-only. type DownloadRequestNamespaceLister interface { // List lists all DownloadRequests in the indexer for a given namespace. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.DownloadRequest, err error) // Get retrieves the DownloadRequest from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. Get(name string) (*v1.DownloadRequest, error) DownloadRequestNamespaceListerExpansion } diff --git a/pkg/generated/listers/velero/v1/podvolumebackup.go b/pkg/generated/listers/velero/v1/podvolumebackup.go index f92949685..08ed20d6f 100644 --- a/pkg/generated/listers/velero/v1/podvolumebackup.go +++ b/pkg/generated/listers/velero/v1/podvolumebackup.go @@ -26,8 +26,10 @@ import ( ) // PodVolumeBackupLister helps list PodVolumeBackups. +// All objects returned here must be treated as read-only. type PodVolumeBackupLister interface { // List lists all PodVolumeBackups in the indexer. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.PodVolumeBackup, err error) // PodVolumeBackups returns an object that can list and get PodVolumeBackups. PodVolumeBackups(namespace string) PodVolumeBackupNamespaceLister @@ -58,10 +60,13 @@ func (s *podVolumeBackupLister) PodVolumeBackups(namespace string) PodVolumeBack } // PodVolumeBackupNamespaceLister helps list and get PodVolumeBackups. +// All objects returned here must be treated as read-only. type PodVolumeBackupNamespaceLister interface { // List lists all PodVolumeBackups in the indexer for a given namespace. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.PodVolumeBackup, err error) // Get retrieves the PodVolumeBackup from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. Get(name string) (*v1.PodVolumeBackup, error) PodVolumeBackupNamespaceListerExpansion } diff --git a/pkg/generated/listers/velero/v1/podvolumerestore.go b/pkg/generated/listers/velero/v1/podvolumerestore.go index c17dd3857..93f96b24b 100644 --- a/pkg/generated/listers/velero/v1/podvolumerestore.go +++ b/pkg/generated/listers/velero/v1/podvolumerestore.go @@ -26,8 +26,10 @@ import ( ) // PodVolumeRestoreLister helps list PodVolumeRestores. +// All objects returned here must be treated as read-only. type PodVolumeRestoreLister interface { // List lists all PodVolumeRestores in the indexer. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.PodVolumeRestore, err error) // PodVolumeRestores returns an object that can list and get PodVolumeRestores. PodVolumeRestores(namespace string) PodVolumeRestoreNamespaceLister @@ -58,10 +60,13 @@ func (s *podVolumeRestoreLister) PodVolumeRestores(namespace string) PodVolumeRe } // PodVolumeRestoreNamespaceLister helps list and get PodVolumeRestores. +// All objects returned here must be treated as read-only. type PodVolumeRestoreNamespaceLister interface { // List lists all PodVolumeRestores in the indexer for a given namespace. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.PodVolumeRestore, err error) // Get retrieves the PodVolumeRestore from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. Get(name string) (*v1.PodVolumeRestore, error) PodVolumeRestoreNamespaceListerExpansion } diff --git a/pkg/generated/listers/velero/v1/resticrepository.go b/pkg/generated/listers/velero/v1/resticrepository.go index 67a8168dc..96bcfdc7c 100644 --- a/pkg/generated/listers/velero/v1/resticrepository.go +++ b/pkg/generated/listers/velero/v1/resticrepository.go @@ -26,8 +26,10 @@ import ( ) // ResticRepositoryLister helps list ResticRepositories. +// All objects returned here must be treated as read-only. type ResticRepositoryLister interface { // List lists all ResticRepositories in the indexer. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.ResticRepository, err error) // ResticRepositories returns an object that can list and get ResticRepositories. ResticRepositories(namespace string) ResticRepositoryNamespaceLister @@ -58,10 +60,13 @@ func (s *resticRepositoryLister) ResticRepositories(namespace string) ResticRepo } // ResticRepositoryNamespaceLister helps list and get ResticRepositories. +// All objects returned here must be treated as read-only. type ResticRepositoryNamespaceLister interface { // List lists all ResticRepositories in the indexer for a given namespace. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.ResticRepository, err error) // Get retrieves the ResticRepository from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. Get(name string) (*v1.ResticRepository, error) ResticRepositoryNamespaceListerExpansion } diff --git a/pkg/generated/listers/velero/v1/restore.go b/pkg/generated/listers/velero/v1/restore.go index 9c1360554..de0b89ce8 100644 --- a/pkg/generated/listers/velero/v1/restore.go +++ b/pkg/generated/listers/velero/v1/restore.go @@ -26,8 +26,10 @@ import ( ) // RestoreLister helps list Restores. +// All objects returned here must be treated as read-only. type RestoreLister interface { // List lists all Restores in the indexer. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.Restore, err error) // Restores returns an object that can list and get Restores. Restores(namespace string) RestoreNamespaceLister @@ -58,10 +60,13 @@ func (s *restoreLister) Restores(namespace string) RestoreNamespaceLister { } // RestoreNamespaceLister helps list and get Restores. +// All objects returned here must be treated as read-only. type RestoreNamespaceLister interface { // List lists all Restores in the indexer for a given namespace. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.Restore, err error) // Get retrieves the Restore from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. Get(name string) (*v1.Restore, error) RestoreNamespaceListerExpansion } diff --git a/pkg/generated/listers/velero/v1/schedule.go b/pkg/generated/listers/velero/v1/schedule.go index b1431af94..90a262a46 100644 --- a/pkg/generated/listers/velero/v1/schedule.go +++ b/pkg/generated/listers/velero/v1/schedule.go @@ -26,8 +26,10 @@ import ( ) // ScheduleLister helps list Schedules. +// All objects returned here must be treated as read-only. type ScheduleLister interface { // List lists all Schedules in the indexer. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.Schedule, err error) // Schedules returns an object that can list and get Schedules. Schedules(namespace string) ScheduleNamespaceLister @@ -58,10 +60,13 @@ func (s *scheduleLister) Schedules(namespace string) ScheduleNamespaceLister { } // ScheduleNamespaceLister helps list and get Schedules. +// All objects returned here must be treated as read-only. type ScheduleNamespaceLister interface { // List lists all Schedules in the indexer for a given namespace. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.Schedule, err error) // Get retrieves the Schedule from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. Get(name string) (*v1.Schedule, error) ScheduleNamespaceListerExpansion } diff --git a/pkg/generated/listers/velero/v1/serverstatusrequest.go b/pkg/generated/listers/velero/v1/serverstatusrequest.go index ae35c2d6c..c03b60c48 100644 --- a/pkg/generated/listers/velero/v1/serverstatusrequest.go +++ b/pkg/generated/listers/velero/v1/serverstatusrequest.go @@ -26,8 +26,10 @@ import ( ) // ServerStatusRequestLister helps list ServerStatusRequests. +// All objects returned here must be treated as read-only. type ServerStatusRequestLister interface { // List lists all ServerStatusRequests in the indexer. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.ServerStatusRequest, err error) // ServerStatusRequests returns an object that can list and get ServerStatusRequests. ServerStatusRequests(namespace string) ServerStatusRequestNamespaceLister @@ -58,10 +60,13 @@ func (s *serverStatusRequestLister) ServerStatusRequests(namespace string) Serve } // ServerStatusRequestNamespaceLister helps list and get ServerStatusRequests. +// All objects returned here must be treated as read-only. type ServerStatusRequestNamespaceLister interface { // List lists all ServerStatusRequests in the indexer for a given namespace. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.ServerStatusRequest, err error) // Get retrieves the ServerStatusRequest from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. Get(name string) (*v1.ServerStatusRequest, error) ServerStatusRequestNamespaceListerExpansion } diff --git a/pkg/generated/listers/velero/v1/volumesnapshotlocation.go b/pkg/generated/listers/velero/v1/volumesnapshotlocation.go index b5ff7fe53..8c8aa432f 100644 --- a/pkg/generated/listers/velero/v1/volumesnapshotlocation.go +++ b/pkg/generated/listers/velero/v1/volumesnapshotlocation.go @@ -26,8 +26,10 @@ import ( ) // VolumeSnapshotLocationLister helps list VolumeSnapshotLocations. +// All objects returned here must be treated as read-only. type VolumeSnapshotLocationLister interface { // List lists all VolumeSnapshotLocations in the indexer. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.VolumeSnapshotLocation, err error) // VolumeSnapshotLocations returns an object that can list and get VolumeSnapshotLocations. VolumeSnapshotLocations(namespace string) VolumeSnapshotLocationNamespaceLister @@ -58,10 +60,13 @@ func (s *volumeSnapshotLocationLister) VolumeSnapshotLocations(namespace string) } // VolumeSnapshotLocationNamespaceLister helps list and get VolumeSnapshotLocations. +// All objects returned here must be treated as read-only. type VolumeSnapshotLocationNamespaceLister interface { // List lists all VolumeSnapshotLocations in the indexer for a given namespace. + // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*v1.VolumeSnapshotLocation, err error) // Get retrieves the VolumeSnapshotLocation from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. Get(name string) (*v1.VolumeSnapshotLocation, error) VolumeSnapshotLocationNamespaceListerExpansion } diff --git a/pkg/install/install.go b/pkg/install/install.go index 85257f4ad..ebe4c3751 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -50,6 +50,7 @@ var kindToResource = map[string]string{ "Deployment": "deployments", "DaemonSet": "daemonsets", "Secret": "secrets", + "ConfigMap": "configmaps", "BackupStorageLocation": "backupstoragelocations", "VolumeSnapshotLocation": "volumesnapshotlocations", } diff --git a/pkg/install/resources.go b/pkg/install/resources.go index dbe19fba9..0e86698cf 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -27,7 +27,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" v1crds "github.com/vmware-tanzu/velero/config/crd/v1/crds" - v1beta1crds "github.com/vmware-tanzu/velero/config/crd/v1beta1/crds" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) @@ -216,26 +215,17 @@ type VeleroOptions struct { NoDefaultBackupLocation bool CACertData []byte Features []string - CRDsVersion string DefaultVolumesToRestic bool } -func AllCRDs(perferredAPIVersion string) *unstructured.UnstructuredList { +func AllCRDs() *unstructured.UnstructuredList { resources := new(unstructured.UnstructuredList) // Set the GVK so that the serialization framework outputs the list properly resources.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "List"}) - switch perferredAPIVersion { - case "v1beta1": - for _, crd := range v1beta1crds.CRDs { - crd.SetLabels(Labels()) - appendUnstructured(resources, crd) - } - case "v1": - for _, crd := range v1crds.CRDs { - crd.SetLabels(Labels()) - appendUnstructured(resources, crd) - } + for _, crd := range v1crds.CRDs { + crd.SetLabels(Labels()) + appendUnstructured(resources, crd) } return resources @@ -244,7 +234,7 @@ func AllCRDs(perferredAPIVersion string) *unstructured.UnstructuredList { // AllResources returns a list of all resources necessary to install Velero, in the appropriate order, into a Kubernetes cluster. // Items are unstructured, since there are different data types returned. func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { - resources := AllCRDs(o.CRDsVersion) + resources := AllCRDs() ns := Namespace(o.Namespace) appendUnstructured(resources, ns) diff --git a/pkg/persistence/mocks/backup_store.go b/pkg/persistence/mocks/backup_store.go index d3a7d7c5a..b41125e56 100644 --- a/pkg/persistence/mocks/backup_store.go +++ b/pkg/persistence/mocks/backup_store.go @@ -283,3 +283,7 @@ func (_m *BackupStore) GetCSIVolumeSnapshotContents(backup string) ([]*snapshotv panic("Not implemented") return nil, nil } + +func (_m *BackupStore) GetItemSnapshots(name string) ([]*volume.ItemSnapshot, error) { + panic("implement me") +} diff --git a/pkg/persistence/object_store.go b/pkg/persistence/object_store.go index d9f2d8a39..744f2baaa 100644 --- a/pkg/persistence/object_store.go +++ b/pkg/persistence/object_store.go @@ -44,6 +44,7 @@ type BackupInfo struct { Log, PodVolumeBackups, VolumeSnapshots, + ItemSnapshots, BackupResourceList, CSIVolumeSnapshots, CSIVolumeSnapshotContents io.Reader @@ -58,6 +59,7 @@ type BackupStore interface { PutBackup(info BackupInfo) error GetBackupMetadata(name string) (*velerov1api.Backup, error) + GetItemSnapshots(name string) ([]*volume.ItemSnapshot, error) GetBackupVolumeSnapshots(name string) ([]*volume.Snapshot, error) GetPodVolumeBackups(name string) ([]*velerov1api.PodVolumeBackup, error) GetBackupContents(name string) (io.ReadCloser, error) @@ -231,13 +233,6 @@ func (s *objectBackupStore) PutBackup(info BackupInfo) error { s.logger.WithError(err).WithField("backup", info.Name).Error("Error uploading log file") } - if info.Metadata == nil { - // If we don't have metadata, something failed, and there's no point in continuing. An object - // storage bucket that is missing the metadata file can't be restored, nor can its logs be - // viewed. - return nil - } - if err := seekAndPutObject(s.objectStore, s.bucket, s.layout.getBackupMetadataKey(info.Name), info.Metadata); err != nil { // failure to upload metadata file is a hard-stop return err @@ -253,6 +248,7 @@ func (s *objectBackupStore) PutBackup(info BackupInfo) error { var backupObjs = map[string]io.Reader{ s.layout.getPodVolumeBackupsKey(info.Name): info.PodVolumeBackups, s.layout.getBackupVolumeSnapshotsKey(info.Name): info.VolumeSnapshots, + s.layout.getItemSnapshotsKey(info.Name): info.ItemSnapshots, s.layout.getBackupResourceListKey(info.Name): info.BackupResourceList, s.layout.getCSIVolumeSnapshotKey(info.Name): info.CSIVolumeSnapshots, s.layout.getCSIVolumeSnapshotContentsKey(info.Name): info.CSIVolumeSnapshotContents, @@ -324,6 +320,27 @@ func (s *objectBackupStore) GetBackupVolumeSnapshots(name string) ([]*volume.Sna return volumeSnapshots, nil } +func (s *objectBackupStore) GetItemSnapshots(name string) ([]*volume.ItemSnapshot, error) { + // if the itemsnapshots file doesn't exist, we don't want to return an error, since + // a legacy backup or a backup with no snapshots would not have this file, so check for + // its existence before attempting to get its contents. + res, err := tryGet(s.objectStore, s.bucket, s.layout.getItemSnapshotsKey(name)) + if err != nil { + return nil, err + } + if res == nil { + return nil, nil + } + defer res.Close() + + var itemSnapshots []*volume.ItemSnapshot + if err := decode(res, &itemSnapshots); err != nil { + return nil, err + } + + return itemSnapshots, nil +} + // tryGet returns the object with the given key if it exists, nil if it does not exist, // or an error if it was unable to check existence or get the object. func tryGet(objectStore velero.ObjectStore, bucket, key string) (io.ReadCloser, error) { @@ -473,6 +490,8 @@ func (s *objectBackupStore) GetDownloadURL(target velerov1api.DownloadTarget) (s return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupLogKey(target.Name), DownloadURLTTL) case velerov1api.DownloadTargetKindBackupVolumeSnapshots: return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupVolumeSnapshotsKey(target.Name), DownloadURLTTL) + case velerov1api.DownloadTargetKindBackupItemSnapshots: + return s.objectStore.CreateSignedURL(s.bucket, s.layout.getItemSnapshotsKey(target.Name), DownloadURLTTL) case velerov1api.DownloadTargetKindBackupResourceList: return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupResourceListKey(target.Name), DownloadURLTTL) case velerov1api.DownloadTargetKindRestoreLog: diff --git a/pkg/persistence/object_store_layout.go b/pkg/persistence/object_store_layout.go index 025da966b..046b99a68 100644 --- a/pkg/persistence/object_store_layout.go +++ b/pkg/persistence/object_store_layout.go @@ -88,6 +88,10 @@ func (l *ObjectStoreLayout) getBackupVolumeSnapshotsKey(backup string) string { return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-volumesnapshots.json.gz", backup)) } +func (l *ObjectStoreLayout) getItemSnapshotsKey(backup string) string { + return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-itemsnapshots.json.gz", backup)) +} + func (l *ObjectStoreLayout) getBackupResourceListKey(backup string) string { return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-resource-list.json.gz", backup)) } diff --git a/pkg/persistence/object_store_test.go b/pkg/persistence/object_store_test.go index 76d37bd40..f3cb55486 100644 --- a/pkg/persistence/object_store_test.go +++ b/pkg/persistence/object_store_test.go @@ -223,6 +223,7 @@ func TestPutBackup(t *testing.T) { log io.Reader podVolumeBackup io.Reader snapshots io.Reader + itemSnapshots io.Reader resourceList io.Reader expectedErr string expectedKeys []string @@ -234,6 +235,7 @@ func TestPutBackup(t *testing.T) { log: newStringReadSeeker("log"), podVolumeBackup: newStringReadSeeker("podVolumeBackup"), snapshots: newStringReadSeeker("snapshots"), + itemSnapshots: newStringReadSeeker("itemSnapshots"), resourceList: newStringReadSeeker("resourceList"), expectedErr: "", expectedKeys: []string{ @@ -242,6 +244,7 @@ func TestPutBackup(t *testing.T) { "backups/backup-1/backup-1-logs.gz", "backups/backup-1/backup-1-podvolumebackups.json.gz", "backups/backup-1/backup-1-volumesnapshots.json.gz", + "backups/backup-1/backup-1-itemsnapshots.json.gz", "backups/backup-1/backup-1-resource-list.json.gz", }, }, @@ -253,6 +256,7 @@ func TestPutBackup(t *testing.T) { log: newStringReadSeeker("log"), podVolumeBackup: newStringReadSeeker("podVolumeBackup"), snapshots: newStringReadSeeker("snapshots"), + itemSnapshots: newStringReadSeeker("itemSnapshots"), resourceList: newStringReadSeeker("resourceList"), expectedErr: "", expectedKeys: []string{ @@ -261,6 +265,7 @@ func TestPutBackup(t *testing.T) { "prefix-1/backups/backup-1/backup-1-logs.gz", "prefix-1/backups/backup-1/backup-1-podvolumebackups.json.gz", "prefix-1/backups/backup-1/backup-1-volumesnapshots.json.gz", + "prefix-1/backups/backup-1/backup-1-itemsnapshots.json.gz", "prefix-1/backups/backup-1/backup-1-resource-list.json.gz", }, }, @@ -271,19 +276,21 @@ func TestPutBackup(t *testing.T) { log: newStringReadSeeker("log"), podVolumeBackup: newStringReadSeeker("podVolumeBackup"), snapshots: newStringReadSeeker("snapshots"), + itemSnapshots: newStringReadSeeker("itemSnapshots"), resourceList: newStringReadSeeker("resourceList"), expectedErr: "error readers return errors", expectedKeys: []string{"backups/backup-1/backup-1-logs.gz"}, }, { - name: "error on data upload deletes metadata", - metadata: newStringReadSeeker("metadata"), - contents: new(errorReader), - log: newStringReadSeeker("log"), - snapshots: newStringReadSeeker("snapshots"), - resourceList: newStringReadSeeker("resourceList"), - expectedErr: "error readers return errors", - expectedKeys: []string{"backups/backup-1/backup-1-logs.gz"}, + name: "error on data upload deletes metadata", + metadata: newStringReadSeeker("metadata"), + contents: new(errorReader), + log: newStringReadSeeker("log"), + snapshots: newStringReadSeeker("snapshots"), + itemSnapshots: newStringReadSeeker("itemSnapshots"), + resourceList: newStringReadSeeker("resourceList"), + expectedErr: "error readers return errors", + expectedKeys: []string{"backups/backup-1/backup-1-logs.gz"}, }, { name: "error on log upload is ok", @@ -292,6 +299,7 @@ func TestPutBackup(t *testing.T) { log: new(errorReader), podVolumeBackup: newStringReadSeeker("podVolumeBackup"), snapshots: newStringReadSeeker("snapshots"), + itemSnapshots: newStringReadSeeker("itemSnapshots"), resourceList: newStringReadSeeker("resourceList"), expectedErr: "", expectedKeys: []string{ @@ -299,11 +307,12 @@ func TestPutBackup(t *testing.T) { "backups/backup-1/backup-1.tar.gz", "backups/backup-1/backup-1-podvolumebackups.json.gz", "backups/backup-1/backup-1-volumesnapshots.json.gz", + "backups/backup-1/backup-1-itemsnapshots.json.gz", "backups/backup-1/backup-1-resource-list.json.gz", }, }, { - name: "don't upload data when metadata is nil", + name: "data should be uploaded even when metadata is nil", metadata: nil, contents: newStringReadSeeker("contents"), log: newStringReadSeeker("log"), @@ -311,7 +320,13 @@ func TestPutBackup(t *testing.T) { snapshots: newStringReadSeeker("snapshots"), resourceList: newStringReadSeeker("resourceList"), expectedErr: "", - expectedKeys: []string{"backups/backup-1/backup-1-logs.gz"}, + expectedKeys: []string{ + "backups/backup-1/backup-1.tar.gz", + "backups/backup-1/backup-1-logs.gz", + "backups/backup-1/backup-1-podvolumebackups.json.gz", + "backups/backup-1/backup-1-volumesnapshots.json.gz", + "backups/backup-1/backup-1-resource-list.json.gz", + }, }, } @@ -326,6 +341,7 @@ func TestPutBackup(t *testing.T) { Log: tc.log, PodVolumeBackups: tc.podVolumeBackup, VolumeSnapshots: tc.snapshots, + ItemSnapshots: tc.itemSnapshots, BackupResourceList: tc.resourceList, } err := harness.PutBackup(backupInfo) @@ -426,6 +442,48 @@ func TestGetBackupVolumeSnapshots(t *testing.T) { assert.EqualValues(t, snapshots, res) } +func TestGetItemSnapshots(t *testing.T) { + harness := newObjectBackupStoreTestHarness("test-bucket", "") + + // volumesnapshots file not found should not error + harness.objectStore.PutObject(harness.bucket, "backups/test-backup/velero-backup.json", newStringReadSeeker("foo")) + res, err := harness.GetItemSnapshots("test-backup") + assert.NoError(t, err) + assert.Nil(t, res) + + // volumesnapshots file containing invalid data should error + harness.objectStore.PutObject(harness.bucket, "backups/test-backup/test-backup-itemsnapshots.json.gz", newStringReadSeeker("foo")) + res, err = harness.GetItemSnapshots("test-backup") + assert.NotNil(t, err) + + // volumesnapshots file containing gzipped json data should return correctly + snapshots := []*volume.ItemSnapshot{ + { + Spec: volume.ItemSnapshotSpec{ + BackupName: "test-backup", + ResourceIdentifier: "item-1", + }, + }, + { + Spec: volume.ItemSnapshotSpec{ + BackupName: "test-backup", + ResourceIdentifier: "item-2", + }, + }, + } + + obj := new(bytes.Buffer) + gzw := gzip.NewWriter(obj) + + require.NoError(t, json.NewEncoder(gzw).Encode(snapshots)) + require.NoError(t, gzw.Close()) + require.NoError(t, harness.objectStore.PutObject(harness.bucket, "backups/test-backup/test-backup-itemsnapshots.json.gz", obj)) + + res, err = harness.GetItemSnapshots("test-backup") + assert.NoError(t, err) + assert.EqualValues(t, snapshots, res) +} + func TestGetBackupContents(t *testing.T) { harness := newObjectBackupStoreTestHarness("test-bucket", "") @@ -506,6 +564,7 @@ func TestGetDownloadURL(t *testing.T) { velerov1api.DownloadTargetKindBackupContents: "backups/my-backup/my-backup.tar.gz", velerov1api.DownloadTargetKindBackupLog: "backups/my-backup/my-backup-logs.gz", velerov1api.DownloadTargetKindBackupVolumeSnapshots: "backups/my-backup/my-backup-volumesnapshots.json.gz", + velerov1api.DownloadTargetKindBackupItemSnapshots: "backups/my-backup/my-backup-itemsnapshots.json.gz", velerov1api.DownloadTargetKindBackupResourceList: "backups/my-backup/my-backup-resource-list.json.gz", }, }, @@ -517,6 +576,7 @@ func TestGetDownloadURL(t *testing.T) { velerov1api.DownloadTargetKindBackupContents: "velero-backups/backups/my-backup/my-backup.tar.gz", velerov1api.DownloadTargetKindBackupLog: "velero-backups/backups/my-backup/my-backup-logs.gz", velerov1api.DownloadTargetKindBackupVolumeSnapshots: "velero-backups/backups/my-backup/my-backup-volumesnapshots.json.gz", + velerov1api.DownloadTargetKindBackupItemSnapshots: "velero-backups/backups/my-backup/my-backup-itemsnapshots.json.gz", velerov1api.DownloadTargetKindBackupResourceList: "velero-backups/backups/my-backup/my-backup-resource-list.json.gz", }, }, @@ -527,6 +587,7 @@ func TestGetDownloadURL(t *testing.T) { velerov1api.DownloadTargetKindBackupContents: "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902.tar.gz", velerov1api.DownloadTargetKindBackupLog: "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902-logs.gz", velerov1api.DownloadTargetKindBackupVolumeSnapshots: "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902-volumesnapshots.json.gz", + velerov1api.DownloadTargetKindBackupItemSnapshots: "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902-itemsnapshots.json.gz", velerov1api.DownloadTargetKindBackupResourceList: "backups/b-cool-20170913154901-20170913154902/b-cool-20170913154901-20170913154902-resource-list.json.gz", }, }, @@ -537,6 +598,7 @@ func TestGetDownloadURL(t *testing.T) { velerov1api.DownloadTargetKindBackupContents: "backups/my-backup-20170913154901/my-backup-20170913154901.tar.gz", velerov1api.DownloadTargetKindBackupLog: "backups/my-backup-20170913154901/my-backup-20170913154901-logs.gz", velerov1api.DownloadTargetKindBackupVolumeSnapshots: "backups/my-backup-20170913154901/my-backup-20170913154901-volumesnapshots.json.gz", + velerov1api.DownloadTargetKindBackupItemSnapshots: "backups/my-backup-20170913154901/my-backup-20170913154901-itemsnapshots.json.gz", velerov1api.DownloadTargetKindBackupResourceList: "backups/my-backup-20170913154901/my-backup-20170913154901-resource-list.json.gz", }, }, @@ -548,6 +610,7 @@ func TestGetDownloadURL(t *testing.T) { velerov1api.DownloadTargetKindBackupContents: "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901.tar.gz", velerov1api.DownloadTargetKindBackupLog: "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901-logs.gz", velerov1api.DownloadTargetKindBackupVolumeSnapshots: "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901-volumesnapshots.json.gz", + velerov1api.DownloadTargetKindBackupItemSnapshots: "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901-itemsnapshots.json.gz", velerov1api.DownloadTargetKindBackupResourceList: "velero-backups/backups/my-backup-20170913154901/my-backup-20170913154901-resource-list.json.gz", }, }, diff --git a/pkg/plugin/clientmgmt/client_builder.go b/pkg/plugin/clientmgmt/client_builder.go index 60d278727..76e3b1985 100644 --- a/pkg/plugin/clientmgmt/client_builder.go +++ b/pkg/plugin/clientmgmt/client_builder.go @@ -73,6 +73,7 @@ func (b *clientBuilder) clientConfig() *hcplugin.ClientConfig { string(framework.PluginKindPluginLister): &framework.PluginListerPlugin{}, string(framework.PluginKindRestoreItemAction): framework.NewRestoreItemActionPlugin(framework.ClientLogger(b.clientLogger)), string(framework.PluginKindDeleteItemAction): framework.NewDeleteItemActionPlugin(framework.ClientLogger(b.clientLogger)), + string(framework.PluginKindItemSnapshotter): framework.NewItemSnapshotterPlugin(framework.ClientLogger(b.clientLogger)), }, Logger: b.pluginLogger, Cmd: exec.Command(b.commandName, b.commandArgs...), diff --git a/pkg/plugin/clientmgmt/client_builder_test.go b/pkg/plugin/clientmgmt/client_builder_test.go index e7fff7bef..e922c3d00 100644 --- a/pkg/plugin/clientmgmt/client_builder_test.go +++ b/pkg/plugin/clientmgmt/client_builder_test.go @@ -66,6 +66,7 @@ func TestClientConfig(t *testing.T) { string(framework.PluginKindPluginLister): &framework.PluginListerPlugin{}, string(framework.PluginKindRestoreItemAction): framework.NewRestoreItemActionPlugin(framework.ClientLogger(logger)), string(framework.PluginKindDeleteItemAction): framework.NewDeleteItemActionPlugin(framework.ClientLogger(logger)), + string(framework.PluginKindItemSnapshotter): framework.NewItemSnapshotterPlugin(framework.ClientLogger(logger)), }, Logger: cb.pluginLogger, Cmd: exec.Command(cb.commandName, cb.commandArgs...), diff --git a/pkg/plugin/clientmgmt/logrus_adapter.go b/pkg/plugin/clientmgmt/logrus_adapter.go index 77bc98424..3bb55d464 100644 --- a/pkg/plugin/clientmgmt/logrus_adapter.go +++ b/pkg/plugin/clientmgmt/logrus_adapter.go @@ -18,6 +18,7 @@ package clientmgmt import ( "fmt" + "io" "log" hclog "github.com/hashicorp/go-hclog" @@ -162,3 +163,37 @@ func (l *logrusAdapter) StandardLogger(opts *hclog.StandardLoggerOptions) *log.L func (l *logrusAdapter) SetLevel(_ hclog.Level) { return } + +// ImpliedArgs returns With key/value pairs +func (l *logrusAdapter) ImpliedArgs() []interface{} { + panic("not implemented") +} + +// Args are alternating key, val pairs +// keys must be strings +// vals can be any type, but display is implementation specific +// Emit a message and key/value pairs at a provided log level +func (l *logrusAdapter) Log(level hclog.Level, msg string, args ...interface{}) { + switch level { + case hclog.Trace: + l.Trace(msg, args...) + case hclog.Debug: + l.Debug(msg, args...) + case hclog.Info: + l.Info(msg, args...) + case hclog.Warn: + l.Warn(msg, args...) + case hclog.Error: + l.Error(msg, args...) + } +} + +// Returns the Name of the logger +func (l *logrusAdapter) Name() string { + return l.name +} + +// Return a value that conforms to io.Writer, which can be passed into log.SetOutput() +func (l *logrusAdapter) StandardWriter(opts *hclog.StandardLoggerOptions) io.Writer { + panic("not implemented") +} diff --git a/pkg/plugin/clientmgmt/manager.go b/pkg/plugin/clientmgmt/manager.go index 411b3b0bd..b1a952ac5 100644 --- a/pkg/plugin/clientmgmt/manager.go +++ b/pkg/plugin/clientmgmt/manager.go @@ -20,6 +20,8 @@ import ( "strings" "sync" + v1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/item_snapshotter/v1" + "github.com/sirupsen/logrus" "github.com/vmware-tanzu/velero/pkg/plugin/framework" @@ -52,6 +54,12 @@ type Manager interface { // GetDeleteItemAction returns the delete item action plugin for name. GetDeleteItemAction(name string) (velero.DeleteItemAction, error) + // GetItemSnapshotter returns the item snapshotter plugin for name + GetItemSnapshotter(name string) (v1.ItemSnapshotter, error) + + // GetItemSnapshotters returns all item snapshotter plugins + GetItemSnapshotters() ([]v1.ItemSnapshotter, error) + // CleanupClients terminates all of the Manager's running plugin processes. CleanupClients() } @@ -256,6 +264,37 @@ func (m *manager) GetDeleteItemAction(name string) (velero.DeleteItemAction, err return r, nil } +func (m *manager) GetItemSnapshotter(name string) (v1.ItemSnapshotter, error) { + name = sanitizeName(name) + + restartableProcess, err := m.getRestartableProcess(framework.PluginKindItemSnapshotter, name) + if err != nil { + return nil, err + } + + r := newRestartableItemSnapshotter(name, restartableProcess) + return r, nil +} + +func (m *manager) GetItemSnapshotters() ([]v1.ItemSnapshotter, error) { + list := m.registry.List(framework.PluginKindItemSnapshotter) + + actions := make([]v1.ItemSnapshotter, 0, len(list)) + + for i := range list { + id := list[i] + + r, err := m.GetItemSnapshotter(id.Name) + if err != nil { + return nil, err + } + + actions = append(actions, r) + } + + return actions, nil +} + // sanitizeName adds "velero.io" to legacy plugins that weren't namespaced. func sanitizeName(name string) string { // Backwards compatibility with non-namespaced Velero plugins, following principle of least surprise diff --git a/pkg/plugin/clientmgmt/restartable_item_snapshotter.go b/pkg/plugin/clientmgmt/restartable_item_snapshotter.go new file mode 100644 index 000000000..e211bcf28 --- /dev/null +++ b/pkg/plugin/clientmgmt/restartable_item_snapshotter.go @@ -0,0 +1,131 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clientmgmt + +import ( + "context" + + "github.com/pkg/errors" + + isv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/item_snapshotter/v1" + + "github.com/vmware-tanzu/velero/pkg/plugin/framework" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" +) + +type restartableItemSnapshotter struct { + key kindAndName + sharedPluginProcess RestartableProcess +} + +// newRestartableItemSnapshotter returns a new newRestartableItemSnapshotter. +func newRestartableItemSnapshotter(name string, sharedPluginProcess RestartableProcess) *restartableItemSnapshotter { + r := &restartableItemSnapshotter{ + key: kindAndName{kind: framework.PluginKindItemSnapshotter, name: name}, + sharedPluginProcess: sharedPluginProcess, + } + return r +} + +// getItemSnapshotter returns the item snapshotter for this restartableItemSnapshotter. It does *not* restart the +// plugin process. +func (r *restartableItemSnapshotter) getItemSnapshotter() (isv1.ItemSnapshotter, error) { + plugin, err := r.sharedPluginProcess.getByKindAndName(r.key) + if err != nil { + return nil, err + } + + itemSnapshotter, ok := plugin.(isv1.ItemSnapshotter) + if !ok { + return nil, errors.Errorf("%T is not an ItemSnapshotter!", plugin) + } + + return itemSnapshotter, nil +} + +// getDelegate restarts the plugin process (if needed) and returns the item snapshotter for this restartableItemSnapshotter. +func (r *restartableItemSnapshotter) getDelegate() (isv1.ItemSnapshotter, error) { + if err := r.sharedPluginProcess.resetIfNeeded(); err != nil { + return nil, err + } + + return r.getItemSnapshotter() +} + +func (r *restartableItemSnapshotter) Init(config map[string]string) error { + delegate, err := r.getDelegate() + if err != nil { + return err + } + + return delegate.Init(config) +} + +// AppliesTo restarts the plugin's process if needed, then delegates the call. +func (r *restartableItemSnapshotter) AppliesTo() (velero.ResourceSelector, error) { + delegate, err := r.getDelegate() + if err != nil { + return velero.ResourceSelector{}, err + } + + return delegate.AppliesTo() +} + +func (r *restartableItemSnapshotter) AlsoHandles(input *isv1.AlsoHandlesInput) ([]velero.ResourceIdentifier, error) { + delegate, err := r.getDelegate() + if err != nil { + return nil, err + } + + return delegate.AlsoHandles(input) +} + +func (r *restartableItemSnapshotter) SnapshotItem(ctx context.Context, input *isv1.SnapshotItemInput) (*isv1.SnapshotItemOutput, error) { + delegate, err := r.getDelegate() + if err != nil { + return nil, err + } + + return delegate.SnapshotItem(ctx, input) +} + +func (r *restartableItemSnapshotter) Progress(input *isv1.ProgressInput) (*isv1.ProgressOutput, error) { + delegate, err := r.getDelegate() + if err != nil { + return nil, err + } + + return delegate.Progress(input) +} + +func (r *restartableItemSnapshotter) DeleteSnapshot(ctx context.Context, input *isv1.DeleteSnapshotInput) error { + delegate, err := r.getDelegate() + if err != nil { + return err + } + + return delegate.DeleteSnapshot(ctx, input) +} + +func (r *restartableItemSnapshotter) CreateItemFromSnapshot(ctx context.Context, input *isv1.CreateItemInput) (*isv1.CreateItemOutput, error) { + delegate, err := r.getDelegate() + if err != nil { + return nil, err + } + + return delegate.CreateItemFromSnapshot(ctx, input) +} diff --git a/pkg/plugin/clientmgmt/restartable_item_snapshotter_test.go b/pkg/plugin/clientmgmt/restartable_item_snapshotter_test.go new file mode 100644 index 000000000..30b2aab43 --- /dev/null +++ b/pkg/plugin/clientmgmt/restartable_item_snapshotter_test.go @@ -0,0 +1,233 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clientmgmt + +import ( + "context" + "testing" + "time" + + isv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/item_snapshotter/v1" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + + v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/plugin/velero/item_snapshotter/v1/mocks" + + "github.com/vmware-tanzu/velero/pkg/plugin/framework" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" +) + +func TestRestartableGetItemSnapshotter(t *testing.T) { + tests := []struct { + name string + plugin interface{} + getError error + expectedError string + }{ + { + name: "error getting by kind and name", + getError: errors.Errorf("get error"), + expectedError: "get error", + }, + { + name: "wrong type", + plugin: 3, + expectedError: "int is not an ItemSnapshotter!", + }, + { + name: "happy path", + plugin: new(mocks.ItemSnapshotter), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + p := new(mockRestartableProcess) + defer p.AssertExpectations(t) + + name := "pvc" + key := kindAndName{kind: framework.PluginKindItemSnapshotter, name: name} + p.On("getByKindAndName", key).Return(tc.plugin, tc.getError) + + r := newRestartableItemSnapshotter(name, p) + a, err := r.getItemSnapshotter() + if tc.expectedError != "" { + assert.EqualError(t, err, tc.expectedError) + return + } + require.NoError(t, err) + + assert.Equal(t, tc.plugin, a) + }) + } +} + +func TestRestartableItemSnapshotterGetDelegate(t *testing.T) { + p := new(mockRestartableProcess) + defer p.AssertExpectations(t) + + // Reset error + p.On("resetIfNeeded").Return(errors.Errorf("reset error")).Once() + name := "pvc" + r := newRestartableItemSnapshotter(name, p) + a, err := r.getDelegate() + assert.Nil(t, a) + assert.EqualError(t, err, "reset error") + + // Happy path + p.On("resetIfNeeded").Return(nil) + expected := new(mocks.ItemSnapshotter) + key := kindAndName{kind: framework.PluginKindItemSnapshotter, name: name} + p.On("getByKindAndName", key).Return(expected, nil) + + a, err = r.getDelegate() + assert.NoError(t, err) + assert.Equal(t, expected, a) +} + +func TestRestartableItemSnasphotterDelegatedFunctions(t *testing.T) { + b := new(v1.Backup) + + pv := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "color": "blue", + }, + } + + sii := &isv1.SnapshotItemInput{ + Item: pv, + Params: nil, + Backup: b, + } + + ctx := context.Background() + + pvToReturn := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "color": "green", + }, + } + + additionalItems := []velero.ResourceIdentifier{ + { + GroupResource: schema.GroupResource{Group: "velero.io", Resource: "backups"}, + }, + } + + sio := &isv1.SnapshotItemOutput{ + UpdatedItem: pvToReturn, + SnapshotID: "", + SnapshotMetadata: nil, + AdditionalItems: additionalItems, + HandledItems: nil, + } + + cii := &isv1.CreateItemInput{ + SnapshottedItem: nil, + SnapshotID: "", + ItemFromBackup: nil, + SnapshotMetadata: nil, + Params: nil, + Restore: nil, + } + + cio := &isv1.CreateItemOutput{ + UpdatedItem: nil, + AdditionalItems: nil, + SkipRestore: false, + } + + pi := &isv1.ProgressInput{ + ItemID: velero.ResourceIdentifier{}, + SnapshotID: "", + Backup: nil, + } + po := &isv1.ProgressOutput{ + Phase: isv1.SnapshotPhaseInProgress, + Err: "", + ItemsCompleted: 0, + ItemsToComplete: 0, + Started: time.Time{}, + Updated: time.Time{}, + } + dsi := &isv1.DeleteSnapshotInput{ + SnapshotID: "", + ItemFromBackup: nil, + SnapshotMetadata: nil, + Params: nil, + } + runRestartableDelegateTests( + t, + framework.PluginKindItemSnapshotter, + func(key kindAndName, p RestartableProcess) interface{} { + return &restartableItemSnapshotter{ + key: key, + sharedPluginProcess: p, + } + }, + func() mockable { + return new(mocks.ItemSnapshotter) + }, + restartableDelegateTest{ + function: "Init", + inputs: []interface{}{map[string]string{}}, + expectedErrorOutputs: []interface{}{errors.Errorf("reset error")}, + expectedDelegateOutputs: []interface{}{errors.Errorf("delegate error")}, + }, + restartableDelegateTest{ + function: "AppliesTo", + inputs: []interface{}{}, + expectedErrorOutputs: []interface{}{velero.ResourceSelector{}, errors.Errorf("reset error")}, + expectedDelegateOutputs: []interface{}{velero.ResourceSelector{IncludedNamespaces: []string{"a"}}, errors.Errorf("delegate error")}, + }, + restartableDelegateTest{ + function: "AlsoHandles", + inputs: []interface{}{&isv1.AlsoHandlesInput{}}, + expectedErrorOutputs: []interface{}{[]velero.ResourceIdentifier([]velero.ResourceIdentifier(nil)), errors.Errorf("reset error")}, + expectedDelegateOutputs: []interface{}{[]velero.ResourceIdentifier([]velero.ResourceIdentifier(nil)), errors.Errorf("delegate error")}, + }, + restartableDelegateTest{ + function: "SnapshotItem", + inputs: []interface{}{ctx, sii}, + expectedErrorOutputs: []interface{}{nil, errors.Errorf("reset error")}, + expectedDelegateOutputs: []interface{}{sio, errors.Errorf("delegate error")}, + }, + restartableDelegateTest{ + function: "CreateItemFromSnapshot", + inputs: []interface{}{ctx, cii}, + expectedErrorOutputs: []interface{}{nil, errors.Errorf("reset error")}, + expectedDelegateOutputs: []interface{}{cio, errors.Errorf("delegate error")}, + }, + restartableDelegateTest{ + function: "Progress", + inputs: []interface{}{pi}, + expectedErrorOutputs: []interface{}{nil, errors.Errorf("reset error")}, + expectedDelegateOutputs: []interface{}{po, errors.Errorf("delegate error")}, + }, + restartableDelegateTest{ + function: "DeleteSnapshot", + inputs: []interface{}{ctx, dsi}, + expectedErrorOutputs: []interface{}{errors.Errorf("reset error")}, + expectedDelegateOutputs: []interface{}{errors.Errorf("delegate error")}, + }, + ) +} diff --git a/pkg/plugin/framework/action_resolver.go b/pkg/plugin/framework/action_resolver.go new file mode 100644 index 000000000..9797ba526 --- /dev/null +++ b/pkg/plugin/framework/action_resolver.go @@ -0,0 +1,242 @@ +/* +Copyright the Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + isv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/item_snapshotter/v1" + + "github.com/vmware-tanzu/velero/pkg/discovery" + "github.com/vmware-tanzu/velero/pkg/util/collections" +) + +/* +Velero has a variety of Actions that can be executed on Kubernetes resources. The Actions (BackupItemAction, RestoreItemAction +and others) implement the Applicable interface which returns a ResourceSelector for the Action. The ResourceSelector +can specify namespaces, resource names and labels to include or exclude. The ResourceSelector is resolved into lists +of namespaces and resources present in the backup to be matched against. These lists and the label selector are then used to +decide whether or not the ResolvedAction should be used for a particular resource. +*/ + +// ResolvedAction is an action that has had the namespaces, resources names and labels to include or exclude resolved +type ResolvedAction interface { + // ShouldUse returns true if the resolved namespaces, resource names and labels match those passed in the parameters. + // metadata is optional and may be nil + ShouldUse(groupResource schema.GroupResource, namespace string, metadata metav1.Object, + log logrus.FieldLogger) bool +} + +// resolvedAction is a core struct that holds the resolved namespaces, resource names and labels +type resolvedAction struct { + ResourceIncludesExcludes *collections.IncludesExcludes + NamespaceIncludesExcludes *collections.IncludesExcludes + Selector labels.Selector +} + +func (recv resolvedAction) ShouldUse(groupResource schema.GroupResource, namespace string, metadata metav1.Object, + log logrus.FieldLogger) bool { + if !recv.ResourceIncludesExcludes.ShouldInclude(groupResource.String()) { + log.Debug("Skipping action because it does not apply to this resource") + return false + } + + if namespace != "" && !recv.NamespaceIncludesExcludes.ShouldInclude(namespace) { + log.Debug("Skipping action because it does not apply to this namespace") + return false + } + + if namespace == "" && !recv.NamespaceIncludesExcludes.IncludeEverything() { + log.Debug("Skipping action because resource is cluster-scoped and action only applies to specific namespaces") + return false + } + + if metadata != nil && !recv.Selector.Matches(labels.Set(metadata.GetLabels())) { + log.Debug("Skipping action because label selector does not match") + return false + } + return true +} + +// resolveAction resolves the resources, namespaces and selector into fully-qualified versions +func resolveAction(helper discovery.Helper, action velero.Applicable) (resources *collections.IncludesExcludes, + namespaces *collections.IncludesExcludes, selector labels.Selector, err error) { + resourceSelector, err := action.AppliesTo() + if err != nil { + return nil, nil, nil, err + } + + resources = collections.GetResourceIncludesExcludes(helper, resourceSelector.IncludedResources, resourceSelector.ExcludedResources) + namespaces = collections.NewIncludesExcludes().Includes(resourceSelector.IncludedNamespaces...).Excludes(resourceSelector.ExcludedNamespaces...) + + selector = labels.Everything() + if resourceSelector.LabelSelector != "" { + if selector, err = labels.Parse(resourceSelector.LabelSelector); err != nil { + return nil, nil, nil, err + } + } + + return +} + +type BackupItemResolvedAction struct { + velero.BackupItemAction + resolvedAction +} + +func NewBackupItemActionResolver(actions []velero.BackupItemAction) BackupItemActionResolver { + return BackupItemActionResolver{ + actions: actions, + } +} + +func NewRestoreItemActionResolver(actions []velero.RestoreItemAction) RestoreItemActionResolver { + return RestoreItemActionResolver{ + actions: actions, + } +} + +func NewDeleteItemActionResolver(actions []velero.DeleteItemAction) DeleteItemActionResolver { + return DeleteItemActionResolver{ + actions: actions, + } +} + +func NewItemSnapshotterResolver(actions []isv1.ItemSnapshotter) ItemSnapshotterResolver { + return ItemSnapshotterResolver{ + actions: actions, + } +} + +type ActionResolver interface { + ResolveAction(helper discovery.Helper, action velero.Applicable) (ResolvedAction, error) +} + +type BackupItemActionResolver struct { + actions []velero.BackupItemAction +} + +func (recv BackupItemActionResolver) ResolveActions(helper discovery.Helper) ([]BackupItemResolvedAction, error) { + var resolved []BackupItemResolvedAction + for _, action := range recv.actions { + resources, namespaces, selector, err := resolveAction(helper, action) + if err != nil { + return nil, err + } + res := BackupItemResolvedAction{ + BackupItemAction: action, + resolvedAction: resolvedAction{ + ResourceIncludesExcludes: resources, + NamespaceIncludesExcludes: namespaces, + Selector: selector, + }, + } + resolved = append(resolved, res) + } + return resolved, nil +} + +type RestoreItemResolvedAction struct { + velero.RestoreItemAction + resolvedAction +} + +type RestoreItemActionResolver struct { + actions []velero.RestoreItemAction +} + +func (recv RestoreItemActionResolver) ResolveActions(helper discovery.Helper) ([]RestoreItemResolvedAction, error) { + var resolved []RestoreItemResolvedAction + for _, action := range recv.actions { + resources, namespaces, selector, err := resolveAction(helper, action) + if err != nil { + return nil, err + } + res := RestoreItemResolvedAction{ + RestoreItemAction: action, + resolvedAction: resolvedAction{ + ResourceIncludesExcludes: resources, + NamespaceIncludesExcludes: namespaces, + Selector: selector, + }, + } + resolved = append(resolved, res) + } + return resolved, nil +} + +type DeleteItemResolvedAction struct { + velero.DeleteItemAction + resolvedAction +} + +type DeleteItemActionResolver struct { + actions []velero.DeleteItemAction +} + +func (recv DeleteItemActionResolver) ResolveActions(helper discovery.Helper) ([]DeleteItemResolvedAction, error) { + var resolved []DeleteItemResolvedAction + for _, action := range recv.actions { + resources, namespaces, selector, err := resolveAction(helper, action) + if err != nil { + return nil, err + } + res := DeleteItemResolvedAction{ + DeleteItemAction: action, + resolvedAction: resolvedAction{ + ResourceIncludesExcludes: resources, + NamespaceIncludesExcludes: namespaces, + Selector: selector, + }, + } + resolved = append(resolved, res) + } + return resolved, nil +} + +type ItemSnapshotterResolvedAction struct { + isv1.ItemSnapshotter + resolvedAction +} + +type ItemSnapshotterResolver struct { + actions []isv1.ItemSnapshotter +} + +func (recv ItemSnapshotterResolver) ResolveActions(helper discovery.Helper) ([]ItemSnapshotterResolvedAction, error) { + var resolved []ItemSnapshotterResolvedAction + for _, action := range recv.actions { + resources, namespaces, selector, err := resolveAction(helper, action) + if err != nil { + return nil, err + } + res := ItemSnapshotterResolvedAction{ + ItemSnapshotter: action, + resolvedAction: resolvedAction{ + ResourceIncludesExcludes: resources, + NamespaceIncludesExcludes: namespaces, + Selector: selector, + }, + } + resolved = append(resolved, res) + } + return resolved, nil +} diff --git a/pkg/plugin/framework/action_resolver_test.go b/pkg/plugin/framework/action_resolver_test.go new file mode 100644 index 000000000..cf0e411da --- /dev/null +++ b/pkg/plugin/framework/action_resolver_test.go @@ -0,0 +1,93 @@ +/* +Copyright the Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "testing" + + "k8s.io/apimachinery/pkg/labels" + + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + velerotest "github.com/vmware-tanzu/velero/pkg/test" +) + +type mockApplicable struct { + selector velero.ResourceSelector +} + +func (recv mockApplicable) AppliesTo() (velero.ResourceSelector, error) { + return recv.selector, nil +} + +func TestActionResolverNamespace(t *testing.T) { + discoveryHelper := velerotest.NewFakeDiscoveryHelper(false, map[schema.GroupVersionResource]schema.GroupVersionResource{}) + namespaceMatchApplicable := mockApplicable{ + selector: velero.ResourceSelector{ + IncludedNamespaces: []string{"default"}, + }, + } + resources, namespaces, selector, err := resolveAction(discoveryHelper, namespaceMatchApplicable) + require.NoError(t, err) + require.Equal(t, []string{"default"}, namespaces.GetIncludes()) + require.Empty(t, namespaces.GetExcludes()) + require.Empty(t, resources.GetIncludes()) + require.Empty(t, resources.GetExcludes()) + require.True(t, selector.Empty()) +} + +func TestActionResolverResource(t *testing.T) { + pvGVR := schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "persistentvolumes", + } + discoveryHelper := velerotest.NewFakeDiscoveryHelper(false, map[schema.GroupVersionResource]schema.GroupVersionResource{pvGVR: pvGVR}) + namespaceMatchApplicable := mockApplicable{ + selector: velero.ResourceSelector{ + IncludedResources: []string{"persistentvolumes"}, + }, + } + resources, namespaces, selector, err := resolveAction(discoveryHelper, namespaceMatchApplicable) + require.NoError(t, err) + require.Empty(t, namespaces.GetIncludes()) + require.Empty(t, namespaces.GetExcludes()) + require.True(t, resources.ShouldInclude("persistentvolumes")) + require.Empty(t, resources.GetExcludes()) + require.True(t, selector.Empty()) +} + +func TestActionResolverLabel(t *testing.T) { + discoveryHelper := velerotest.NewFakeDiscoveryHelper(false, map[schema.GroupVersionResource]schema.GroupVersionResource{}) + namespaceMatchApplicable := mockApplicable{ + selector: velero.ResourceSelector{ + LabelSelector: "myLabel=true", + }, + } + checkLabel, err := labels.ConvertSelectorToLabelsMap("myLabel=true") + require.NoError(t, err) + + resources, namespaces, selector, err := resolveAction(discoveryHelper, namespaceMatchApplicable) + require.NoError(t, err) + require.Empty(t, namespaces.GetIncludes()) + require.Empty(t, namespaces.GetExcludes()) + require.Empty(t, resources.GetIncludes()) + require.Empty(t, resources.GetExcludes()) + require.True(t, selector.Matches(checkLabel)) +} diff --git a/pkg/plugin/framework/item_snapshotter.go b/pkg/plugin/framework/item_snapshotter.go new file mode 100644 index 000000000..6e6f91439 --- /dev/null +++ b/pkg/plugin/framework/item_snapshotter.go @@ -0,0 +1,44 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + plugin "github.com/hashicorp/go-plugin" + "golang.org/x/net/context" + "google.golang.org/grpc" + + proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" +) + +// ItemSnapshotterPlugin is an implementation of go-plugin's Plugin +// interface with support for gRPC for the ItemSnapshotter +// interface. +type ItemSnapshotterPlugin struct { + plugin.NetRPCUnsupportedPlugin + *pluginBase +} + +// GRPCClient returns a clientDispenser for ItemSnapshotter gRPC clients. +func (p *ItemSnapshotterPlugin) GRPCClient(_ context.Context, _ *plugin.GRPCBroker, clientConn *grpc.ClientConn) (interface{}, error) { + return newClientDispenser(p.clientLogger, clientConn, newItemSnapshotterGRPCClient), nil +} + +// GRPCServer registers an ItemSnapshotter gRPC server. +func (p *ItemSnapshotterPlugin) GRPCServer(_ *plugin.GRPCBroker, server *grpc.Server) error { + proto.RegisterItemSnapshotterServer(server, &ItemSnapshotterGRPCServer{mux: p.serverMux}) + return nil +} diff --git a/pkg/plugin/framework/item_snapshotter_client.go b/pkg/plugin/framework/item_snapshotter_client.go new file mode 100644 index 000000000..dd341b966 --- /dev/null +++ b/pkg/plugin/framework/item_snapshotter_client.go @@ -0,0 +1,240 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + "encoding/json" + "time" + + isv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/item_snapshotter/v1" + + "github.com/pkg/errors" + "google.golang.org/grpc" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" +) + +// NewItemSnapshotterPlugin constructs a ItemSnapshotterPlugin. +func NewItemSnapshotterPlugin(options ...PluginOption) *ItemSnapshotterPlugin { + return &ItemSnapshotterPlugin{ + pluginBase: newPluginBase(options...), + } +} + +func newItemSnapshotterGRPCClient(base *clientBase, clientConn *grpc.ClientConn) interface{} { + return &ItemSnapshotterGRPCClient{ + clientBase: base, + grpcClient: proto.NewItemSnapshotterClient(clientConn), + } +} + +// ItemSnapshotterGRPCClient implements the ItemSnapshotter interface and uses a +// gRPC client to make calls to the plugin server. +type ItemSnapshotterGRPCClient struct { + *clientBase + grpcClient proto.ItemSnapshotterClient +} + +func (recv ItemSnapshotterGRPCClient) Init(config map[string]string) error { + req := &proto.ItemSnapshotterInitRequest{ + Plugin: recv.plugin, + Config: config, + } + + _, err := recv.grpcClient.Init(context.Background(), req) + return err +} + +func (recv ItemSnapshotterGRPCClient) AppliesTo() (velero.ResourceSelector, error) { + req := &proto.ItemSnapshotterAppliesToRequest{ + Plugin: recv.plugin, + } + + res, err := recv.grpcClient.AppliesTo(context.Background(), req) + if err != nil { + return velero.ResourceSelector{}, fromGRPCError(err) + } + + if res.ResourceSelector == nil { + return velero.ResourceSelector{}, nil + } + + return velero.ResourceSelector{ + IncludedNamespaces: res.ResourceSelector.IncludedNamespaces, + ExcludedNamespaces: res.ResourceSelector.ExcludedNamespaces, + IncludedResources: res.ResourceSelector.IncludedResources, + ExcludedResources: res.ResourceSelector.ExcludedResources, + LabelSelector: res.ResourceSelector.Selector, + }, nil +} + +func (recv ItemSnapshotterGRPCClient) AlsoHandles(input *isv1.AlsoHandlesInput) ([]velero.ResourceIdentifier, error) { + itemJSON, err := json.Marshal(input.Item.UnstructuredContent()) + if err != nil { + return nil, errors.WithStack(err) + } + + backupJSON, err := json.Marshal(input.Backup) + if err != nil { + return nil, errors.WithStack(err) + } + req := &proto.AlsoHandlesRequest{ + Plugin: recv.plugin, + Item: itemJSON, + Backup: backupJSON, + } + res, err := recv.grpcClient.AlsoHandles(context.Background(), req) + if err != nil { + return nil, errors.WithStack(err) + } + + handledItems := unpackResourceIdentifiers(res.HandledItems) + + return handledItems, nil +} + +func (recv ItemSnapshotterGRPCClient) SnapshotItem(ctx context.Context, input *isv1.SnapshotItemInput) (*isv1.SnapshotItemOutput, error) { + itemJSON, err := json.Marshal(input.Item.UnstructuredContent()) + if err != nil { + return nil, errors.WithStack(err) + } + + backupJSON, err := json.Marshal(input.Backup) + if err != nil { + return nil, errors.WithStack(err) + } + req := &proto.SnapshotItemRequest{ + Plugin: recv.plugin, + Item: itemJSON, + Backup: backupJSON, + } + res, err := recv.grpcClient.SnapshotItem(ctx, req) + if err != nil { + return nil, errors.WithStack(err) + } + var updatedItem unstructured.Unstructured + if err := json.Unmarshal(res.Item, &updatedItem); err != nil { + return nil, errors.WithStack(err) + } + + additionalItems := unpackResourceIdentifiers(res.AdditionalItems) + handledItems := unpackResourceIdentifiers(res.HandledItems) + + sio := isv1.SnapshotItemOutput{ + UpdatedItem: &updatedItem, + SnapshotID: res.SnapshotID, + SnapshotMetadata: res.SnapshotMetadata, + AdditionalItems: additionalItems, + HandledItems: handledItems, + } + return &sio, nil +} + +func (recv ItemSnapshotterGRPCClient) Progress(input *isv1.ProgressInput) (*isv1.ProgressOutput, error) { + backupJSON, err := json.Marshal(input.Backup) + if err != nil { + return nil, errors.WithStack(err) + } + req := &proto.ProgressRequest{ + Plugin: recv.plugin, + ItemID: resourceIdentifierToProto(input.ItemID), + SnapshotID: input.SnapshotID, + Backup: backupJSON, + } + + res, err := recv.grpcClient.Progress(context.Background(), req) + + if err != nil { + return nil, errors.WithStack(err) + } + // Validate phase + + phase, err := isv1.SnapshotPhaseFromString(res.Phase) + if err != nil { + return nil, errors.WithStack(err) + } + + up := isv1.ProgressOutput{ + Phase: phase, + Err: res.Err, + ItemsCompleted: res.ItemsCompleted, + ItemsToComplete: res.ItemsToComplete, + Started: time.Unix(res.Started, res.StartedNano), + Updated: time.Unix(res.Updated, res.UpdatedNano), + } + return &up, nil +} + +func (recv ItemSnapshotterGRPCClient) DeleteSnapshot(ctx context.Context, input *isv1.DeleteSnapshotInput) error { + req := &proto.DeleteItemSnapshotRequest{ + Plugin: recv.plugin, + Params: input.Params, + SnapshotID: input.SnapshotID, + } + _, err := recv.grpcClient.DeleteSnapshot(ctx, req) // Returns Empty as first arg so just ignore + + if err != nil { + return errors.WithStack(err) + } + return nil +} + +func (recv ItemSnapshotterGRPCClient) CreateItemFromSnapshot(ctx context.Context, input *isv1.CreateItemInput) (*isv1.CreateItemOutput, error) { + itemJSON, err := json.Marshal(input.SnapshottedItem.UnstructuredContent()) + if err != nil { + return nil, errors.WithStack(err) + } + itemFromBackupJSON, err := json.Marshal(input.ItemFromBackup.UnstructuredContent()) + if err != nil { + return nil, errors.WithStack(err) + } + restoreJSON, err := json.Marshal(input.Restore) + if err != nil { + return nil, errors.WithStack(err) + } + req := &proto.CreateItemFromSnapshotRequest{ + Plugin: recv.plugin, + Item: itemJSON, + SnapshotID: input.SnapshotID, + ItemFromBackup: itemFromBackupJSON, + SnapshotMetadata: input.SnapshotMetadata, + Params: input.Params, + Restore: restoreJSON, + } + + res, err := recv.grpcClient.CreateItemFromSnapshot(ctx, req) + if err != nil { + return nil, errors.WithStack(err) + } + + var updatedItem unstructured.Unstructured + if err := json.Unmarshal(res.Item, &updatedItem); err != nil { + return nil, errors.WithStack(err) + } + + additionalItems := unpackResourceIdentifiers(res.AdditionalItems) + + cio := isv1.CreateItemOutput{ + UpdatedItem: &updatedItem, + AdditionalItems: additionalItems, + SkipRestore: res.SkipRestore, + } + return &cio, nil +} diff --git a/pkg/plugin/framework/item_snapshotter_server.go b/pkg/plugin/framework/item_snapshotter_server.go new file mode 100644 index 000000000..166e6d07f --- /dev/null +++ b/pkg/plugin/framework/item_snapshotter_server.go @@ -0,0 +1,311 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package framework + +import ( + "context" + "encoding/json" + + isv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/item_snapshotter/v1" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + + "github.com/pkg/errors" + + proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" +) + +// ItemSnapshotterGRPCServer implements the proto-generated ItemSnapshotterServer interface, and accepts +// gRPC calls and forwards them to an implementation of the pluggable interface. +type ItemSnapshotterGRPCServer struct { + mux *serverMux +} + +func (recv *ItemSnapshotterGRPCServer) getImpl(name string) (isv1.ItemSnapshotter, error) { + impl, err := recv.mux.getHandler(name) + if err != nil { + return nil, err + } + + itemAction, ok := impl.(isv1.ItemSnapshotter) + if !ok { + return nil, errors.Errorf("%T is not an item snapshotter", impl) + } + + return itemAction, nil +} + +func (recv *ItemSnapshotterGRPCServer) Init(c context.Context, req *proto.ItemSnapshotterInitRequest) (response *proto.Empty, err error) { + defer func() { + if recoveredErr := handlePanic(recover()); recoveredErr != nil { + err = recoveredErr + } + }() + + impl, err := recv.getImpl(req.Plugin) + if err != nil { + return nil, newGRPCError(err) + } + + err = impl.Init(req.Config) + if err != nil { + return nil, newGRPCError(err) + } + + return &proto.Empty{}, nil +} + +func (recv *ItemSnapshotterGRPCServer) AppliesTo(ctx context.Context, req *proto.ItemSnapshotterAppliesToRequest) (response *proto.ItemSnapshotterAppliesToResponse, err error) { + defer func() { + if recoveredErr := handlePanic(recover()); recoveredErr != nil { + err = recoveredErr + } + }() + + impl, err := recv.getImpl(req.Plugin) + if err != nil { + return nil, newGRPCError(err) + } + + resourceSelector, err := impl.AppliesTo() + if err != nil { + return nil, newGRPCError(err) + } + + return &proto.ItemSnapshotterAppliesToResponse{ + &proto.ResourceSelector{ + IncludedNamespaces: resourceSelector.IncludedNamespaces, + ExcludedNamespaces: resourceSelector.ExcludedNamespaces, + IncludedResources: resourceSelector.IncludedResources, + ExcludedResources: resourceSelector.ExcludedResources, + Selector: resourceSelector.LabelSelector, + }, + }, nil +} + +func (recv *ItemSnapshotterGRPCServer) AlsoHandles(ctx context.Context, req *proto.AlsoHandlesRequest) (res *proto.AlsoHandlesResponse, err error) { + defer func() { + if recoveredErr := handlePanic(recover()); recoveredErr != nil { + err = recoveredErr + } + }() + + impl, err := recv.getImpl(req.Plugin) + if err != nil { + return nil, newGRPCError(err) + } + var item unstructured.Unstructured + var backup api.Backup + + if err := json.Unmarshal(req.Item, &item); err != nil { + return nil, newGRPCError(errors.WithStack(err)) + } + if err := json.Unmarshal(req.Backup, &backup); err != nil { + return nil, newGRPCError(errors.WithStack(err)) + } + ahi := isv1.AlsoHandlesInput{ + Item: &item, + Backup: &backup, + } + alsoHandles, err := impl.AlsoHandles(&ahi) + if err != nil { + return nil, newGRPCError(err) + } + res = &proto.AlsoHandlesResponse{} + + for _, item := range alsoHandles { + res.HandledItems = append(res.HandledItems, resourceIdentifierToProto(item)) + } + return res, nil +} + +func (recv *ItemSnapshotterGRPCServer) SnapshotItem(ctx context.Context, req *proto.SnapshotItemRequest) (res *proto.SnapshotItemResponse, err error) { + defer func() { + if recoveredErr := handlePanic(recover()); recoveredErr != nil { + err = recoveredErr + } + }() + + impl, err := recv.getImpl(req.Plugin) + if err != nil { + return nil, newGRPCError(err) + } + var item unstructured.Unstructured + var backup api.Backup + + if err := json.Unmarshal(req.Item, &item); err != nil { + return nil, newGRPCError(errors.WithStack(err)) + } + if err := json.Unmarshal(req.Backup, &backup); err != nil { + return nil, newGRPCError(errors.WithStack(err)) + } + sii := isv1.SnapshotItemInput{ + Item: &item, + Params: req.Params, + Backup: &backup, + } + sio, err := impl.SnapshotItem(ctx, &sii) + + // If the plugin implementation returned a nil updatedItem (meaning no modifications), reset updatedItem to the + // original item. + var updatedItemJSON []byte + if sio.UpdatedItem == nil { + updatedItemJSON = req.Item + } else { + updatedItemJSON, err = json.Marshal(sio.UpdatedItem.UnstructuredContent()) + if err != nil { + return nil, newGRPCError(errors.WithStack(err)) + } + } + res = &proto.SnapshotItemResponse{ + Item: updatedItemJSON, + SnapshotID: sio.SnapshotID, + SnapshotMetadata: sio.SnapshotMetadata, + } + res.AdditionalItems = packResourceIdentifiers(sio.AdditionalItems) + res.HandledItems = packResourceIdentifiers(sio.HandledItems) + return res, err +} + +func (recv *ItemSnapshotterGRPCServer) Progress(ctx context.Context, req *proto.ProgressRequest) (res *proto.ProgressResponse, err error) { + defer func() { + if recoveredErr := handlePanic(recover()); recoveredErr != nil { + err = recoveredErr + } + }() + impl, err := recv.getImpl(req.Plugin) + if err != nil { + return nil, newGRPCError(err) + } + var backup api.Backup + + if err := json.Unmarshal(req.Backup, &backup); err != nil { + return nil, newGRPCError(errors.WithStack(err)) + } + sipi := &isv1.ProgressInput{ + ItemID: protoToResourceIdentifier(req.ItemID), + SnapshotID: req.SnapshotID, + Backup: &backup, + } + + sipo, err := impl.Progress(sipi) + if err != nil { + return nil, newGRPCError(err) + } + + res = &proto.ProgressResponse{ + Phase: string(sipo.Phase), + ItemsCompleted: sipo.ItemsCompleted, + ItemsToComplete: sipo.ItemsToComplete, + Started: sipo.Started.Unix(), + StartedNano: sipo.Started.UnixNano(), + Updated: sipo.Updated.Unix(), + UpdatedNano: sipo.Updated.UnixNano(), + Err: sipo.Err, + } + return res, nil +} + +func (recv *ItemSnapshotterGRPCServer) DeleteSnapshot(ctx context.Context, req *proto.DeleteItemSnapshotRequest) (empty *proto.Empty, err error) { + defer func() { + if recoveredErr := handlePanic(recover()); recoveredErr != nil { + err = recoveredErr + } + }() + impl, err := recv.getImpl(req.Plugin) + if err != nil { + return nil, newGRPCError(err) + } + + var itemFromBackup unstructured.Unstructured + if err := json.Unmarshal(req.ItemFromBackup, &itemFromBackup); err != nil { + return nil, newGRPCError(errors.WithStack(err)) + } + + disi := isv1.DeleteSnapshotInput{ + SnapshotID: req.SnapshotID, + ItemFromBackup: &itemFromBackup, + SnapshotMetadata: req.Metadata, + Params: req.Params, + } + + err = impl.DeleteSnapshot(ctx, &disi) + if err != nil { + return nil, newGRPCError(err) + } + return +} + +func (recv *ItemSnapshotterGRPCServer) CreateItemFromSnapshot(ctx context.Context, req *proto.CreateItemFromSnapshotRequest) (res *proto.CreateItemFromSnapshotResponse, err error) { + defer func() { + if recoveredErr := handlePanic(recover()); recoveredErr != nil { + err = recoveredErr + } + }() + impl, err := recv.getImpl(req.Plugin) + if err != nil { + return nil, newGRPCError(err) + } + + var snapshottedItem unstructured.Unstructured + if err := json.Unmarshal(req.Item, &snapshottedItem); err != nil { + return nil, newGRPCError(errors.WithStack(err)) + } + + var itemFromBackup unstructured.Unstructured + if err := json.Unmarshal(req.Item, &itemFromBackup); err != nil { + return nil, newGRPCError(errors.WithStack(err)) + } + + var restore api.Restore + + if err := json.Unmarshal(req.Restore, &restore); err != nil { + return nil, newGRPCError(errors.WithStack(err)) + } + + cii := isv1.CreateItemInput{ + SnapshottedItem: &snapshottedItem, + SnapshotID: req.SnapshotID, + ItemFromBackup: &itemFromBackup, + SnapshotMetadata: req.SnapshotMetadata, + Params: req.Params, + Restore: &restore, + } + + cio, err := impl.CreateItemFromSnapshot(ctx, &cii) + if err != nil { + return nil, newGRPCError(err) + } + + var updatedItemJSON []byte + if cio.UpdatedItem == nil { + updatedItemJSON = req.Item + } else { + updatedItemJSON, err = json.Marshal(cio.UpdatedItem.UnstructuredContent()) + if err != nil { + return nil, newGRPCError(errors.WithStack(err)) + } + } + res = &proto.CreateItemFromSnapshotResponse{ + Item: updatedItemJSON, + SkipRestore: cio.SkipRestore, + } + res.AdditionalItems = packResourceIdentifiers(cio.AdditionalItems) + + return +} diff --git a/pkg/plugin/framework/plugin_kinds.go b/pkg/plugin/framework/plugin_kinds.go index 640ae30b8..f48f7f498 100644 --- a/pkg/plugin/framework/plugin_kinds.go +++ b/pkg/plugin/framework/plugin_kinds.go @@ -41,6 +41,9 @@ const ( // PluginKindDeleteItemAction represents a delete item action plugin. PluginKindDeleteItemAction PluginKind = "DeleteItemAction" + // PluginKindItemSnapshotter represents an item snapshotter plugin + PluginKindItemSnapshotter PluginKind = "ItemSnapshotter" + // PluginKindPluginLister represents a plugin lister plugin. PluginKindPluginLister PluginKind = "PluginLister" ) @@ -54,5 +57,6 @@ func AllPluginKinds() map[string]PluginKind { allPluginKinds[PluginKindBackupItemAction.String()] = PluginKindBackupItemAction allPluginKinds[PluginKindRestoreItemAction.String()] = PluginKindRestoreItemAction allPluginKinds[PluginKindDeleteItemAction.String()] = PluginKindDeleteItemAction + allPluginKinds[PluginKindItemSnapshotter.String()] = PluginKindItemSnapshotter return allPluginKinds } diff --git a/pkg/plugin/framework/plugin_types_test.go b/pkg/plugin/framework/plugin_types_test.go index f9b4d1f80..2ebb78104 100644 --- a/pkg/plugin/framework/plugin_types_test.go +++ b/pkg/plugin/framework/plugin_types_test.go @@ -30,6 +30,7 @@ func TestPluginImplementationsAreGRPCPlugins(t *testing.T) { new(ObjectStorePlugin), new(PluginListerPlugin), new(RestoreItemActionPlugin), + new(ItemSnapshotterPlugin), } for _, impl := range pluginImpls { diff --git a/pkg/plugin/framework/server.go b/pkg/plugin/framework/server.go index f02757e4b..066a44db6 100644 --- a/pkg/plugin/framework/server.go +++ b/pkg/plugin/framework/server.go @@ -74,6 +74,10 @@ type Server interface { // RegisterDeleteItemActions registers multiple Delete item actions. RegisterDeleteItemActions(map[string]HandlerInitializer) Server + RegisterItemSnapshotter(pluginName string, initializer HandlerInitializer) Server + + // RegisterItemSnapshotters registers multiple Item Snapshotters + RegisterItemSnapshotters(map[string]HandlerInitializer) Server // Server runs the plugin server. Serve() } @@ -89,6 +93,7 @@ type server struct { objectStore *ObjectStorePlugin restoreItemAction *RestoreItemActionPlugin deleteItemAction *DeleteItemActionPlugin + itemSnapshotter *ItemSnapshotterPlugin } // NewServer returns a new Server @@ -105,6 +110,7 @@ func NewServer() Server { objectStore: NewObjectStorePlugin(serverLogger(log)), restoreItemAction: NewRestoreItemActionPlugin(serverLogger(log)), deleteItemAction: NewDeleteItemActionPlugin(serverLogger(log)), + itemSnapshotter: NewItemSnapshotterPlugin(serverLogger(log)), } } @@ -177,6 +183,17 @@ func (s *server) RegisterDeleteItemActions(m map[string]HandlerInitializer) Serv return s } +func (s *server) RegisterItemSnapshotter(name string, initializer HandlerInitializer) Server { + s.itemSnapshotter.register(name, initializer) + return s +} +func (s *server) RegisterItemSnapshotters(m map[string]HandlerInitializer) Server { + for name := range m { + s.RegisterItemSnapshotter(name, m[name]) + } + return s +} + // getNames returns a list of PluginIdentifiers registered with plugin. func getNames(command string, kind PluginKind, plugin Interface) []PluginIdentifier { var pluginIdentifiers []PluginIdentifier @@ -206,6 +223,7 @@ func (s *server) Serve() { pluginIdentifiers = append(pluginIdentifiers, getNames(command, PluginKindObjectStore, s.objectStore)...) pluginIdentifiers = append(pluginIdentifiers, getNames(command, PluginKindRestoreItemAction, s.restoreItemAction)...) pluginIdentifiers = append(pluginIdentifiers, getNames(command, PluginKindDeleteItemAction, s.deleteItemAction)...) + pluginIdentifiers = append(pluginIdentifiers, getNames(command, PluginKindItemSnapshotter, s.itemSnapshotter)...) pluginLister := NewPluginLister(pluginIdentifiers...) @@ -218,6 +236,7 @@ func (s *server) Serve() { string(PluginKindPluginLister): NewPluginListerPlugin(pluginLister), string(PluginKindRestoreItemAction): s.restoreItemAction, string(PluginKindDeleteItemAction): s.deleteItemAction, + string(PluginKindItemSnapshotter): s.itemSnapshotter, }, GRPCServer: plugin.DefaultGRPCServer, }) diff --git a/pkg/plugin/framework/util.go b/pkg/plugin/framework/util.go new file mode 100644 index 000000000..2ac0b1938 --- /dev/null +++ b/pkg/plugin/framework/util.go @@ -0,0 +1,58 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + + proto "github.com/vmware-tanzu/velero/pkg/plugin/generated" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" +) + +func packResourceIdentifiers(resourcesIDs []velero.ResourceIdentifier) (protoIDs []*proto.ResourceIdentifier) { + for _, item := range resourcesIDs { + protoIDs = append(protoIDs, resourceIdentifierToProto(item)) + } + return +} + +func unpackResourceIdentifiers(protoIDs []*proto.ResourceIdentifier) (resourceIDs []velero.ResourceIdentifier) { + for _, itm := range protoIDs { + resourceIDs = append(resourceIDs, protoToResourceIdentifier(itm)) + } + return +} + +func protoToResourceIdentifier(proto *proto.ResourceIdentifier) velero.ResourceIdentifier { + return velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{ + Group: proto.Group, + Resource: proto.Resource, + }, + Namespace: proto.Namespace, + Name: proto.Name, + } +} + +func resourceIdentifierToProto(id velero.ResourceIdentifier) *proto.ResourceIdentifier { + return &proto.ResourceIdentifier{ + Group: id.Group, + Resource: id.Resource, + Namespace: id.Namespace, + Name: id.Name, + } +} diff --git a/pkg/plugin/generated/BackupItemAction.pb.go b/pkg/plugin/generated/BackupItemAction.pb.go index 9e3afd9fc..937da01d0 100644 --- a/pkg/plugin/generated/BackupItemAction.pb.go +++ b/pkg/plugin/generated/BackupItemAction.pb.go @@ -7,6 +7,7 @@ Package generated is a generated protocol buffer package. It is generated from these files: BackupItemAction.proto DeleteItemAction.proto + ItemSnapshotter.proto ObjectStore.proto PluginLister.proto RestoreItemAction.proto @@ -21,6 +22,18 @@ It has these top-level messages: DeleteItemActionExecuteRequest DeleteItemActionAppliesToRequest DeleteItemActionAppliesToResponse + ItemSnapshotterAppliesToRequest + ItemSnapshotterAppliesToResponse + AlsoHandlesRequest + AlsoHandlesResponse + SnapshotItemRequest + SnapshotItemResponse + ProgressRequest + ProgressResponse + DeleteItemSnapshotRequest + CreateItemFromSnapshotRequest + CreateItemFromSnapshotResponse + ItemSnapshotterInitRequest PutObjectRequest ObjectExistsRequest ObjectExistsResponse diff --git a/pkg/plugin/generated/ItemSnapshotter.pb.go b/pkg/plugin/generated/ItemSnapshotter.pb.go new file mode 100644 index 000000000..dfcf73c87 --- /dev/null +++ b/pkg/plugin/generated/ItemSnapshotter.pb.go @@ -0,0 +1,819 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: ItemSnapshotter.proto + +package generated + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type ItemSnapshotterAppliesToRequest struct { + Plugin string `protobuf:"bytes,1,opt,name=plugin" json:"plugin,omitempty"` +} + +func (m *ItemSnapshotterAppliesToRequest) Reset() { *m = ItemSnapshotterAppliesToRequest{} } +func (m *ItemSnapshotterAppliesToRequest) String() string { return proto.CompactTextString(m) } +func (*ItemSnapshotterAppliesToRequest) ProtoMessage() {} +func (*ItemSnapshotterAppliesToRequest) Descriptor() ([]byte, []int) { + return fileDescriptor2, []int{0} +} + +func (m *ItemSnapshotterAppliesToRequest) GetPlugin() string { + if m != nil { + return m.Plugin + } + return "" +} + +type ItemSnapshotterAppliesToResponse struct { + ResourceSelector *ResourceSelector `protobuf:"bytes,1,opt,name=ResourceSelector" json:"ResourceSelector,omitempty"` +} + +func (m *ItemSnapshotterAppliesToResponse) Reset() { *m = ItemSnapshotterAppliesToResponse{} } +func (m *ItemSnapshotterAppliesToResponse) String() string { return proto.CompactTextString(m) } +func (*ItemSnapshotterAppliesToResponse) ProtoMessage() {} +func (*ItemSnapshotterAppliesToResponse) Descriptor() ([]byte, []int) { + return fileDescriptor2, []int{1} +} + +func (m *ItemSnapshotterAppliesToResponse) GetResourceSelector() *ResourceSelector { + if m != nil { + return m.ResourceSelector + } + return nil +} + +type AlsoHandlesRequest struct { + Plugin string `protobuf:"bytes,1,opt,name=plugin" json:"plugin,omitempty"` + Item []byte `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` + Backup []byte `protobuf:"bytes,3,opt,name=backup,proto3" json:"backup,omitempty"` +} + +func (m *AlsoHandlesRequest) Reset() { *m = AlsoHandlesRequest{} } +func (m *AlsoHandlesRequest) String() string { return proto.CompactTextString(m) } +func (*AlsoHandlesRequest) ProtoMessage() {} +func (*AlsoHandlesRequest) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{2} } + +func (m *AlsoHandlesRequest) GetPlugin() string { + if m != nil { + return m.Plugin + } + return "" +} + +func (m *AlsoHandlesRequest) GetItem() []byte { + if m != nil { + return m.Item + } + return nil +} + +func (m *AlsoHandlesRequest) GetBackup() []byte { + if m != nil { + return m.Backup + } + return nil +} + +type AlsoHandlesResponse struct { + HandledItems []*ResourceIdentifier `protobuf:"bytes,1,rep,name=handledItems" json:"handledItems,omitempty"` +} + +func (m *AlsoHandlesResponse) Reset() { *m = AlsoHandlesResponse{} } +func (m *AlsoHandlesResponse) String() string { return proto.CompactTextString(m) } +func (*AlsoHandlesResponse) ProtoMessage() {} +func (*AlsoHandlesResponse) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{3} } + +func (m *AlsoHandlesResponse) GetHandledItems() []*ResourceIdentifier { + if m != nil { + return m.HandledItems + } + return nil +} + +type SnapshotItemRequest struct { + Plugin string `protobuf:"bytes,1,opt,name=plugin" json:"plugin,omitempty"` + Item []byte `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` + Params map[string]string `protobuf:"bytes,3,rep,name=params" json:"params,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Backup []byte `protobuf:"bytes,4,opt,name=backup,proto3" json:"backup,omitempty"` +} + +func (m *SnapshotItemRequest) Reset() { *m = SnapshotItemRequest{} } +func (m *SnapshotItemRequest) String() string { return proto.CompactTextString(m) } +func (*SnapshotItemRequest) ProtoMessage() {} +func (*SnapshotItemRequest) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{4} } + +func (m *SnapshotItemRequest) GetPlugin() string { + if m != nil { + return m.Plugin + } + return "" +} + +func (m *SnapshotItemRequest) GetItem() []byte { + if m != nil { + return m.Item + } + return nil +} + +func (m *SnapshotItemRequest) GetParams() map[string]string { + if m != nil { + return m.Params + } + return nil +} + +func (m *SnapshotItemRequest) GetBackup() []byte { + if m != nil { + return m.Backup + } + return nil +} + +type SnapshotItemResponse struct { + Item []byte `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"` + SnapshotID string `protobuf:"bytes,2,opt,name=snapshotID" json:"snapshotID,omitempty"` + SnapshotMetadata map[string]string `protobuf:"bytes,3,rep,name=snapshotMetadata" json:"snapshotMetadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + AdditionalItems []*ResourceIdentifier `protobuf:"bytes,4,rep,name=additionalItems" json:"additionalItems,omitempty"` + HandledItems []*ResourceIdentifier `protobuf:"bytes,5,rep,name=handledItems" json:"handledItems,omitempty"` +} + +func (m *SnapshotItemResponse) Reset() { *m = SnapshotItemResponse{} } +func (m *SnapshotItemResponse) String() string { return proto.CompactTextString(m) } +func (*SnapshotItemResponse) ProtoMessage() {} +func (*SnapshotItemResponse) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{5} } + +func (m *SnapshotItemResponse) GetItem() []byte { + if m != nil { + return m.Item + } + return nil +} + +func (m *SnapshotItemResponse) GetSnapshotID() string { + if m != nil { + return m.SnapshotID + } + return "" +} + +func (m *SnapshotItemResponse) GetSnapshotMetadata() map[string]string { + if m != nil { + return m.SnapshotMetadata + } + return nil +} + +func (m *SnapshotItemResponse) GetAdditionalItems() []*ResourceIdentifier { + if m != nil { + return m.AdditionalItems + } + return nil +} + +func (m *SnapshotItemResponse) GetHandledItems() []*ResourceIdentifier { + if m != nil { + return m.HandledItems + } + return nil +} + +type ProgressRequest struct { + Plugin string `protobuf:"bytes,1,opt,name=plugin" json:"plugin,omitempty"` + ItemID *ResourceIdentifier `protobuf:"bytes,2,opt,name=itemID" json:"itemID,omitempty"` + SnapshotID string `protobuf:"bytes,3,opt,name=snapshotID" json:"snapshotID,omitempty"` + Backup []byte `protobuf:"bytes,4,opt,name=backup,proto3" json:"backup,omitempty"` +} + +func (m *ProgressRequest) Reset() { *m = ProgressRequest{} } +func (m *ProgressRequest) String() string { return proto.CompactTextString(m) } +func (*ProgressRequest) ProtoMessage() {} +func (*ProgressRequest) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{6} } + +func (m *ProgressRequest) GetPlugin() string { + if m != nil { + return m.Plugin + } + return "" +} + +func (m *ProgressRequest) GetItemID() *ResourceIdentifier { + if m != nil { + return m.ItemID + } + return nil +} + +func (m *ProgressRequest) GetSnapshotID() string { + if m != nil { + return m.SnapshotID + } + return "" +} + +func (m *ProgressRequest) GetBackup() []byte { + if m != nil { + return m.Backup + } + return nil +} + +type ProgressResponse struct { + Phase string `protobuf:"bytes,1,opt,name=phase" json:"phase,omitempty"` + ItemsCompleted int64 `protobuf:"varint,2,opt,name=itemsCompleted" json:"itemsCompleted,omitempty"` + ItemsToComplete int64 `protobuf:"varint,3,opt,name=itemsToComplete" json:"itemsToComplete,omitempty"` + Started int64 `protobuf:"varint,4,opt,name=started" json:"started,omitempty"` + StartedNano int64 `protobuf:"varint,5,opt,name=startedNano" json:"startedNano,omitempty"` + Updated int64 `protobuf:"varint,6,opt,name=updated" json:"updated,omitempty"` + UpdatedNano int64 `protobuf:"varint,7,opt,name=updatedNano" json:"updatedNano,omitempty"` + Err string `protobuf:"bytes,8,opt,name=err" json:"err,omitempty"` +} + +func (m *ProgressResponse) Reset() { *m = ProgressResponse{} } +func (m *ProgressResponse) String() string { return proto.CompactTextString(m) } +func (*ProgressResponse) ProtoMessage() {} +func (*ProgressResponse) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{7} } + +func (m *ProgressResponse) GetPhase() string { + if m != nil { + return m.Phase + } + return "" +} + +func (m *ProgressResponse) GetItemsCompleted() int64 { + if m != nil { + return m.ItemsCompleted + } + return 0 +} + +func (m *ProgressResponse) GetItemsToComplete() int64 { + if m != nil { + return m.ItemsToComplete + } + return 0 +} + +func (m *ProgressResponse) GetStarted() int64 { + if m != nil { + return m.Started + } + return 0 +} + +func (m *ProgressResponse) GetStartedNano() int64 { + if m != nil { + return m.StartedNano + } + return 0 +} + +func (m *ProgressResponse) GetUpdated() int64 { + if m != nil { + return m.Updated + } + return 0 +} + +func (m *ProgressResponse) GetUpdatedNano() int64 { + if m != nil { + return m.UpdatedNano + } + return 0 +} + +func (m *ProgressResponse) GetErr() string { + if m != nil { + return m.Err + } + return "" +} + +type DeleteItemSnapshotRequest struct { + Plugin string `protobuf:"bytes,1,opt,name=plugin" json:"plugin,omitempty"` + SnapshotID string `protobuf:"bytes,2,opt,name=snapshotID" json:"snapshotID,omitempty"` + ItemFromBackup []byte `protobuf:"bytes,3,opt,name=itemFromBackup,proto3" json:"itemFromBackup,omitempty"` + Metadata map[string]string `protobuf:"bytes,4,rep,name=metadata" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Params map[string]string `protobuf:"bytes,5,rep,name=params" json:"params,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` +} + +func (m *DeleteItemSnapshotRequest) Reset() { *m = DeleteItemSnapshotRequest{} } +func (m *DeleteItemSnapshotRequest) String() string { return proto.CompactTextString(m) } +func (*DeleteItemSnapshotRequest) ProtoMessage() {} +func (*DeleteItemSnapshotRequest) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{8} } + +func (m *DeleteItemSnapshotRequest) GetPlugin() string { + if m != nil { + return m.Plugin + } + return "" +} + +func (m *DeleteItemSnapshotRequest) GetSnapshotID() string { + if m != nil { + return m.SnapshotID + } + return "" +} + +func (m *DeleteItemSnapshotRequest) GetItemFromBackup() []byte { + if m != nil { + return m.ItemFromBackup + } + return nil +} + +func (m *DeleteItemSnapshotRequest) GetMetadata() map[string]string { + if m != nil { + return m.Metadata + } + return nil +} + +func (m *DeleteItemSnapshotRequest) GetParams() map[string]string { + if m != nil { + return m.Params + } + return nil +} + +type CreateItemFromSnapshotRequest struct { + Plugin string `protobuf:"bytes,1,opt,name=plugin" json:"plugin,omitempty"` + Item []byte `protobuf:"bytes,2,opt,name=item,proto3" json:"item,omitempty"` + SnapshotID string `protobuf:"bytes,3,opt,name=snapshotID" json:"snapshotID,omitempty"` + ItemFromBackup []byte `protobuf:"bytes,4,opt,name=itemFromBackup,proto3" json:"itemFromBackup,omitempty"` + SnapshotMetadata map[string]string `protobuf:"bytes,5,rep,name=snapshotMetadata" json:"snapshotMetadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Params map[string]string `protobuf:"bytes,6,rep,name=params" json:"params,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Restore []byte `protobuf:"bytes,7,opt,name=restore,proto3" json:"restore,omitempty"` +} + +func (m *CreateItemFromSnapshotRequest) Reset() { *m = CreateItemFromSnapshotRequest{} } +func (m *CreateItemFromSnapshotRequest) String() string { return proto.CompactTextString(m) } +func (*CreateItemFromSnapshotRequest) ProtoMessage() {} +func (*CreateItemFromSnapshotRequest) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{9} } + +func (m *CreateItemFromSnapshotRequest) GetPlugin() string { + if m != nil { + return m.Plugin + } + return "" +} + +func (m *CreateItemFromSnapshotRequest) GetItem() []byte { + if m != nil { + return m.Item + } + return nil +} + +func (m *CreateItemFromSnapshotRequest) GetSnapshotID() string { + if m != nil { + return m.SnapshotID + } + return "" +} + +func (m *CreateItemFromSnapshotRequest) GetItemFromBackup() []byte { + if m != nil { + return m.ItemFromBackup + } + return nil +} + +func (m *CreateItemFromSnapshotRequest) GetSnapshotMetadata() map[string]string { + if m != nil { + return m.SnapshotMetadata + } + return nil +} + +func (m *CreateItemFromSnapshotRequest) GetParams() map[string]string { + if m != nil { + return m.Params + } + return nil +} + +func (m *CreateItemFromSnapshotRequest) GetRestore() []byte { + if m != nil { + return m.Restore + } + return nil +} + +type CreateItemFromSnapshotResponse struct { + Item []byte `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"` + AdditionalItems []*ResourceIdentifier `protobuf:"bytes,2,rep,name=additionalItems" json:"additionalItems,omitempty"` + SkipRestore bool `protobuf:"varint,3,opt,name=skipRestore" json:"skipRestore,omitempty"` +} + +func (m *CreateItemFromSnapshotResponse) Reset() { *m = CreateItemFromSnapshotResponse{} } +func (m *CreateItemFromSnapshotResponse) String() string { return proto.CompactTextString(m) } +func (*CreateItemFromSnapshotResponse) ProtoMessage() {} +func (*CreateItemFromSnapshotResponse) Descriptor() ([]byte, []int) { + return fileDescriptor2, []int{10} +} + +func (m *CreateItemFromSnapshotResponse) GetItem() []byte { + if m != nil { + return m.Item + } + return nil +} + +func (m *CreateItemFromSnapshotResponse) GetAdditionalItems() []*ResourceIdentifier { + if m != nil { + return m.AdditionalItems + } + return nil +} + +func (m *CreateItemFromSnapshotResponse) GetSkipRestore() bool { + if m != nil { + return m.SkipRestore + } + return false +} + +type ItemSnapshotterInitRequest struct { + Plugin string `protobuf:"bytes,1,opt,name=plugin" json:"plugin,omitempty"` + Config map[string]string `protobuf:"bytes,2,rep,name=config" json:"config,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` +} + +func (m *ItemSnapshotterInitRequest) Reset() { *m = ItemSnapshotterInitRequest{} } +func (m *ItemSnapshotterInitRequest) String() string { return proto.CompactTextString(m) } +func (*ItemSnapshotterInitRequest) ProtoMessage() {} +func (*ItemSnapshotterInitRequest) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{11} } + +func (m *ItemSnapshotterInitRequest) GetPlugin() string { + if m != nil { + return m.Plugin + } + return "" +} + +func (m *ItemSnapshotterInitRequest) GetConfig() map[string]string { + if m != nil { + return m.Config + } + return nil +} + +func init() { + proto.RegisterType((*ItemSnapshotterAppliesToRequest)(nil), "generated.ItemSnapshotterAppliesToRequest") + proto.RegisterType((*ItemSnapshotterAppliesToResponse)(nil), "generated.ItemSnapshotterAppliesToResponse") + proto.RegisterType((*AlsoHandlesRequest)(nil), "generated.AlsoHandlesRequest") + proto.RegisterType((*AlsoHandlesResponse)(nil), "generated.AlsoHandlesResponse") + proto.RegisterType((*SnapshotItemRequest)(nil), "generated.SnapshotItemRequest") + proto.RegisterType((*SnapshotItemResponse)(nil), "generated.SnapshotItemResponse") + proto.RegisterType((*ProgressRequest)(nil), "generated.ProgressRequest") + proto.RegisterType((*ProgressResponse)(nil), "generated.ProgressResponse") + proto.RegisterType((*DeleteItemSnapshotRequest)(nil), "generated.DeleteItemSnapshotRequest") + proto.RegisterType((*CreateItemFromSnapshotRequest)(nil), "generated.CreateItemFromSnapshotRequest") + proto.RegisterType((*CreateItemFromSnapshotResponse)(nil), "generated.CreateItemFromSnapshotResponse") + proto.RegisterType((*ItemSnapshotterInitRequest)(nil), "generated.ItemSnapshotterInitRequest") +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// Client API for ItemSnapshotter service + +type ItemSnapshotterClient interface { + Init(ctx context.Context, in *ItemSnapshotterInitRequest, opts ...grpc.CallOption) (*Empty, error) + AppliesTo(ctx context.Context, in *ItemSnapshotterAppliesToRequest, opts ...grpc.CallOption) (*ItemSnapshotterAppliesToResponse, error) + AlsoHandles(ctx context.Context, in *AlsoHandlesRequest, opts ...grpc.CallOption) (*AlsoHandlesResponse, error) + SnapshotItem(ctx context.Context, in *SnapshotItemRequest, opts ...grpc.CallOption) (*SnapshotItemResponse, error) + Progress(ctx context.Context, in *ProgressRequest, opts ...grpc.CallOption) (*ProgressResponse, error) + DeleteSnapshot(ctx context.Context, in *DeleteItemSnapshotRequest, opts ...grpc.CallOption) (*Empty, error) + CreateItemFromSnapshot(ctx context.Context, in *CreateItemFromSnapshotRequest, opts ...grpc.CallOption) (*CreateItemFromSnapshotResponse, error) +} + +type itemSnapshotterClient struct { + cc *grpc.ClientConn +} + +func NewItemSnapshotterClient(cc *grpc.ClientConn) ItemSnapshotterClient { + return &itemSnapshotterClient{cc} +} + +func (c *itemSnapshotterClient) Init(ctx context.Context, in *ItemSnapshotterInitRequest, opts ...grpc.CallOption) (*Empty, error) { + out := new(Empty) + err := grpc.Invoke(ctx, "/generated.ItemSnapshotter/Init", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *itemSnapshotterClient) AppliesTo(ctx context.Context, in *ItemSnapshotterAppliesToRequest, opts ...grpc.CallOption) (*ItemSnapshotterAppliesToResponse, error) { + out := new(ItemSnapshotterAppliesToResponse) + err := grpc.Invoke(ctx, "/generated.ItemSnapshotter/AppliesTo", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *itemSnapshotterClient) AlsoHandles(ctx context.Context, in *AlsoHandlesRequest, opts ...grpc.CallOption) (*AlsoHandlesResponse, error) { + out := new(AlsoHandlesResponse) + err := grpc.Invoke(ctx, "/generated.ItemSnapshotter/AlsoHandles", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *itemSnapshotterClient) SnapshotItem(ctx context.Context, in *SnapshotItemRequest, opts ...grpc.CallOption) (*SnapshotItemResponse, error) { + out := new(SnapshotItemResponse) + err := grpc.Invoke(ctx, "/generated.ItemSnapshotter/SnapshotItem", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *itemSnapshotterClient) Progress(ctx context.Context, in *ProgressRequest, opts ...grpc.CallOption) (*ProgressResponse, error) { + out := new(ProgressResponse) + err := grpc.Invoke(ctx, "/generated.ItemSnapshotter/Progress", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *itemSnapshotterClient) DeleteSnapshot(ctx context.Context, in *DeleteItemSnapshotRequest, opts ...grpc.CallOption) (*Empty, error) { + out := new(Empty) + err := grpc.Invoke(ctx, "/generated.ItemSnapshotter/DeleteSnapshot", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *itemSnapshotterClient) CreateItemFromSnapshot(ctx context.Context, in *CreateItemFromSnapshotRequest, opts ...grpc.CallOption) (*CreateItemFromSnapshotResponse, error) { + out := new(CreateItemFromSnapshotResponse) + err := grpc.Invoke(ctx, "/generated.ItemSnapshotter/CreateItemFromSnapshot", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for ItemSnapshotter service + +type ItemSnapshotterServer interface { + Init(context.Context, *ItemSnapshotterInitRequest) (*Empty, error) + AppliesTo(context.Context, *ItemSnapshotterAppliesToRequest) (*ItemSnapshotterAppliesToResponse, error) + AlsoHandles(context.Context, *AlsoHandlesRequest) (*AlsoHandlesResponse, error) + SnapshotItem(context.Context, *SnapshotItemRequest) (*SnapshotItemResponse, error) + Progress(context.Context, *ProgressRequest) (*ProgressResponse, error) + DeleteSnapshot(context.Context, *DeleteItemSnapshotRequest) (*Empty, error) + CreateItemFromSnapshot(context.Context, *CreateItemFromSnapshotRequest) (*CreateItemFromSnapshotResponse, error) +} + +func RegisterItemSnapshotterServer(s *grpc.Server, srv ItemSnapshotterServer) { + s.RegisterService(&_ItemSnapshotter_serviceDesc, srv) +} + +func _ItemSnapshotter_Init_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ItemSnapshotterInitRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ItemSnapshotterServer).Init(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/generated.ItemSnapshotter/Init", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ItemSnapshotterServer).Init(ctx, req.(*ItemSnapshotterInitRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ItemSnapshotter_AppliesTo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ItemSnapshotterAppliesToRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ItemSnapshotterServer).AppliesTo(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/generated.ItemSnapshotter/AppliesTo", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ItemSnapshotterServer).AppliesTo(ctx, req.(*ItemSnapshotterAppliesToRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ItemSnapshotter_AlsoHandles_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AlsoHandlesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ItemSnapshotterServer).AlsoHandles(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/generated.ItemSnapshotter/AlsoHandles", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ItemSnapshotterServer).AlsoHandles(ctx, req.(*AlsoHandlesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ItemSnapshotter_SnapshotItem_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SnapshotItemRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ItemSnapshotterServer).SnapshotItem(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/generated.ItemSnapshotter/SnapshotItem", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ItemSnapshotterServer).SnapshotItem(ctx, req.(*SnapshotItemRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ItemSnapshotter_Progress_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ProgressRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ItemSnapshotterServer).Progress(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/generated.ItemSnapshotter/Progress", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ItemSnapshotterServer).Progress(ctx, req.(*ProgressRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ItemSnapshotter_DeleteSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteItemSnapshotRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ItemSnapshotterServer).DeleteSnapshot(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/generated.ItemSnapshotter/DeleteSnapshot", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ItemSnapshotterServer).DeleteSnapshot(ctx, req.(*DeleteItemSnapshotRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ItemSnapshotter_CreateItemFromSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateItemFromSnapshotRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ItemSnapshotterServer).CreateItemFromSnapshot(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/generated.ItemSnapshotter/CreateItemFromSnapshot", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ItemSnapshotterServer).CreateItemFromSnapshot(ctx, req.(*CreateItemFromSnapshotRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _ItemSnapshotter_serviceDesc = grpc.ServiceDesc{ + ServiceName: "generated.ItemSnapshotter", + HandlerType: (*ItemSnapshotterServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Init", + Handler: _ItemSnapshotter_Init_Handler, + }, + { + MethodName: "AppliesTo", + Handler: _ItemSnapshotter_AppliesTo_Handler, + }, + { + MethodName: "AlsoHandles", + Handler: _ItemSnapshotter_AlsoHandles_Handler, + }, + { + MethodName: "SnapshotItem", + Handler: _ItemSnapshotter_SnapshotItem_Handler, + }, + { + MethodName: "Progress", + Handler: _ItemSnapshotter_Progress_Handler, + }, + { + MethodName: "DeleteSnapshot", + Handler: _ItemSnapshotter_DeleteSnapshot_Handler, + }, + { + MethodName: "CreateItemFromSnapshot", + Handler: _ItemSnapshotter_CreateItemFromSnapshot_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "ItemSnapshotter.proto", +} + +func init() { proto.RegisterFile("ItemSnapshotter.proto", fileDescriptor2) } + +var fileDescriptor2 = []byte{ + // 887 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x5f, 0x8f, 0xdb, 0x44, + 0x10, 0x97, 0xe3, 0x5c, 0xee, 0x6e, 0x12, 0x7a, 0xd1, 0xf6, 0x5a, 0x19, 0x57, 0xbd, 0x46, 0x16, + 0xa0, 0x50, 0xa4, 0x08, 0x0e, 0x2a, 0x51, 0x90, 0x40, 0xd7, 0xb4, 0xf4, 0x4e, 0x2a, 0xa5, 0xf2, + 0xf5, 0xa1, 0xaf, 0xdb, 0x78, 0x9b, 0x98, 0xd8, 0x5e, 0xb3, 0xbb, 0x41, 0xba, 0x77, 0x5e, 0x79, + 0xe7, 0x8d, 0xef, 0xc1, 0xf7, 0x40, 0xe2, 0x1b, 0xf0, 0x2d, 0x10, 0xda, 0x3f, 0x0e, 0x1b, 0xc7, + 0x3e, 0xfb, 0xee, 0x78, 0xf3, 0xce, 0xce, 0xcc, 0xfe, 0x66, 0x7e, 0xb3, 0xb3, 0x63, 0xb8, 0x73, + 0x26, 0x48, 0x7a, 0x9e, 0xe1, 0x9c, 0x2f, 0xa8, 0x10, 0x84, 0x4d, 0x72, 0x46, 0x05, 0x45, 0xfb, + 0x73, 0x92, 0x11, 0x86, 0x05, 0x89, 0xfc, 0xc1, 0xf9, 0x02, 0x33, 0x12, 0xe9, 0x8d, 0xe0, 0x31, + 0x3c, 0x28, 0x59, 0x9c, 0xe4, 0x79, 0x12, 0x13, 0xfe, 0x9a, 0x86, 0xe4, 0xa7, 0x15, 0xe1, 0x02, + 0xdd, 0x85, 0x5e, 0x9e, 0xac, 0xe6, 0x71, 0xe6, 0x39, 0x23, 0x67, 0xbc, 0x1f, 0x9a, 0x55, 0xb0, + 0x84, 0x51, 0xbd, 0x29, 0xcf, 0x69, 0xc6, 0x09, 0x7a, 0x0e, 0xc3, 0x90, 0x70, 0xba, 0x62, 0x33, + 0x72, 0x4e, 0x12, 0x32, 0x13, 0x94, 0x29, 0x2f, 0xfd, 0xe3, 0x7b, 0x93, 0x35, 0xa4, 0x49, 0x59, + 0x25, 0xdc, 0x32, 0x0a, 0xde, 0x00, 0x3a, 0x49, 0x38, 0x3d, 0xc5, 0x59, 0x94, 0x10, 0xde, 0x00, + 0x0d, 0x21, 0xe8, 0xc6, 0x82, 0xa4, 0x5e, 0x67, 0xe4, 0x8c, 0x07, 0xa1, 0xfa, 0x96, 0xba, 0x6f, + 0xf1, 0x6c, 0xb9, 0xca, 0x3d, 0x57, 0x49, 0xcd, 0x2a, 0x78, 0x03, 0xb7, 0x37, 0x3c, 0x1b, 0xe4, + 0x27, 0x30, 0x58, 0x28, 0x51, 0x24, 0x83, 0xe4, 0x9e, 0x33, 0x72, 0xc7, 0xfd, 0xe3, 0xfb, 0x15, + 0xa8, 0xcf, 0x22, 0x92, 0x89, 0xf8, 0x5d, 0x4c, 0x58, 0xb8, 0x61, 0x12, 0xfc, 0xe5, 0xc0, 0xed, + 0x22, 0x3b, 0x52, 0x72, 0x1d, 0xd4, 0x4f, 0xa0, 0x97, 0x63, 0x86, 0x53, 0xee, 0xb9, 0x0a, 0xc0, + 0x43, 0x0b, 0x40, 0x85, 0xef, 0xc9, 0x2b, 0xa5, 0xfc, 0x2c, 0x13, 0xec, 0x22, 0x34, 0x96, 0x56, + 0xe4, 0x5d, 0x3b, 0x72, 0xff, 0x31, 0xf4, 0x2d, 0x75, 0x34, 0x04, 0x77, 0x49, 0x2e, 0x0c, 0x26, + 0xf9, 0x89, 0x0e, 0x61, 0xe7, 0x67, 0x9c, 0xac, 0x88, 0x42, 0xb4, 0x1f, 0xea, 0xc5, 0x57, 0x9d, + 0x2f, 0x9d, 0xe0, 0x9f, 0x0e, 0x1c, 0x6e, 0x1e, 0x6f, 0xd2, 0x56, 0xc4, 0xe0, 0x58, 0x31, 0x1c, + 0x01, 0xf0, 0x42, 0xf7, 0xa9, 0xf1, 0x65, 0x49, 0x10, 0x86, 0x61, 0xb1, 0xfa, 0x9e, 0x08, 0x1c, + 0x61, 0x81, 0x4d, 0xb4, 0x8f, 0x6a, 0xa3, 0xd5, 0xc7, 0xad, 0x85, 0x85, 0x9d, 0x0e, 0x7c, 0xcb, + 0x1d, 0x7a, 0x0e, 0x07, 0x38, 0x8a, 0x62, 0x11, 0xd3, 0x0c, 0x27, 0x9a, 0xd0, 0x6e, 0x1b, 0x42, + 0xcb, 0x56, 0x5b, 0x65, 0xb1, 0x73, 0xe5, 0xb2, 0xf0, 0xa7, 0x70, 0xa7, 0x12, 0xf6, 0x95, 0x08, + 0xf8, 0xcd, 0x81, 0x83, 0x57, 0x8c, 0xce, 0x19, 0xe1, 0x8d, 0xb7, 0xe1, 0x11, 0xf4, 0x24, 0x0f, + 0x26, 0xf7, 0x8d, 0x68, 0x8d, 0x72, 0x89, 0x36, 0x77, 0x8b, 0xb6, 0x9a, 0xb2, 0x0a, 0x7e, 0xe9, + 0xc0, 0xf0, 0x3f, 0x68, 0xa6, 0x2e, 0x0e, 0x61, 0x27, 0x5f, 0x60, 0x4e, 0x0c, 0x34, 0xbd, 0x40, + 0x1f, 0xc1, 0x2d, 0x79, 0x18, 0x9f, 0xd2, 0x34, 0x4f, 0x88, 0x20, 0x91, 0x42, 0xe8, 0x86, 0x25, + 0x29, 0x1a, 0xc3, 0x81, 0x92, 0xbc, 0xa6, 0x85, 0x4c, 0xe1, 0x71, 0xc3, 0xb2, 0x18, 0x79, 0xb0, + 0xcb, 0x05, 0x66, 0xd2, 0x55, 0x57, 0x69, 0x14, 0x4b, 0x34, 0x82, 0xbe, 0xf9, 0x7c, 0x89, 0x33, + 0xea, 0xed, 0xa8, 0x5d, 0x5b, 0x24, 0x6d, 0x57, 0x79, 0x24, 0xd3, 0xe2, 0xf5, 0xb4, 0xad, 0x59, + 0x4a, 0x5b, 0xf3, 0xa9, 0x6c, 0x77, 0xb5, 0xad, 0x25, 0x92, 0xdc, 0x11, 0xc6, 0xbc, 0x3d, 0xcd, + 0x1d, 0x61, 0x2c, 0xf8, 0xd5, 0x85, 0xf7, 0x9f, 0x12, 0x09, 0xca, 0xee, 0x92, 0x4d, 0x5c, 0x35, + 0xdd, 0x15, 0x93, 0xb1, 0xef, 0x18, 0x4d, 0x9f, 0xd8, 0xdd, 0xac, 0x24, 0x45, 0x2f, 0x61, 0x2f, + 0x2d, 0xee, 0x92, 0xae, 0xf4, 0x63, 0x8b, 0xf5, 0x5a, 0x5c, 0x93, 0xcd, 0x8b, 0xb4, 0xf6, 0x81, + 0x4e, 0xd7, 0x7d, 0x48, 0x57, 0xfc, 0xa7, 0xad, 0xbc, 0x55, 0x74, 0x23, 0xff, 0x6b, 0x78, 0xef, + 0xda, 0x65, 0x7f, 0x93, 0x96, 0xf5, 0xb7, 0x0b, 0xf7, 0xa7, 0x8c, 0x60, 0x8d, 0x54, 0xa6, 0xaa, + 0x2d, 0x27, 0x55, 0x7d, 0xb9, 0xe9, 0x72, 0x6c, 0xf3, 0xd4, 0xad, 0xe4, 0xe9, 0xc7, 0x8a, 0xde, + 0xa7, 0x33, 0xfc, 0x8d, 0x95, 0xe1, 0x4b, 0x71, 0xb7, 0x6e, 0x82, 0x2f, 0xd6, 0x1c, 0xf6, 0xd4, + 0x09, 0x5f, 0xb4, 0x3e, 0xa1, 0xea, 0x55, 0xf1, 0x60, 0x97, 0x11, 0x2e, 0x28, 0x23, 0xea, 0x3e, + 0x0c, 0xc2, 0x62, 0xf9, 0xbf, 0x34, 0xb8, 0x9b, 0x30, 0xfd, 0xbb, 0x03, 0x47, 0x75, 0xf1, 0x5c, + 0xf2, 0x4c, 0x55, 0xbc, 0x11, 0x9d, 0x6b, 0xbd, 0x11, 0xb2, 0xd3, 0x2c, 0xe3, 0x3c, 0x34, 0xd9, + 0x91, 0xc5, 0xb1, 0x17, 0xda, 0xa2, 0xe0, 0x0f, 0x07, 0xfc, 0xd2, 0xec, 0x74, 0x96, 0xc5, 0x8d, + 0x85, 0x78, 0x06, 0xbd, 0x19, 0xcd, 0xde, 0xc5, 0x73, 0x03, 0xec, 0x33, 0x0b, 0x58, 0xbd, 0xbb, + 0xc9, 0x54, 0xd9, 0x18, 0xf6, 0xb4, 0x03, 0x99, 0x5e, 0x4b, 0x7c, 0x95, 0xf4, 0x1e, 0xff, 0xd9, + 0x85, 0x83, 0xd2, 0x69, 0xe8, 0x5b, 0xe8, 0xca, 0x13, 0xd1, 0x87, 0xad, 0x10, 0xf9, 0x43, 0x4b, + 0xed, 0x59, 0x9a, 0x8b, 0x0b, 0x14, 0xc1, 0xfe, 0x7a, 0x7a, 0x44, 0x0f, 0xeb, 0xbd, 0x94, 0xa7, + 0x53, 0xff, 0x93, 0x56, 0xba, 0x86, 0xf6, 0x17, 0xd0, 0xb7, 0x66, 0x3d, 0x64, 0x13, 0xbb, 0x3d, + 0x5d, 0xfa, 0x47, 0x75, 0xdb, 0xc6, 0xdb, 0x0f, 0x30, 0xb0, 0x87, 0x12, 0x74, 0x74, 0xf9, 0x6c, + 0xe6, 0x3f, 0x68, 0x98, 0x66, 0xd0, 0x14, 0xf6, 0x8a, 0x87, 0x13, 0xf9, 0x96, 0x72, 0xe9, 0xa1, + 0xf7, 0xef, 0x55, 0xee, 0x19, 0x27, 0xa7, 0x70, 0x4b, 0x37, 0xe4, 0xe2, 0x08, 0xf4, 0x41, 0x9b, + 0x5e, 0x5d, 0xc1, 0x49, 0x0a, 0x77, 0xab, 0xaf, 0x11, 0x1a, 0xb7, 0xed, 0x1c, 0xfe, 0xc7, 0x2d, + 0x34, 0x35, 0xf0, 0xb7, 0x3d, 0xf5, 0x47, 0xf2, 0xf9, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x51, + 0xd9, 0x74, 0xbb, 0xc3, 0x0c, 0x00, 0x00, +} diff --git a/pkg/plugin/generated/ObjectStore.pb.go b/pkg/plugin/generated/ObjectStore.pb.go index 121d70d45..b22cf1379 100644 --- a/pkg/plugin/generated/ObjectStore.pb.go +++ b/pkg/plugin/generated/ObjectStore.pb.go @@ -27,7 +27,7 @@ type PutObjectRequest struct { func (m *PutObjectRequest) Reset() { *m = PutObjectRequest{} } func (m *PutObjectRequest) String() string { return proto.CompactTextString(m) } func (*PutObjectRequest) ProtoMessage() {} -func (*PutObjectRequest) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{0} } +func (*PutObjectRequest) Descriptor() ([]byte, []int) { return fileDescriptor3, []int{0} } func (m *PutObjectRequest) GetPlugin() string { if m != nil { @@ -66,7 +66,7 @@ type ObjectExistsRequest struct { func (m *ObjectExistsRequest) Reset() { *m = ObjectExistsRequest{} } func (m *ObjectExistsRequest) String() string { return proto.CompactTextString(m) } func (*ObjectExistsRequest) ProtoMessage() {} -func (*ObjectExistsRequest) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{1} } +func (*ObjectExistsRequest) Descriptor() ([]byte, []int) { return fileDescriptor3, []int{1} } func (m *ObjectExistsRequest) GetPlugin() string { if m != nil { @@ -96,7 +96,7 @@ type ObjectExistsResponse struct { func (m *ObjectExistsResponse) Reset() { *m = ObjectExistsResponse{} } func (m *ObjectExistsResponse) String() string { return proto.CompactTextString(m) } func (*ObjectExistsResponse) ProtoMessage() {} -func (*ObjectExistsResponse) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{2} } +func (*ObjectExistsResponse) Descriptor() ([]byte, []int) { return fileDescriptor3, []int{2} } func (m *ObjectExistsResponse) GetExists() bool { if m != nil { @@ -114,7 +114,7 @@ type GetObjectRequest struct { func (m *GetObjectRequest) Reset() { *m = GetObjectRequest{} } func (m *GetObjectRequest) String() string { return proto.CompactTextString(m) } func (*GetObjectRequest) ProtoMessage() {} -func (*GetObjectRequest) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{3} } +func (*GetObjectRequest) Descriptor() ([]byte, []int) { return fileDescriptor3, []int{3} } func (m *GetObjectRequest) GetPlugin() string { if m != nil { @@ -144,7 +144,7 @@ type Bytes struct { func (m *Bytes) Reset() { *m = Bytes{} } func (m *Bytes) String() string { return proto.CompactTextString(m) } func (*Bytes) ProtoMessage() {} -func (*Bytes) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{4} } +func (*Bytes) Descriptor() ([]byte, []int) { return fileDescriptor3, []int{4} } func (m *Bytes) GetData() []byte { if m != nil { @@ -163,7 +163,7 @@ type ListCommonPrefixesRequest struct { func (m *ListCommonPrefixesRequest) Reset() { *m = ListCommonPrefixesRequest{} } func (m *ListCommonPrefixesRequest) String() string { return proto.CompactTextString(m) } func (*ListCommonPrefixesRequest) ProtoMessage() {} -func (*ListCommonPrefixesRequest) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{5} } +func (*ListCommonPrefixesRequest) Descriptor() ([]byte, []int) { return fileDescriptor3, []int{5} } func (m *ListCommonPrefixesRequest) GetPlugin() string { if m != nil { @@ -200,7 +200,7 @@ type ListCommonPrefixesResponse struct { func (m *ListCommonPrefixesResponse) Reset() { *m = ListCommonPrefixesResponse{} } func (m *ListCommonPrefixesResponse) String() string { return proto.CompactTextString(m) } func (*ListCommonPrefixesResponse) ProtoMessage() {} -func (*ListCommonPrefixesResponse) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{6} } +func (*ListCommonPrefixesResponse) Descriptor() ([]byte, []int) { return fileDescriptor3, []int{6} } func (m *ListCommonPrefixesResponse) GetPrefixes() []string { if m != nil { @@ -218,7 +218,7 @@ type ListObjectsRequest struct { func (m *ListObjectsRequest) Reset() { *m = ListObjectsRequest{} } func (m *ListObjectsRequest) String() string { return proto.CompactTextString(m) } func (*ListObjectsRequest) ProtoMessage() {} -func (*ListObjectsRequest) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{7} } +func (*ListObjectsRequest) Descriptor() ([]byte, []int) { return fileDescriptor3, []int{7} } func (m *ListObjectsRequest) GetPlugin() string { if m != nil { @@ -248,7 +248,7 @@ type ListObjectsResponse struct { func (m *ListObjectsResponse) Reset() { *m = ListObjectsResponse{} } func (m *ListObjectsResponse) String() string { return proto.CompactTextString(m) } func (*ListObjectsResponse) ProtoMessage() {} -func (*ListObjectsResponse) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{8} } +func (*ListObjectsResponse) Descriptor() ([]byte, []int) { return fileDescriptor3, []int{8} } func (m *ListObjectsResponse) GetKeys() []string { if m != nil { @@ -266,7 +266,7 @@ type DeleteObjectRequest struct { func (m *DeleteObjectRequest) Reset() { *m = DeleteObjectRequest{} } func (m *DeleteObjectRequest) String() string { return proto.CompactTextString(m) } func (*DeleteObjectRequest) ProtoMessage() {} -func (*DeleteObjectRequest) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{9} } +func (*DeleteObjectRequest) Descriptor() ([]byte, []int) { return fileDescriptor3, []int{9} } func (m *DeleteObjectRequest) GetPlugin() string { if m != nil { @@ -299,7 +299,7 @@ type CreateSignedURLRequest struct { func (m *CreateSignedURLRequest) Reset() { *m = CreateSignedURLRequest{} } func (m *CreateSignedURLRequest) String() string { return proto.CompactTextString(m) } func (*CreateSignedURLRequest) ProtoMessage() {} -func (*CreateSignedURLRequest) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{10} } +func (*CreateSignedURLRequest) Descriptor() ([]byte, []int) { return fileDescriptor3, []int{10} } func (m *CreateSignedURLRequest) GetPlugin() string { if m != nil { @@ -336,7 +336,7 @@ type CreateSignedURLResponse struct { func (m *CreateSignedURLResponse) Reset() { *m = CreateSignedURLResponse{} } func (m *CreateSignedURLResponse) String() string { return proto.CompactTextString(m) } func (*CreateSignedURLResponse) ProtoMessage() {} -func (*CreateSignedURLResponse) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{11} } +func (*CreateSignedURLResponse) Descriptor() ([]byte, []int) { return fileDescriptor3, []int{11} } func (m *CreateSignedURLResponse) GetUrl() string { if m != nil { @@ -353,7 +353,7 @@ type ObjectStoreInitRequest struct { func (m *ObjectStoreInitRequest) Reset() { *m = ObjectStoreInitRequest{} } func (m *ObjectStoreInitRequest) String() string { return proto.CompactTextString(m) } func (*ObjectStoreInitRequest) ProtoMessage() {} -func (*ObjectStoreInitRequest) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{12} } +func (*ObjectStoreInitRequest) Descriptor() ([]byte, []int) { return fileDescriptor3, []int{12} } func (m *ObjectStoreInitRequest) GetPlugin() string { if m != nil { @@ -750,9 +750,9 @@ var _ObjectStore_serviceDesc = grpc.ServiceDesc{ Metadata: "ObjectStore.proto", } -func init() { proto.RegisterFile("ObjectStore.proto", fileDescriptor2) } +func init() { proto.RegisterFile("ObjectStore.proto", fileDescriptor3) } -var fileDescriptor2 = []byte{ +var fileDescriptor3 = []byte{ // 577 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x40, 0x10, 0xd6, 0xc6, 0x69, 0x54, 0x4f, 0x22, 0x61, 0xb6, 0x55, 0x30, 0x2e, 0x94, 0xb0, 0x02, 0x29, diff --git a/pkg/plugin/generated/PluginLister.pb.go b/pkg/plugin/generated/PluginLister.pb.go index 0f4fe7c87..a2d7957d9 100644 --- a/pkg/plugin/generated/PluginLister.pb.go +++ b/pkg/plugin/generated/PluginLister.pb.go @@ -26,7 +26,7 @@ type PluginIdentifier struct { func (m *PluginIdentifier) Reset() { *m = PluginIdentifier{} } func (m *PluginIdentifier) String() string { return proto.CompactTextString(m) } func (*PluginIdentifier) ProtoMessage() {} -func (*PluginIdentifier) Descriptor() ([]byte, []int) { return fileDescriptor3, []int{0} } +func (*PluginIdentifier) Descriptor() ([]byte, []int) { return fileDescriptor4, []int{0} } func (m *PluginIdentifier) GetCommand() string { if m != nil { @@ -56,7 +56,7 @@ type ListPluginsResponse struct { func (m *ListPluginsResponse) Reset() { *m = ListPluginsResponse{} } func (m *ListPluginsResponse) String() string { return proto.CompactTextString(m) } func (*ListPluginsResponse) ProtoMessage() {} -func (*ListPluginsResponse) Descriptor() ([]byte, []int) { return fileDescriptor3, []int{1} } +func (*ListPluginsResponse) Descriptor() ([]byte, []int) { return fileDescriptor4, []int{1} } func (m *ListPluginsResponse) GetPlugins() []*PluginIdentifier { if m != nil { @@ -142,9 +142,9 @@ var _PluginLister_serviceDesc = grpc.ServiceDesc{ Metadata: "PluginLister.proto", } -func init() { proto.RegisterFile("PluginLister.proto", fileDescriptor3) } +func init() { proto.RegisterFile("PluginLister.proto", fileDescriptor4) } -var fileDescriptor3 = []byte{ +var fileDescriptor4 = []byte{ // 201 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x0a, 0xc8, 0x29, 0x4d, 0xcf, 0xcc, 0xf3, 0xc9, 0x2c, 0x2e, 0x49, 0x2d, 0xd2, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, diff --git a/pkg/plugin/generated/RestoreItemAction.pb.go b/pkg/plugin/generated/RestoreItemAction.pb.go index 63a92b4c2..da06ffdac 100644 --- a/pkg/plugin/generated/RestoreItemAction.pb.go +++ b/pkg/plugin/generated/RestoreItemAction.pb.go @@ -28,7 +28,7 @@ func (m *RestoreItemActionExecuteRequest) Reset() { *m = RestoreItemActi func (m *RestoreItemActionExecuteRequest) String() string { return proto.CompactTextString(m) } func (*RestoreItemActionExecuteRequest) ProtoMessage() {} func (*RestoreItemActionExecuteRequest) Descriptor() ([]byte, []int) { - return fileDescriptor4, []int{0} + return fileDescriptor5, []int{0} } func (m *RestoreItemActionExecuteRequest) GetPlugin() string { @@ -69,7 +69,7 @@ func (m *RestoreItemActionExecuteResponse) Reset() { *m = RestoreItemAct func (m *RestoreItemActionExecuteResponse) String() string { return proto.CompactTextString(m) } func (*RestoreItemActionExecuteResponse) ProtoMessage() {} func (*RestoreItemActionExecuteResponse) Descriptor() ([]byte, []int) { - return fileDescriptor4, []int{1} + return fileDescriptor5, []int{1} } func (m *RestoreItemActionExecuteResponse) GetItem() []byte { @@ -101,7 +101,7 @@ func (m *RestoreItemActionAppliesToRequest) Reset() { *m = RestoreItemAc func (m *RestoreItemActionAppliesToRequest) String() string { return proto.CompactTextString(m) } func (*RestoreItemActionAppliesToRequest) ProtoMessage() {} func (*RestoreItemActionAppliesToRequest) Descriptor() ([]byte, []int) { - return fileDescriptor4, []int{2} + return fileDescriptor5, []int{2} } func (m *RestoreItemActionAppliesToRequest) GetPlugin() string { @@ -119,7 +119,7 @@ func (m *RestoreItemActionAppliesToResponse) Reset() { *m = RestoreItemA func (m *RestoreItemActionAppliesToResponse) String() string { return proto.CompactTextString(m) } func (*RestoreItemActionAppliesToResponse) ProtoMessage() {} func (*RestoreItemActionAppliesToResponse) Descriptor() ([]byte, []int) { - return fileDescriptor4, []int{3} + return fileDescriptor5, []int{3} } func (m *RestoreItemActionAppliesToResponse) GetResourceSelector() *ResourceSelector { @@ -241,9 +241,9 @@ var _RestoreItemAction_serviceDesc = grpc.ServiceDesc{ Metadata: "RestoreItemAction.proto", } -func init() { proto.RegisterFile("RestoreItemAction.proto", fileDescriptor4) } +func init() { proto.RegisterFile("RestoreItemAction.proto", fileDescriptor5) } -var fileDescriptor4 = []byte{ +var fileDescriptor5 = []byte{ // 332 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x52, 0xdd, 0x4e, 0xc2, 0x30, 0x14, 0x4e, 0x81, 0x80, 0x1c, 0x88, 0x3f, 0xbd, 0xd0, 0x06, 0x63, 0x9c, 0xbb, 0x30, 0xc4, 0x1f, diff --git a/pkg/plugin/generated/Shared.pb.go b/pkg/plugin/generated/Shared.pb.go index 29714972e..8f2716da5 100644 --- a/pkg/plugin/generated/Shared.pb.go +++ b/pkg/plugin/generated/Shared.pb.go @@ -18,7 +18,7 @@ type Empty struct { func (m *Empty) Reset() { *m = Empty{} } func (m *Empty) String() string { return proto.CompactTextString(m) } func (*Empty) ProtoMessage() {} -func (*Empty) Descriptor() ([]byte, []int) { return fileDescriptor5, []int{0} } +func (*Empty) Descriptor() ([]byte, []int) { return fileDescriptor6, []int{0} } type Stack struct { Frames []*StackFrame `protobuf:"bytes,1,rep,name=frames" json:"frames,omitempty"` @@ -27,7 +27,7 @@ type Stack struct { func (m *Stack) Reset() { *m = Stack{} } func (m *Stack) String() string { return proto.CompactTextString(m) } func (*Stack) ProtoMessage() {} -func (*Stack) Descriptor() ([]byte, []int) { return fileDescriptor5, []int{1} } +func (*Stack) Descriptor() ([]byte, []int) { return fileDescriptor6, []int{1} } func (m *Stack) GetFrames() []*StackFrame { if m != nil { @@ -45,7 +45,7 @@ type StackFrame struct { func (m *StackFrame) Reset() { *m = StackFrame{} } func (m *StackFrame) String() string { return proto.CompactTextString(m) } func (*StackFrame) ProtoMessage() {} -func (*StackFrame) Descriptor() ([]byte, []int) { return fileDescriptor5, []int{2} } +func (*StackFrame) Descriptor() ([]byte, []int) { return fileDescriptor6, []int{2} } func (m *StackFrame) GetFile() string { if m != nil { @@ -78,7 +78,7 @@ type ResourceIdentifier struct { func (m *ResourceIdentifier) Reset() { *m = ResourceIdentifier{} } func (m *ResourceIdentifier) String() string { return proto.CompactTextString(m) } func (*ResourceIdentifier) ProtoMessage() {} -func (*ResourceIdentifier) Descriptor() ([]byte, []int) { return fileDescriptor5, []int{3} } +func (*ResourceIdentifier) Descriptor() ([]byte, []int) { return fileDescriptor6, []int{3} } func (m *ResourceIdentifier) GetGroup() string { if m != nil { @@ -119,7 +119,7 @@ type ResourceSelector struct { func (m *ResourceSelector) Reset() { *m = ResourceSelector{} } func (m *ResourceSelector) String() string { return proto.CompactTextString(m) } func (*ResourceSelector) ProtoMessage() {} -func (*ResourceSelector) Descriptor() ([]byte, []int) { return fileDescriptor5, []int{4} } +func (*ResourceSelector) Descriptor() ([]byte, []int) { return fileDescriptor6, []int{4} } func (m *ResourceSelector) GetIncludedNamespaces() []string { if m != nil { @@ -164,9 +164,9 @@ func init() { proto.RegisterType((*ResourceSelector)(nil), "generated.ResourceSelector") } -func init() { proto.RegisterFile("Shared.proto", fileDescriptor5) } +func init() { proto.RegisterFile("Shared.proto", fileDescriptor6) } -var fileDescriptor5 = []byte{ +var fileDescriptor6 = []byte{ // 294 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x92, 0xc1, 0x4e, 0xb5, 0x30, 0x10, 0x85, 0xc3, 0x05, 0xee, 0xff, 0x33, 0xba, 0xd0, 0x46, 0x93, 0xc6, 0xb8, 0x20, 0xac, 0x58, diff --git a/pkg/plugin/generated/VolumeSnapshotter.pb.go b/pkg/plugin/generated/VolumeSnapshotter.pb.go index b814bae9a..ddb8a2d74 100644 --- a/pkg/plugin/generated/VolumeSnapshotter.pb.go +++ b/pkg/plugin/generated/VolumeSnapshotter.pb.go @@ -28,7 +28,7 @@ type CreateVolumeRequest struct { func (m *CreateVolumeRequest) Reset() { *m = CreateVolumeRequest{} } func (m *CreateVolumeRequest) String() string { return proto.CompactTextString(m) } func (*CreateVolumeRequest) ProtoMessage() {} -func (*CreateVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor6, []int{0} } +func (*CreateVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor7, []int{0} } func (m *CreateVolumeRequest) GetPlugin() string { if m != nil { @@ -72,7 +72,7 @@ type CreateVolumeResponse struct { func (m *CreateVolumeResponse) Reset() { *m = CreateVolumeResponse{} } func (m *CreateVolumeResponse) String() string { return proto.CompactTextString(m) } func (*CreateVolumeResponse) ProtoMessage() {} -func (*CreateVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor6, []int{1} } +func (*CreateVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor7, []int{1} } func (m *CreateVolumeResponse) GetVolumeID() string { if m != nil { @@ -90,7 +90,7 @@ type GetVolumeInfoRequest struct { func (m *GetVolumeInfoRequest) Reset() { *m = GetVolumeInfoRequest{} } func (m *GetVolumeInfoRequest) String() string { return proto.CompactTextString(m) } func (*GetVolumeInfoRequest) ProtoMessage() {} -func (*GetVolumeInfoRequest) Descriptor() ([]byte, []int) { return fileDescriptor6, []int{2} } +func (*GetVolumeInfoRequest) Descriptor() ([]byte, []int) { return fileDescriptor7, []int{2} } func (m *GetVolumeInfoRequest) GetPlugin() string { if m != nil { @@ -121,7 +121,7 @@ type GetVolumeInfoResponse struct { func (m *GetVolumeInfoResponse) Reset() { *m = GetVolumeInfoResponse{} } func (m *GetVolumeInfoResponse) String() string { return proto.CompactTextString(m) } func (*GetVolumeInfoResponse) ProtoMessage() {} -func (*GetVolumeInfoResponse) Descriptor() ([]byte, []int) { return fileDescriptor6, []int{3} } +func (*GetVolumeInfoResponse) Descriptor() ([]byte, []int) { return fileDescriptor7, []int{3} } func (m *GetVolumeInfoResponse) GetVolumeType() string { if m != nil { @@ -147,7 +147,7 @@ type CreateSnapshotRequest struct { func (m *CreateSnapshotRequest) Reset() { *m = CreateSnapshotRequest{} } func (m *CreateSnapshotRequest) String() string { return proto.CompactTextString(m) } func (*CreateSnapshotRequest) ProtoMessage() {} -func (*CreateSnapshotRequest) Descriptor() ([]byte, []int) { return fileDescriptor6, []int{4} } +func (*CreateSnapshotRequest) Descriptor() ([]byte, []int) { return fileDescriptor7, []int{4} } func (m *CreateSnapshotRequest) GetPlugin() string { if m != nil { @@ -184,7 +184,7 @@ type CreateSnapshotResponse struct { func (m *CreateSnapshotResponse) Reset() { *m = CreateSnapshotResponse{} } func (m *CreateSnapshotResponse) String() string { return proto.CompactTextString(m) } func (*CreateSnapshotResponse) ProtoMessage() {} -func (*CreateSnapshotResponse) Descriptor() ([]byte, []int) { return fileDescriptor6, []int{5} } +func (*CreateSnapshotResponse) Descriptor() ([]byte, []int) { return fileDescriptor7, []int{5} } func (m *CreateSnapshotResponse) GetSnapshotID() string { if m != nil { @@ -201,7 +201,7 @@ type DeleteSnapshotRequest struct { func (m *DeleteSnapshotRequest) Reset() { *m = DeleteSnapshotRequest{} } func (m *DeleteSnapshotRequest) String() string { return proto.CompactTextString(m) } func (*DeleteSnapshotRequest) ProtoMessage() {} -func (*DeleteSnapshotRequest) Descriptor() ([]byte, []int) { return fileDescriptor6, []int{6} } +func (*DeleteSnapshotRequest) Descriptor() ([]byte, []int) { return fileDescriptor7, []int{6} } func (m *DeleteSnapshotRequest) GetPlugin() string { if m != nil { @@ -225,7 +225,7 @@ type GetVolumeIDRequest struct { func (m *GetVolumeIDRequest) Reset() { *m = GetVolumeIDRequest{} } func (m *GetVolumeIDRequest) String() string { return proto.CompactTextString(m) } func (*GetVolumeIDRequest) ProtoMessage() {} -func (*GetVolumeIDRequest) Descriptor() ([]byte, []int) { return fileDescriptor6, []int{7} } +func (*GetVolumeIDRequest) Descriptor() ([]byte, []int) { return fileDescriptor7, []int{7} } func (m *GetVolumeIDRequest) GetPlugin() string { if m != nil { @@ -248,7 +248,7 @@ type GetVolumeIDResponse struct { func (m *GetVolumeIDResponse) Reset() { *m = GetVolumeIDResponse{} } func (m *GetVolumeIDResponse) String() string { return proto.CompactTextString(m) } func (*GetVolumeIDResponse) ProtoMessage() {} -func (*GetVolumeIDResponse) Descriptor() ([]byte, []int) { return fileDescriptor6, []int{8} } +func (*GetVolumeIDResponse) Descriptor() ([]byte, []int) { return fileDescriptor7, []int{8} } func (m *GetVolumeIDResponse) GetVolumeID() string { if m != nil { @@ -266,7 +266,7 @@ type SetVolumeIDRequest struct { func (m *SetVolumeIDRequest) Reset() { *m = SetVolumeIDRequest{} } func (m *SetVolumeIDRequest) String() string { return proto.CompactTextString(m) } func (*SetVolumeIDRequest) ProtoMessage() {} -func (*SetVolumeIDRequest) Descriptor() ([]byte, []int) { return fileDescriptor6, []int{9} } +func (*SetVolumeIDRequest) Descriptor() ([]byte, []int) { return fileDescriptor7, []int{9} } func (m *SetVolumeIDRequest) GetPlugin() string { if m != nil { @@ -296,7 +296,7 @@ type SetVolumeIDResponse struct { func (m *SetVolumeIDResponse) Reset() { *m = SetVolumeIDResponse{} } func (m *SetVolumeIDResponse) String() string { return proto.CompactTextString(m) } func (*SetVolumeIDResponse) ProtoMessage() {} -func (*SetVolumeIDResponse) Descriptor() ([]byte, []int) { return fileDescriptor6, []int{10} } +func (*SetVolumeIDResponse) Descriptor() ([]byte, []int) { return fileDescriptor7, []int{10} } func (m *SetVolumeIDResponse) GetPersistentVolume() []byte { if m != nil { @@ -313,7 +313,7 @@ type VolumeSnapshotterInitRequest struct { func (m *VolumeSnapshotterInitRequest) Reset() { *m = VolumeSnapshotterInitRequest{} } func (m *VolumeSnapshotterInitRequest) String() string { return proto.CompactTextString(m) } func (*VolumeSnapshotterInitRequest) ProtoMessage() {} -func (*VolumeSnapshotterInitRequest) Descriptor() ([]byte, []int) { return fileDescriptor6, []int{11} } +func (*VolumeSnapshotterInitRequest) Descriptor() ([]byte, []int) { return fileDescriptor7, []int{11} } func (m *VolumeSnapshotterInitRequest) GetPlugin() string { if m != nil { @@ -614,9 +614,9 @@ var _VolumeSnapshotter_serviceDesc = grpc.ServiceDesc{ Metadata: "VolumeSnapshotter.proto", } -func init() { proto.RegisterFile("VolumeSnapshotter.proto", fileDescriptor6) } +func init() { proto.RegisterFile("VolumeSnapshotter.proto", fileDescriptor7) } -var fileDescriptor6 = []byte{ +var fileDescriptor7 = []byte{ // 566 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0xc1, 0x6e, 0xd3, 0x40, 0x10, 0xd5, 0xda, 0x6e, 0x44, 0x26, 0xa5, 0x0a, 0x9b, 0xa4, 0x58, 0x16, 0x04, 0xe3, 0x0b, 0x51, diff --git a/pkg/plugin/mocks/manager.go b/pkg/plugin/mocks/manager.go index ebf09de96..588f9b74d 100644 --- a/pkg/plugin/mocks/manager.go +++ b/pkg/plugin/mocks/manager.go @@ -5,6 +5,7 @@ package mocks import ( mock "github.com/stretchr/testify/mock" velero "github.com/vmware-tanzu/velero/pkg/plugin/velero" + isv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/item_snapshotter/v1" ) // Manager is an autogenerated mock type for the Manager type @@ -200,3 +201,48 @@ func (_m *Manager) GetVolumeSnapshotter(name string) (velero.VolumeSnapshotter, return r0, r1 } + +// GetItemSnapshotter provides a mock function with given fields: name +func (_m *Manager) GetItemSnapshotter(name string) (isv1.ItemSnapshotter, error) { + ret := _m.Called(name) + + var r0 isv1.ItemSnapshotter + if rf, ok := ret.Get(0).(func(string) isv1.ItemSnapshotter); ok { + r0 = rf(name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(isv1.ItemSnapshotter) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} +// GetItemSnapshotters provides a mock function with given fields: +func (_m *Manager) GetItemSnapshotters() ([]isv1.ItemSnapshotter, error) { + ret := _m.Called() + + var r0 []isv1.ItemSnapshotter + if rf, ok := ret.Get(0).(func() []isv1.ItemSnapshotter); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]isv1.ItemSnapshotter) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} \ No newline at end of file diff --git a/pkg/plugin/proto/ItemSnapshotter.proto b/pkg/plugin/proto/ItemSnapshotter.proto new file mode 100644 index 000000000..8aa62a899 --- /dev/null +++ b/pkg/plugin/proto/ItemSnapshotter.proto @@ -0,0 +1,94 @@ +syntax = "proto3"; +package generated; + +import "Shared.proto"; + +message ItemSnapshotterAppliesToRequest { + string plugin = 1; +} + +message ItemSnapshotterAppliesToResponse { + ResourceSelector ResourceSelector = 1; +} + +message AlsoHandlesRequest { + string plugin = 1; + bytes item = 2; + bytes backup = 3; +} + +message AlsoHandlesResponse { + repeated ResourceIdentifier handledItems = 1; +} + +message SnapshotItemRequest { + string plugin = 1; + bytes item = 2; + map params = 3; + bytes backup = 4; +} + +message SnapshotItemResponse { + bytes item = 1; + string snapshotID = 2; + map snapshotMetadata = 3; + repeated ResourceIdentifier additionalItems = 4; + repeated ResourceIdentifier handledItems = 5; +} + +message ProgressRequest { + string plugin = 1; + ResourceIdentifier itemID = 2; + string snapshotID = 3; + bytes backup = 4; +} + +message ProgressResponse { + string phase = 1; + int64 itemsCompleted = 2; + int64 itemsToComplete = 3; + int64 started = 4; + int64 startedNano = 5; + int64 updated = 6; + int64 updatedNano = 7; + string err = 8; +} + +message DeleteItemSnapshotRequest { + string plugin = 1; + string snapshotID = 2; + bytes itemFromBackup = 3; + map metadata = 4; + map params = 5; +} + +message CreateItemFromSnapshotRequest { + string plugin = 1; + bytes item = 2; + string snapshotID = 3; + bytes itemFromBackup = 4; + map snapshotMetadata = 5; + map params = 6; + bytes restore = 7; +} + +message CreateItemFromSnapshotResponse { + bytes item = 1; + repeated ResourceIdentifier additionalItems = 2; + bool skipRestore = 3; +} + +message ItemSnapshotterInitRequest { + string plugin = 1; + map config = 2; +} + +service ItemSnapshotter { + rpc Init(ItemSnapshotterInitRequest) returns (Empty); + rpc AppliesTo(ItemSnapshotterAppliesToRequest) returns (ItemSnapshotterAppliesToResponse); + rpc AlsoHandles(AlsoHandlesRequest) returns (AlsoHandlesResponse); + rpc SnapshotItem(SnapshotItemRequest) returns (SnapshotItemResponse); + rpc Progress(ProgressRequest) returns (ProgressResponse); + rpc DeleteSnapshot(DeleteItemSnapshotRequest) returns (Empty); + rpc CreateItemFromSnapshot(CreateItemFromSnapshotRequest) returns (CreateItemFromSnapshotResponse); +} diff --git a/pkg/plugin/velero/item_snapshotter/v1/item_snapshotter.go b/pkg/plugin/velero/item_snapshotter/v1/item_snapshotter.go new file mode 100644 index 000000000..7d9ddcd00 --- /dev/null +++ b/pkg/plugin/velero/item_snapshotter/v1/item_snapshotter.go @@ -0,0 +1,197 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "context" + "fmt" + "time" + + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + + "k8s.io/apimachinery/pkg/runtime" + + api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" +) + +type AlsoHandlesInput struct { + // Item is the item that will be snapshotted + Item runtime.Unstructured + // Backup is the representation of the backup resource being processed by Velero. + Backup *api.Backup +} + +type SnapshotItemInput struct { + // Item is the item to snapshot + Item runtime.Unstructured + // Params are parameters to the snapshot + Params map[string]string + // Backup is the representation of the backup resource being processed by Velero. + Backup *api.Backup +} + +type SnapshotItemOutput struct { + // UpdatedItem is the Item that should be included in the backup. It can optionally be modified during the Snapshot + UpdatedItem runtime.Unstructured + // SnapshotID identifies the snapshot that was taken + SnapshotID string + // SnapshotMetadata is information in addition to the SnapshotID that the + // plugin wants to store in the backup. SnapshotMetadata will be passed + // back in for DeleteSnapshot and CreateItemFromSnapshot + SnapshotMetadata map[string]string + // AdditionalItems are resources that need to be included in the backup to support this snapshhot + AdditionalItems []velero.ResourceIdentifier + // Items that were handled by this snapshot that should be excluded from the backup + HandledItems []velero.ResourceIdentifier +} + +// ProgressInput contains the input parameters for the ItemSnapshotter's Progress function. +type ProgressInput struct { + // ItemID is the id of item that was stored in the backup + ItemID velero.ResourceIdentifier + // SnapshotID is the snapshot ID returned by ItemSnapshotter + SnapshotID string + // Backup is the representation of the backup resource being processed by Velero. + Backup *api.Backup +} + +// SnapshotPhase is the lifecycle phase of a Velero item snapshot. +type SnapshotPhase string + +const ( + // SnapshotPhaseInProgress means the snapshot of the item has been taken and the point-in-time has been preserved, + // but the snapshot is not ready for use yet + SnapshotPhaseInProgress = SnapshotPhase("InProgress") + + // SnapshotPhaseCompleted means the item snapshot was successfully created and can be restored from + SnapshotPhaseCompleted = SnapshotPhase("Completed") + + // SnapshotPhaseFailed means the item snapshot was unable to be completed + SnapshotPhaseFailed = SnapshotPhase("Failed") +) + +func SnapshotPhaseFromString(phase string) (SnapshotPhase, error) { + switch phase { + case string(SnapshotPhaseInProgress): + return SnapshotPhaseInProgress, nil + case string(SnapshotPhaseCompleted): + return SnapshotPhaseCompleted, nil + case string(SnapshotPhaseFailed): + return SnapshotPhaseFailed, nil + default: + return SnapshotPhase(""), fmt.Errorf("%s is not a valid SnapshotPhase", phase) + } +} + +type ProgressOutput struct { + // Phase of the snapshot. If the phase is SnapshotPhaseFailed, the error will be in the Err string + Phase SnapshotPhase + // Err is a message about the error(s) that occurred during the processing of the snapshot + Err string + // ItemsCompleted is the number of items that have been completed in processing of the snapshot + // This is simply to show progress, when Phase goes to SnapshotPhaseCompleted ItemsCompleted and ItemsToComplete + // should be the same. This could be blocks to copy, files to copy, or anything else. + ItemsCompleted int64 + // ItemsToComplete is the number of items that need to be completed + ItemsToComplete int64 + // Started indicates when processing on the snapshot began (usually the time the snapshot was taken) + Started time.Time + // Updated indicates the time the status was last updated. Time 0 (time.Unix(0, 0)) is returned if unknown. + Updated time.Time +} + +type DeleteSnapshotInput struct { + // SnapshotID is the snapshot to delete + SnapshotID string + // ItemFromBackup is the resource that was included in the backup + ItemFromBackup runtime.Unstructured + // SnapshotMetadata is the metadata that was returned when the snapshot was originally taken + SnapshotMetadata map[string]string + // Params are parameters to the deletion + Params map[string]string +} + +type CreateItemInput struct { + // The snapshotted item at this stage of the restore (RestoreItemActions may + // have modified the item prior to CreateItemFromSnapshot being called) + SnapshottedItem runtime.Unstructured + // SnapshotID is the snapshot to create the item from + SnapshotID string + // ItemFromBackup is the snapshotted item that was stored in the backup + ItemFromBackup runtime.Unstructured + // SnapshotMetadata is the metadata that was returned when the snapshot was originally taken + SnapshotMetadata map[string]string + // Params are parameters to the deletion + Params map[string]string + // Restore is the representation of the restore resource being processed by Velero. + Restore *api.Restore +} + +type CreateItemOutput struct { + // UpdatedItem is the item being restored mutated by ItemAction. + UpdatedItem runtime.Unstructured + + // AdditionalItems is a list of additional related items that should + // be restored. + AdditionalItems []velero.ResourceIdentifier + + // SkipRestore tells velero to stop executing further actions + // on this item, and skip the restore step. When this field's + // value is true, AdditionalItems will be ignored. + SkipRestore bool +} + +// ItemSnapshotter handles snapshots on an individual item being backed up. +type ItemSnapshotter interface { + + // Init prepares the ItemSnapshotter for usage using the provided map of + // configuration key-value pairs. It returns an error if the ItemSnapshotter + // cannot be initialized from the provided config. + Init(config map[string]string) error + + // AppliesTo returns information about which resources this action should be invoked for. + // An ItemSnapshotter's SnapshotItem method will only be invoked on items that match the returned + // selector. A zero-valued ResourceSelector matches all resources. + AppliesTo() (velero.ResourceSelector, error) + + // AlsoHandles is called for each item this ItemSnapshotter should handle and returns any items + // which will be handled by this plugin when snapshotting the item. These items will be excluded from the + // items being backed up. AlsoHandles will be called before SnapshotItem is called. For example, a database may expose + // a database resource that can be snapshotted. If the database uses a PVC that will be snapshotted/backed up as + // part of the database snapshot, that PVC should be returned when AlsoHandles is invoked. This is different from + // AdditionalItems (returned in SnapshotItemOutput and CreateItemOutput) which are specifying additional resources + // that Velero should store in the backup or create. + AlsoHandles(input *AlsoHandlesInput) ([]velero.ResourceIdentifier, error) + + // SnapshotItem causes the ItemSnapshotter to snapshot the specified item. It may also + // perform arbitrary logic with the item being backed up, including mutating the item itself prior to backup. + // The item (unmodified or modified) should be returned, along with an optional slice of ResourceIdentifiers specifying + // additional related items that should be backed up. + // A caller can pass a context that includes a timeout. If the time to take the snapshot exceeds the + // time in the context, the plugin may abort the snapshot. The context timeout does not apply to upload + // time that occurs after SnapshotItem returns + SnapshotItem(ctx context.Context, input *SnapshotItemInput) (*SnapshotItemOutput, error) + + // Progress will return the progress of a snapshot that is being uploaded + Progress(input *ProgressInput) (*ProgressOutput, error) + + // DeleteSnapshot removes a snapshot + DeleteSnapshot(ctx context.Context, input *DeleteSnapshotInput) error + + // CreateItemFromSnapshot creates a new item from the snapshot + CreateItemFromSnapshot(ctx context.Context, input *CreateItemInput) (*CreateItemOutput, error) +} diff --git a/pkg/plugin/velero/item_snapshotter/v1/mocks/item_snapshotter.go b/pkg/plugin/velero/item_snapshotter/v1/mocks/item_snapshotter.go new file mode 100644 index 000000000..940b85426 --- /dev/null +++ b/pkg/plugin/velero/item_snapshotter/v1/mocks/item_snapshotter.go @@ -0,0 +1,157 @@ +// Code generated by mockery v0.0.0-dev. DO NOT EDIT. + +package mocks + +import ( + context "context" + mock "github.com/stretchr/testify/mock" + v1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/item_snapshotter/v1" + + velero "github.com/vmware-tanzu/velero/pkg/plugin/velero" +) + +// ItemSnapshotter is an autogenerated mock type for the ItemSnapshotter type +type ItemSnapshotter struct { + mock.Mock +} + +// AlsoHandles provides a mock function with given fields: input +func (_m *ItemSnapshotter) AlsoHandles(input *v1.AlsoHandlesInput) ([]velero.ResourceIdentifier, error) { + ret := _m.Called(input) + + var r0 []velero.ResourceIdentifier + if rf, ok := ret.Get(0).(func(*v1.AlsoHandlesInput) []velero.ResourceIdentifier); ok { + r0 = rf(input) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]velero.ResourceIdentifier) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*v1.AlsoHandlesInput) error); ok { + r1 = rf(input) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// AppliesTo provides a mock function with given fields: +func (_m *ItemSnapshotter) AppliesTo() (velero.ResourceSelector, error) { + ret := _m.Called() + + var r0 velero.ResourceSelector + if rf, ok := ret.Get(0).(func() velero.ResourceSelector); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(velero.ResourceSelector) + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// CreateItemFromSnapshot provides a mock function with given fields: ctx, input +func (_m *ItemSnapshotter) CreateItemFromSnapshot(ctx context.Context, input *v1.CreateItemInput) (*v1.CreateItemOutput, error) { + ret := _m.Called(ctx, input) + + var r0 *v1.CreateItemOutput + if rf, ok := ret.Get(0).(func(context.Context, *v1.CreateItemInput) *v1.CreateItemOutput); ok { + r0 = rf(ctx, input) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1.CreateItemOutput) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *v1.CreateItemInput) error); ok { + r1 = rf(ctx, input) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// DeleteSnapshot provides a mock function with given fields: ctx, input +func (_m *ItemSnapshotter) DeleteSnapshot(ctx context.Context, input *v1.DeleteSnapshotInput) error { + ret := _m.Called(ctx, input) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *v1.DeleteSnapshotInput) error); ok { + r0 = rf(ctx, input) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Init provides a mock function with given fields: config +func (_m *ItemSnapshotter) Init(config map[string]string) error { + ret := _m.Called(config) + + var r0 error + if rf, ok := ret.Get(0).(func(map[string]string) error); ok { + r0 = rf(config) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Progress provides a mock function with given fields: input +func (_m *ItemSnapshotter) Progress(input *v1.ProgressInput) (*v1.ProgressOutput, error) { + ret := _m.Called(input) + + var r0 *v1.ProgressOutput + if rf, ok := ret.Get(0).(func(*v1.ProgressInput) *v1.ProgressOutput); ok { + r0 = rf(input) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1.ProgressOutput) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*v1.ProgressInput) error); ok { + r1 = rf(input) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SnapshotItem provides a mock function with given fields: ctx, input +func (_m *ItemSnapshotter) SnapshotItem(ctx context.Context, input *v1.SnapshotItemInput) (*v1.SnapshotItemOutput, error) { + ret := _m.Called(ctx, input) + + var r0 *v1.SnapshotItemOutput + if rf, ok := ret.Get(0).(func(context.Context, *v1.SnapshotItemInput) *v1.SnapshotItemOutput); ok { + r0 = rf(ctx, input) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1.SnapshotItemOutput) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *v1.SnapshotItemInput) error); ok { + r1 = rf(ctx, input) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/pkg/plugin/velero/shared.go b/pkg/plugin/velero/shared.go index 138de4886..76c7464ad 100644 --- a/pkg/plugin/velero/shared.go +++ b/pkg/plugin/velero/shared.go @@ -1,5 +1,5 @@ /* -Copyright 2019 the Velero contributors. +Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -48,3 +48,9 @@ type ResourceSelector struct { // for details on syntax. LabelSelector string } + +// Applicable allows actions and plugins to specify which resources they should be invoked for +type Applicable interface { + // AppliesTo returns information about which resources this Responder should be invoked for. + AppliesTo() (ResourceSelector, error) +} diff --git a/pkg/restic/backupper.go b/pkg/restic/backupper.go index cb8e8569a..fd366a36b 100644 --- a/pkg/restic/backupper.go +++ b/pkg/restic/backupper.go @@ -25,6 +25,7 @@ import ( "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/cache" @@ -118,9 +119,10 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api. b.resultsLock.Unlock() var ( - errs []error - podVolumeBackups []*velerov1api.PodVolumeBackup - podVolumes = make(map[string]corev1api.Volume) + errs []error + podVolumeBackups []*velerov1api.PodVolumeBackup + podVolumes = make(map[string]corev1api.Volume) + mountedPodVolumes = sets.String{} ) // put the pod's volumes in a map for efficient lookup below @@ -128,6 +130,12 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api. podVolumes[podVolume.Name] = podVolume } + for _, container := range pod.Spec.Containers { + for _, volumeMount := range container.VolumeMounts { + mountedPodVolumes.Insert(volumeMount.Name) + } + } + var numVolumeSnapshots int for _, volumeName := range volumesToBackup { volume, ok := podVolumes[volumeName] @@ -145,6 +153,11 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api. } } + // ignore non-running pods + if pod.Status.Phase != corev1api.PodRunning { + log.Warnf("Skipping volume %s in pod %s/%s - pod not running", volumeName, pod.Namespace, pod.Name) + continue + } // hostPath volumes are not supported because they're not mounted into /var/lib/kubelet/pods, so our // daemonset pod has no way to access their data. isHostPath, err := isHostPathVolume(&volume, pvc, b.pvClient.PersistentVolumes()) @@ -157,8 +170,10 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api. continue } - // emptyDir volumes on finished pods are not supported because the volume is already gone and would result in an error - if (pod.Status.Phase == corev1api.PodSucceeded || pod.Status.Phase == corev1api.PodFailed) && volume.EmptyDir != nil { + // volumes that are not mounted by any container should not be backed up, because + // its directory is not created + if !mountedPodVolumes.Has(volumeName) { + log.Warnf("Volume %s is declared in pod %s/%s but not mounted by any container, skipping", volumeName, pod.Namespace, pod.Name) continue } diff --git a/pkg/restic/common.go b/pkg/restic/common.go index e122ff4bb..e8648cd6c 100644 --- a/pkg/restic/common.go +++ b/pkg/restic/common.go @@ -100,15 +100,18 @@ func isPVBMatchPod(pvb *velerov1api.PodVolumeBackup, podName string, namespace s return podName == pvb.Spec.Pod.Name && namespace == pvb.Spec.Pod.Namespace } -// volumeIsProjected checks if the given volume exists in the list of podVolumes -// and returns true if the volume has a projected source -func volumeIsProjected(volumeName string, podVolumes []corev1api.Volume) bool { - for _, volume := range podVolumes { - if volume.Name == volumeName && volume.Projected != nil { - return true +// volumeHasNonRestorableSource checks if the given volume exists in the list of podVolumes +// and returns true if the volume's source is not restorable. This is true for volumes with +// a Projected or DownwardAPI source. +func volumeHasNonRestorableSource(volumeName string, podVolumes []corev1api.Volume) bool { + var volume corev1api.Volume + for _, v := range podVolumes { + if v.Name == volumeName { + volume = v + break } } - return false + return volume.Projected != nil || volume.DownwardAPI != nil } // GetVolumeBackupsForPod returns a map, of volume name -> snapshot id, @@ -127,10 +130,10 @@ func GetVolumeBackupsForPod(podVolumeBackups []*velerov1api.PodVolumeBackup, pod continue } - // If the volume came from a projected source, skip its restore. + // If the volume came from a projected or DownwardAPI source, skip its restore. // This allows backups affected by https://github.com/vmware-tanzu/velero/issues/3863 - // to be restored successfully. - if volumeIsProjected(pvb.Spec.Volume, pod.Spec.Volumes) { + // or https://github.com/vmware-tanzu/velero/issues/4053 to be restored successfully. + if volumeHasNonRestorableSource(pvb.Spec.Volume, pod.Spec.Volumes) { continue } @@ -205,6 +208,10 @@ func GetPodVolumesUsingRestic(pod *corev1api.Pod, defaultVolumesToRestic bool) [ if pv.Projected != nil { continue } + // don't backup DownwardAPI volumes, all data in those come from kube state. + if pv.DownwardAPI != nil { + continue + } // don't backup volumes that are included in the exclude list. if contains(volsToExclude, pv.Name) { continue diff --git a/pkg/restic/common_test.go b/pkg/restic/common_test.go index 7e102ca97..954defee0 100644 --- a/pkg/restic/common_test.go +++ b/pkg/restic/common_test.go @@ -152,6 +152,30 @@ func TestGetVolumeBackupsForPod(t *testing.T) { sourcePodNs: "TestNS", expected: map[string]string{"pvb-non-projected": "snapshot1"}, }, + { + name: "volumes from PVBs that correspond to a pod volume from a DownwardAPI source are not returned", + podVolumeBackups: []*velerov1api.PodVolumeBackup{ + builder.ForPodVolumeBackup("velero", "pvb-1").PodName("TestPod").PodNamespace("TestNS").SnapshotID("snapshot1").Volume("pvb-non-downwardapi").Result(), + builder.ForPodVolumeBackup("velero", "pvb-1").PodName("TestPod").PodNamespace("TestNS").SnapshotID("snapshot2").Volume("pvb-downwardapi").Result(), + }, + podVolumes: []corev1api.Volume{ + { + Name: "pvb-non-downwardapi", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{}, + }, + }, + { + Name: "pvb-downwardapi", + VolumeSource: corev1api.VolumeSource{ + DownwardAPI: &corev1api.DownwardAPIVolumeSource{}, + }, + }, + }, + podName: "TestPod", + sourcePodNs: "TestNS", + expected: map[string]string{"pvb-non-downwardapi": "snapshot1"}, + }, } for _, test := range tests { @@ -568,6 +592,39 @@ func TestGetPodVolumesUsingRestic(t *testing.T) { }, expected: []string{"resticPV1", "resticPV2", "resticPV3"}, }, + { + name: "should exclude DownwardAPI volumes", + defaultVolumesToRestic: true, + pod: &corev1api.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + VolumesToExcludeAnnotation: "nonResticPV1,nonResticPV2,nonResticPV3", + }, + }, + Spec: corev1api.PodSpec{ + Volumes: []corev1api.Volume{ + {Name: "resticPV1"}, {Name: "resticPV2"}, {Name: "resticPV3"}, + { + Name: "downwardAPI", + VolumeSource: corev1api.VolumeSource{ + DownwardAPI: &corev1api.DownwardAPIVolumeSource{ + Items: []corev1api.DownwardAPIVolumeFile{ + { + Path: "labels", + FieldRef: &corev1api.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.labels", + }, + }, + }, + }, + }, + }, + }, + }, + }, + expected: []string{"resticPV1", "resticPV2", "resticPV3"}, + }, } for _, tc := range testCases { @@ -656,7 +713,7 @@ func TestIsPVBMatchPod(t *testing.T) { } } -func TestVolumeIsProjected(t *testing.T) { +func TestVolumeHasNonRestorableSource(t *testing.T) { testCases := []struct { name string volumeName string @@ -668,7 +725,7 @@ func TestVolumeIsProjected(t *testing.T) { volumeName: "missing-volume", podVolumes: []corev1api.Volume{ { - Name: "non-projected", + Name: "restorable", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{}, }, @@ -679,15 +736,21 @@ func TestVolumeIsProjected(t *testing.T) { Projected: &corev1api.ProjectedVolumeSource{}, }, }, + { + Name: "downwardapi", + VolumeSource: corev1api.VolumeSource{ + DownwardAPI: &corev1api.DownwardAPIVolumeSource{}, + }, + }, }, expected: false, }, { - name: "volume name in list of volumes but not projected", - volumeName: "non-projected", + name: "volume name in list of volumes but not projected or DownwardAPI", + volumeName: "restorable", podVolumes: []corev1api.Volume{ { - Name: "non-projected", + Name: "restorable", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{}, }, @@ -698,6 +761,12 @@ func TestVolumeIsProjected(t *testing.T) { Projected: &corev1api.ProjectedVolumeSource{}, }, }, + { + Name: "downwardapi", + VolumeSource: corev1api.VolumeSource{ + DownwardAPI: &corev1api.DownwardAPIVolumeSource{}, + }, + }, }, expected: false, }, @@ -706,7 +775,7 @@ func TestVolumeIsProjected(t *testing.T) { volumeName: "projected", podVolumes: []corev1api.Volume{ { - Name: "non-projected", + Name: "restorable", VolumeSource: corev1api.VolumeSource{ PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{}, }, @@ -717,6 +786,37 @@ func TestVolumeIsProjected(t *testing.T) { Projected: &corev1api.ProjectedVolumeSource{}, }, }, + { + Name: "downwardapi", + VolumeSource: corev1api.VolumeSource{ + DownwardAPI: &corev1api.DownwardAPIVolumeSource{}, + }, + }, + }, + expected: true, + }, + { + name: "volume name in list of volumes and is a DownwardAPI volume", + volumeName: "downwardapi", + podVolumes: []corev1api.Volume{ + { + Name: "restorable", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{}, + }, + }, + { + Name: "projected", + VolumeSource: corev1api.VolumeSource{ + Projected: &corev1api.ProjectedVolumeSource{}, + }, + }, + { + Name: "downwardapi", + VolumeSource: corev1api.VolumeSource{ + DownwardAPI: &corev1api.DownwardAPIVolumeSource{}, + }, + }, }, expected: true, }, @@ -724,7 +824,7 @@ func TestVolumeIsProjected(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - actual := volumeIsProjected(tc.volumeName, tc.podVolumes) + actual := volumeHasNonRestorableSource(tc.volumeName, tc.podVolumes) assert.Equal(t, tc.expected, actual) }) diff --git a/pkg/restic/config.go b/pkg/restic/config.go index b5471fa37..4cde8336a 100644 --- a/pkg/restic/config.go +++ b/pkg/restic/config.go @@ -74,10 +74,9 @@ func getRepoPrefix(location *velerov1api.BackupStorageLocation) (string, error) region, err = getAWSBucketRegion(bucket) } if err != nil { - url = "s3.amazonaws.com" - } else { - url = fmt.Sprintf("s3-%s.amazonaws.com", region) + return "", errors.Wrapf(err, "failed to detect the region via bucket: %s", bucket) } + url = fmt.Sprintf("s3-%s.amazonaws.com", region) } return fmt.Sprintf("s3:%s/%s", url, path.Join(bucket, prefix)), nil diff --git a/pkg/restic/config_test.go b/pkg/restic/config_test.go index ad2413104..8418d6808 100644 --- a/pkg/restic/config_test.go +++ b/pkg/restic/config_test.go @@ -85,7 +85,8 @@ func TestGetRepoIdentifier(t *testing.T) { getAWSBucketRegion: func(string) (string, error) { return "", errors.New("no region found") }, - expected: "s3:s3.amazonaws.com/bucket/restic/repo-1", + expected: "", + expectedErr: "failed to detect the region via bucket: bucket: no region found", }, { name: "s3.s3-.amazonaws.com URL format is used if region can be determined for AWS BSL", diff --git a/pkg/restore/admissionwebhook_config_action.go b/pkg/restore/admissionwebhook_config_action.go new file mode 100644 index 000000000..8fd5c1693 --- /dev/null +++ b/pkg/restore/admissionwebhook_config_action.go @@ -0,0 +1,89 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package restore + +import ( + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/vmware-tanzu/velero/pkg/plugin/velero" +) + +// AdmissionWebhookConfigurationAction is a RestoreItemAction plugin applicable to mutatingwebhookconfiguration and +// validatingwebhookconfiguration to reset the invalid value for "sideEffects" of the webhooks. +// More background please refer to https://github.com/vmware-tanzu/velero/issues/3516 +type AdmissionWebhookConfigurationAction struct { + logger logrus.FieldLogger +} + +// NewAdmissionWebhookConfigurationAction creates a new instance of AdmissionWebhookConfigurationAction +func NewAdmissionWebhookConfigurationAction(logger logrus.FieldLogger) *AdmissionWebhookConfigurationAction { + return &AdmissionWebhookConfigurationAction{logger: logger} +} + +// AppliesTo implements the RestoreItemAction plugin interface method. +func (a *AdmissionWebhookConfigurationAction) AppliesTo() (velero.ResourceSelector, error) { + return velero.ResourceSelector{ + IncludedResources: []string{"mutatingwebhookconfigurations", "validatingwebhookconfigurations"}, + }, nil +} + +// Execute will reset the value of "sideEffects" attribute of each item in the "webhooks" list to "None" if they are invalid values for +// v1, such as "Unknown" or "Some" +func (a *AdmissionWebhookConfigurationAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + a.logger.Info("Executing ChangeStorageClassAction") + defer a.logger.Info("Done executing ChangeStorageClassAction") + + item := input.Item + apiVersion, _, err := unstructured.NestedString(item.UnstructuredContent(), "apiVersion") + if err != nil { + return nil, errors.Wrap(err, "failed to get the apiVersion from input item") + } + name, _, _ := unstructured.NestedString(item.UnstructuredContent(), "metadata", "name") + logger := a.logger.WithField("resource_name", name) + if apiVersion != "admissionregistration.k8s.io/v1" { + logger.Infof("unable to handle api version: %s, skip", apiVersion) + return velero.NewRestoreItemActionExecuteOutput(input.Item), nil + } + webhooks, ok, err := unstructured.NestedSlice(item.UnstructuredContent(), "webhooks") + if err != nil { + return nil, errors.Wrap(err, "failed to get webhooks slice from input item") + } + if !ok { + logger.Info("webhooks is not set, skip") + return velero.NewRestoreItemActionExecuteOutput(input.Item), nil + } + newWebhooks := make([]interface{}, 0) + for i, entry := range webhooks { + logger2 := logger.WithField("index", i) + obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&entry) + if err != nil { + logger2.Errorf("failed to convert the webhook entry, error: %v, it will be dropped", err) + continue + } + s, _, _ := unstructured.NestedString(obj, "sideEffects") + if s != "None" && s != "NoneOnDryRun" { + logger2.Infof("reset the invalid sideEffects value '%s' to 'None'", s) + obj["sideEffects"] = "None" + } + newWebhooks = append(newWebhooks, obj) + } + item.UnstructuredContent()["webhooks"] = newWebhooks + return velero.NewRestoreItemActionExecuteOutput(item), nil +} diff --git a/pkg/restore/admissionwebhook_config_action_test.go b/pkg/restore/admissionwebhook_config_action_test.go new file mode 100644 index 000000000..c6c31d221 --- /dev/null +++ b/pkg/restore/admissionwebhook_config_action_test.go @@ -0,0 +1,199 @@ +package restore + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + velerotest "github.com/vmware-tanzu/velero/pkg/test" +) + +func TestNewAdmissionWebhookConfigurationActionExecute(t *testing.T) { + action := NewAdmissionWebhookConfigurationAction(velerotest.NewLogger()) + cases := []struct { + name string + itemJSON string + wantErr bool + NoneSideEffectsIndex []int // the indexes with sideEffects that arereset to None + NotNoneSideEffectsIndex []int // the indexes with sideEffects that are not reset to None + }{ + { + name: "v1 mutatingwebhookconfiguration with sideEffects as Unknown", + itemJSON: `{ + "apiVersion": "admissionregistration.k8s.io/v1", + "kind": "MutatingWebhookConfiguration", + "metadata": { + "name": "my-test-mutating" + }, + "webhooks": [ + { + "clientConfig": { + "url": "https://mytest.org" + }, + "rules": [ + { + "apiGroups": [ + "" + ], + "apiVersions": [ + "v1" + ], + "operations": [ + "CREATE" + ], + "resources": [ + "pods" + ], + "scope": "Namespaced" + } + ], + "sideEffects": "Unknown" + } + ] + }`, + wantErr: false, + NoneSideEffectsIndex: []int{0}, + }, + { + name: "v1 validatingwebhookconfiguration with sideEffects as Some", + itemJSON: `{ + "apiVersion": "admissionregistration.k8s.io/v1", + "kind": "ValidatingWebhookConfiguration", + "metadata": { + "name": "my-test-validating" + }, + "webhooks": [ + { + "clientConfig": { + "url": "https://mytest.org" + }, + "rules": [ + { + "apiGroups": [ + "" + ], + "apiVersions": [ + "v1" + ], + "operations": [ + "CREATE" + ], + "resources": [ + "pods" + ], + "scope": "Namespaced" + } + ], + "sideEffects": "Some" + } + ] + }`, + wantErr: false, + NoneSideEffectsIndex: []int{0}, + }, + { + name: "v1beta1 validatingwebhookconfiguration with sideEffects as Some, nothing should change", + itemJSON: `{ + "apiVersion": "admissionregistration.k8s.io/v1beta1", + "kind": "ValidatingWebhookConfiguration", + "metadata": { + "name": "my-test-validating" + }, + "webhooks": [ + { + "clientConfig": { + "url": "https://mytest.org" + }, + "rules": [ + { + "apiGroups": [ + "" + ], + "apiVersions": [ + "v1" + ], + "operations": [ + "CREATE" + ], + "resources": [ + "pods" + ], + "scope": "Namespaced" + } + ], + "sideEffects": "Some" + } + ] + }`, + wantErr: false, + NotNoneSideEffectsIndex: []int{0}, + }, + { + name: "v1 validatingwebhookconfiguration with multiple invalid sideEffects", + itemJSON: `{ + "apiVersion": "admissionregistration.k8s.io/v1", + "kind": "ValidatingWebhookConfiguration", + "metadata": { + "name": "my-test-validating" + }, + "webhooks": [ + { + "clientConfig": { + "url": "https://mytest.org" + }, + "sideEffects": "Some" + }, + { + "clientConfig": { + "url": "https://mytest2.org" + }, + "sideEffects": "Some" + } + ] + }`, + wantErr: false, + NoneSideEffectsIndex: []int{0, 1}, + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + o := map[string]interface{}{} + json.Unmarshal([]byte(tt.itemJSON), &o) + input := &velero.RestoreItemActionExecuteInput{ + Item: &unstructured.Unstructured{ + Object: o, + }, + } + output, err := action.Execute(input) + if tt.wantErr { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + if tt.NoneSideEffectsIndex != nil { + wb, _, err := unstructured.NestedSlice(output.UpdatedItem.UnstructuredContent(), "webhooks") + assert.Nil(t, err) + for _, i := range tt.NoneSideEffectsIndex { + it, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&wb[i]) + assert.Nil(t, err) + s := it["sideEffects"].(string) + assert.Equal(t, "None", s) + } + } + if tt.NotNoneSideEffectsIndex != nil { + wb, _, err := unstructured.NestedSlice(output.UpdatedItem.UnstructuredContent(), "webhooks") + assert.Nil(t, err) + for _, i := range tt.NotNoneSideEffectsIndex { + it, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&wb[i]) + assert.Nil(t, err) + s := it["sideEffects"].(string) + assert.NotEqual(t, "None", s) + } + } + }) + } +} diff --git a/pkg/restore/change_storageclass_action.go b/pkg/restore/change_storageclass_action.go index effa83dbc..5714a1a7f 100644 --- a/pkg/restore/change_storageclass_action.go +++ b/pkg/restore/change_storageclass_action.go @@ -21,8 +21,11 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" storagev1client "k8s.io/client-go/kubernetes/typed/storage/v1" @@ -55,7 +58,7 @@ func NewChangeStorageClassAction( // be run for. func (a *ChangeStorageClassAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{ - IncludedResources: []string{"persistentvolumeclaims", "persistentvolumes"}, + IncludedResources: []string{"persistentvolumeclaims", "persistentvolumes", "statefulsets"}, }, nil } @@ -87,33 +90,72 @@ func (a *ChangeStorageClassAction) Execute(input *velero.RestoreItemActionExecut "name": obj.GetName(), }) - // use the unstructured helpers here since this code is for both PVs and PVCs, and the - // field names are the same for both types. - storageClass, _, err := unstructured.NestedString(obj.UnstructuredContent(), "spec", "storageClassName") - if err != nil { - return nil, errors.Wrap(err, "error getting item's spec.storageClassName") + // change StatefulSet volumeClaimTemplates storageClassName + if obj.GetKind() == "StatefulSet" { + sts := new(appsv1.StatefulSet) + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), sts); err != nil { + return nil, err + } + + if len(sts.Spec.VolumeClaimTemplates) > 0 { + for index, pvc := range sts.Spec.VolumeClaimTemplates { + exists, newStorageClass, err := a.isStorageClassExist(log, *pvc.Spec.StorageClassName, config) + if err != nil { + return nil, err + } else if !exists { + continue + } + + log.Infof("Updating item's storage class name to %s", newStorageClass) + sts.Spec.VolumeClaimTemplates[index].Spec.StorageClassName = &newStorageClass + } + + newObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(sts) + if err != nil { + return nil, errors.Wrap(err, "convert obj to StatefulSet failed") + } + obj.Object = newObj + } + } else { + // use the unstructured helpers here since this code is for both PVs and PVCs, and the + // field names are the same for both types. + storageClass, _, err := unstructured.NestedString(obj.UnstructuredContent(), "spec", "storageClassName") + if err != nil { + return nil, errors.Wrap(err, "error getting item's spec.storageClassName") + } + + exists, newStorageClass, err := a.isStorageClassExist(log, storageClass, config) + if err != nil { + return nil, err + } else if !exists { + return velero.NewRestoreItemActionExecuteOutput(input.Item), nil + } + + log.Infof("Updating item's storage class name to %s", newStorageClass) + + if err := unstructured.SetNestedField(obj.UnstructuredContent(), newStorageClass, "spec", "storageClassName"); err != nil { + return nil, errors.Wrap(err, "unable to set item's spec.storageClassName") + } } + return velero.NewRestoreItemActionExecuteOutput(obj), nil +} + +func (a *ChangeStorageClassAction) isStorageClassExist(log *logrus.Entry, storageClass string, cm *corev1.ConfigMap) (exists bool, newStorageClass string, err error) { if storageClass == "" { log.Debug("Item has no storage class specified") - return velero.NewRestoreItemActionExecuteOutput(input.Item), nil + return false, "", nil } - newStorageClass, ok := config.Data[storageClass] + newStorageClass, ok := cm.Data[storageClass] if !ok { log.Debugf("No mapping found for storage class %s", storageClass) - return velero.NewRestoreItemActionExecuteOutput(input.Item), nil + return false, "", nil } // validate that new storage class exists if _, err := a.storageClassClient.Get(context.TODO(), newStorageClass, metav1.GetOptions{}); err != nil { - return nil, errors.Wrapf(err, "error getting storage class %s from API", newStorageClass) + return false, "", errors.Wrapf(err, "error getting storage class %s from API", newStorageClass) } - log.Infof("Updating item's storage class name to %s", newStorageClass) - - if err := unstructured.SetNestedField(obj.UnstructuredContent(), newStorageClass, "spec", "storageClassName"); err != nil { - return nil, errors.Wrap(err, "unable to set item's spec.storageClassName") - } - - return velero.NewRestoreItemActionExecuteOutput(obj), nil + return true, newStorageClass, nil } diff --git a/pkg/restore/change_storageclass_action_test.go b/pkg/restore/change_storageclass_action_test.go index 9e6005851..65de052db 100644 --- a/pkg/restore/change_storageclass_action_test.go +++ b/pkg/restore/change_storageclass_action_test.go @@ -41,16 +41,17 @@ import ( // desired result. func TestChangeStorageClassActionExecute(t *testing.T) { tests := []struct { - name string - pvOrPVC interface{} - configMap *corev1api.ConfigMap - storageClass *storagev1api.StorageClass - want interface{} - wantErr error + name string + pvOrPvcOrSTS interface{} + configMap *corev1api.ConfigMap + storageClass *storagev1api.StorageClass + storageClassSlice []*storagev1api.StorageClass + want interface{} + wantErr error }{ { - name: "a valid mapping for a persistent volume is applied correctly", - pvOrPVC: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), + name: "a valid mapping for a persistent volume is applied correctly", + pvOrPvcOrSTS: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "storageclass-2"). @@ -59,8 +60,8 @@ func TestChangeStorageClassActionExecute(t *testing.T) { want: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-2").Result(), }, { - name: "a valid mapping for a persistent volume claim is applied correctly", - pvOrPVC: builder.ForPersistentVolumeClaim("velero", "pvc-1").StorageClass("storageclass-1").Result(), + name: "a valid mapping for a persistent volume claim is applied correctly", + pvOrPvcOrSTS: builder.ForPersistentVolumeClaim("velero", "pvc-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "storageclass-2"). @@ -69,8 +70,8 @@ func TestChangeStorageClassActionExecute(t *testing.T) { want: builder.ForPersistentVolumeClaim("velero", "pvc-1").StorageClass("storageclass-2").Result(), }, { - name: "when no config map exists for the plugin, the item is returned as-is", - pvOrPVC: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), + name: "when no config map exists for the plugin, the item is returned as-is", + pvOrPvcOrSTS: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/some-other-plugin", "RestoreItemAction")). Data("storageclass-1", "storageclass-2"). @@ -78,16 +79,16 @@ func TestChangeStorageClassActionExecute(t *testing.T) { want: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), }, { - name: "when no storage class mappings exist in the plugin config map, the item is returned as-is", - pvOrPVC: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), + name: "when no storage class mappings exist in the plugin config map, the item is returned as-is", + pvOrPvcOrSTS: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). Result(), want: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), }, { - name: "when persistent volume has no storage class, the item is returned as-is", - pvOrPVC: builder.ForPersistentVolume("pv-1").Result(), + name: "when persistent volume has no storage class, the item is returned as-is", + pvOrPvcOrSTS: builder.ForPersistentVolume("pv-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "storageclass-2"). @@ -95,8 +96,8 @@ func TestChangeStorageClassActionExecute(t *testing.T) { want: builder.ForPersistentVolume("pv-1").Result(), }, { - name: "when persistent volume claim has no storage class, the item is returned as-is", - pvOrPVC: builder.ForPersistentVolumeClaim("velero", "pvc-1").Result(), + name: "when persistent volume claim has no storage class, the item is returned as-is", + pvOrPvcOrSTS: builder.ForPersistentVolumeClaim("velero", "pvc-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "storageclass-2"). @@ -104,8 +105,8 @@ func TestChangeStorageClassActionExecute(t *testing.T) { want: builder.ForPersistentVolumeClaim("velero", "pvc-1").Result(), }, { - name: "when persistent volume's storage class has no mapping in the config map, the item is returned as-is", - pvOrPVC: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), + name: "when persistent volume's storage class has no mapping in the config map, the item is returned as-is", + pvOrPvcOrSTS: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-3", "storageclass-4"). @@ -113,8 +114,8 @@ func TestChangeStorageClassActionExecute(t *testing.T) { want: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), }, { - name: "when persistent volume claim's storage class has no mapping in the config map, the item is returned as-is", - pvOrPVC: builder.ForPersistentVolumeClaim("velero", "pvc-1").StorageClass("storageclass-1").Result(), + name: "when persistent volume claim's storage class has no mapping in the config map, the item is returned as-is", + pvOrPvcOrSTS: builder.ForPersistentVolumeClaim("velero", "pvc-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-3", "storageclass-4"). @@ -122,8 +123,8 @@ func TestChangeStorageClassActionExecute(t *testing.T) { want: builder.ForPersistentVolumeClaim("velero", "pvc-1").StorageClass("storageclass-1").Result(), }, { - name: "when persistent volume's storage class is mapped to a nonexistent storage class, an error is returned", - pvOrPVC: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), + name: "when persistent volume's storage class is mapped to a nonexistent storage class, an error is returned", + pvOrPvcOrSTS: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "nonexistent-storage-class"). @@ -131,8 +132,81 @@ func TestChangeStorageClassActionExecute(t *testing.T) { wantErr: errors.New("error getting storage class nonexistent-storage-class from API: storageclasses.storage.k8s.io \"nonexistent-storage-class\" not found"), }, { - name: "when persistent volume claim's storage class is mapped to a nonexistent storage class, an error is returned", - pvOrPVC: builder.ForPersistentVolumeClaim("velero", "pvc-1").StorageClass("storageclass-1").Result(), + name: "when persistent volume claim's storage class is mapped to a nonexistent storage class, an error is returned", + pvOrPvcOrSTS: builder.ForPersistentVolumeClaim("velero", "pvc-1").StorageClass("storageclass-1").Result(), + configMap: builder.ForConfigMap("velero", "change-storage-classs"). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + Data("storageclass-1", "nonexistent-storage-class"). + Result(), + wantErr: errors.New("error getting storage class nonexistent-storage-class from API: storageclasses.storage.k8s.io \"nonexistent-storage-class\" not found"), + }, + { + name: "when statefulset's VolumeClaimTemplates has only one pvc, a valid mapping for a statefulset is applied correctly", + pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), + configMap: builder.ForConfigMap("velero", "change-storage-classs"). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + Data("storageclass-1", "storageclass-2"). + Result(), + storageClass: builder.ForStorageClass("storageclass-2").Result(), + want: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-2").Result(), + }, + { + name: "when statefulset's VolumeClaimTemplates has more than one same pvc's storageClassName, a valid mapping for a statefulset is applied correctly", + pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1", "storageclass-1").Result(), + configMap: builder.ForConfigMap("velero", "change-storage-classs"). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + Data("storageclass-1", "storageclass-2", "storageclass-3", "storageclass-4"). + Result(), + storageClass: builder.ForStorageClass("storageclass-2").Result(), + want: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-2", "storageclass-2").Result(), + }, + { + name: "when statefulset's VolumeClaimTemplates has more than one different pvc's storageClassName, a valid mapping for a statefulset is applied correctly", + pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1", "storageclass-2", "storageclass-3").Result(), + configMap: builder.ForConfigMap("velero", "change-storage-classs"). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + Data("storageclass-1", "storageclass-a", "storageclass-2", "storageclass-b", "storageclass-3", "storageclass-c"). + Result(), + storageClassSlice: builder.ForStorageClassSlice("storageclass-a", "storageclass-b", "storageclass-c").SliceResult(), + want: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-a", "storageclass-b", "storageclass-c").Result(), + }, + { + name: "when no config map exists for the plugin, the statefulset item is returned as-is", + pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), + configMap: builder.ForConfigMap("velero", "change-storage-classs"). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/some-other-plugin", "RestoreItemAction")). + Data("storageclass-1", "storageclass-2"). + Result(), + want: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), + }, + { + name: "when no storage class mappings exist in the plugin config map, the statefulset item is returned as-is", + pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), + configMap: builder.ForConfigMap("velero", "change-storage-classs"). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + Result(), + want: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), + }, + { + name: "when persistent volume claim has no storage class, the statefulset item is returned as-is", + pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").Result(), + configMap: builder.ForConfigMap("velero", "change-storage-classs"). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + Result(), + want: builder.ForStatefulSet("velero", "sts-1").Result(), + }, + { + name: "when statefulset's storage class has no mapping in the config map, the item is returned as-is", + pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), + configMap: builder.ForConfigMap("velero", "change-storage-classs"). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + Data("storageclass-3", "storageclass-4"). + Result(), + want: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), + }, + { + name: "when statefulset's storage class is mapped to a nonexistent storage class, an error is returned", + pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "nonexistent-storage-class"). @@ -161,7 +235,14 @@ func TestChangeStorageClassActionExecute(t *testing.T) { require.NoError(t, err) } - unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pvOrPVC) + if tc.storageClassSlice != nil { + for _, storageClass := range tc.storageClassSlice { + _, err := clientset.StorageV1().StorageClasses().Create(context.TODO(), storageClass, metav1.CreateOptions{}) + require.NoError(t, err) + } + } + + unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pvOrPvcOrSTS) require.NoError(t, err) input := &velero.RestoreItemActionExecuteInput{ diff --git a/pkg/restore/job_action.go b/pkg/restore/job_action.go index ecbea69da..fbaf30b24 100644 --- a/pkg/restore/job_action.go +++ b/pkg/restore/job_action.go @@ -1,5 +1,5 @@ /* -Copyright 2017 the Velero contributors. +Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/restore/merge_service_account.go b/pkg/restore/merge_service_account.go index 279ce9979..dd210f10f 100644 --- a/pkg/restore/merge_service_account.go +++ b/pkg/restore/merge_service_account.go @@ -1,5 +1,5 @@ /* -Copyright 2018 the Velero contributors. +Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package restore import ( diff --git a/pkg/restore/prioritize_group_version.go b/pkg/restore/prioritize_group_version.go index a7a69e00a..e68b867d3 100644 --- a/pkg/restore/prioritize_group_version.go +++ b/pkg/restore/prioritize_group_version.go @@ -221,7 +221,7 @@ func userPriorityConfigMap() (*corev1.ConfigMap, error) { return nil, errors.Wrap(err, "getting Kube client") } - cm, err := kc.CoreV1().ConfigMaps("velero").Get( + cm, err := kc.CoreV1().ConfigMaps(fc.Namespace()).Get( context.Background(), "enableapigroupversions", metav1.GetOptions{}, diff --git a/pkg/restore/restore.go b/pkg/restore/restore.go index bde767773..7eb8c8136 100644 --- a/pkg/restore/restore.go +++ b/pkg/restore/restore.go @@ -56,6 +56,7 @@ import ( listers "github.com/vmware-tanzu/velero/pkg/generated/listers/velero/v1" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/label" + "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/podexec" "github.com/vmware-tanzu/velero/pkg/restic" @@ -66,14 +67,6 @@ import ( "github.com/vmware-tanzu/velero/pkg/volume" ) -// These annotations are taken from the Kubernetes persistent volume/persistent volume claim controller. -// They cannot be directly importing because they are part of the kubernetes/kubernetes package, and importing that package is unsupported. -// Their values are well-known and slow changing. They're duplicated here as constants to provide compile-time checking. -// Originals can be found in kubernetes/kubernetes/pkg/controller/volume/persistentvolume/util/util.go. -const KubeAnnBindCompleted = "pv.kubernetes.io/bind-completed" -const KubeAnnBoundByController = "pv.kubernetes.io/bound-by-controller" -const KubeAnnDynamicallyProvisioned = "pv.kubernetes.io/provisioned-by" - type VolumeSnapshotterGetter interface { GetVolumeSnapshotter(name string) (velero.VolumeSnapshotter, error) } @@ -96,6 +89,13 @@ type Restorer interface { snapshotLocationLister listers.VolumeSnapshotLocationLister, volumeSnapshotterGetter VolumeSnapshotterGetter, ) (Result, Result) + RestoreWithResolvers( + req Request, + restoreItemActionResolver framework.RestoreItemActionResolver, + itemSnapshotterResolver framework.ItemSnapshotterResolver, + snapshotLocationLister listers.VolumeSnapshotLocationLister, + volumeSnapshotterGetter VolumeSnapshotterGetter, + ) (Result, Result) } // kubernetesRestorer implements Restorer for restoring into a Kubernetes cluster. @@ -161,6 +161,18 @@ func (kr *kubernetesRestorer) Restore( actions []velero.RestoreItemAction, snapshotLocationLister listers.VolumeSnapshotLocationLister, volumeSnapshotterGetter VolumeSnapshotterGetter, +) (Result, Result) { + resolver := framework.NewRestoreItemActionResolver(actions) + snapshotItemResolver := framework.NewItemSnapshotterResolver(nil) + return kr.RestoreWithResolvers(req, resolver, snapshotItemResolver, snapshotLocationLister, volumeSnapshotterGetter) +} + +func (kr *kubernetesRestorer) RestoreWithResolvers( + req Request, + restoreItemActionResolver framework.RestoreItemActionResolver, + itemSnapshotterResolver framework.ItemSnapshotterResolver, + snapshotLocationLister listers.VolumeSnapshotLocationLister, + volumeSnapshotterGetter VolumeSnapshotterGetter, ) (Result, Result) { // metav1.LabelSelectorAsSelector converts a nil LabelSelector to a // Nothing Selector, i.e. a selector that matches nothing. We want @@ -188,7 +200,12 @@ func (kr *kubernetesRestorer) Restore( Includes(req.Restore.Spec.IncludedNamespaces...). Excludes(req.Restore.Spec.ExcludedNamespaces...) - resolvedActions, err := resolveActions(actions, kr.discoveryHelper) + resolvedActions, err := restoreItemActionResolver.ResolveActions(kr.discoveryHelper) + if err != nil { + return Result{}, Result{Velero: []string{err.Error()}} + } + + resolvedItemSnapshotterActions, err := itemSnapshotterResolver.ResolveActions(kr.discoveryHelper) if err != nil { return Result{}, Result{Velero: []string{err.Error()}} } @@ -251,7 +268,8 @@ func (kr *kubernetesRestorer) Restore( dynamicFactory: kr.dynamicFactory, fileSystem: kr.fileSystem, namespaceClient: kr.namespaceClient, - actions: resolvedActions, + restoreItemActions: resolvedActions, + itemSnapshotterActions: resolvedItemSnapshotterActions, volumeSnapshotterGetter: volumeSnapshotterGetter, resticRestorer: resticRestorer, resticErrs: make(chan error), @@ -277,46 +295,6 @@ func (kr *kubernetesRestorer) Restore( return restoreCtx.execute() } -type resolvedAction struct { - velero.RestoreItemAction - - resourceIncludesExcludes *collections.IncludesExcludes - namespaceIncludesExcludes *collections.IncludesExcludes - selector labels.Selector -} - -func resolveActions(actions []velero.RestoreItemAction, helper discovery.Helper) ([]resolvedAction, error) { - var resolved []resolvedAction - - for _, action := range actions { - resourceSelector, err := action.AppliesTo() - if err != nil { - return nil, err - } - - resources := collections.GetResourceIncludesExcludes(helper, resourceSelector.IncludedResources, resourceSelector.ExcludedResources) - namespaces := collections.NewIncludesExcludes().Includes(resourceSelector.IncludedNamespaces...).Excludes(resourceSelector.ExcludedNamespaces...) - - selector := labels.Everything() - if resourceSelector.LabelSelector != "" { - if selector, err = labels.Parse(resourceSelector.LabelSelector); err != nil { - return nil, err - } - } - - res := resolvedAction{ - RestoreItemAction: action, - resourceIncludesExcludes: resources, - namespaceIncludesExcludes: namespaces, - selector: selector, - } - - resolved = append(resolved, res) - } - - return resolved, nil -} - type restoreContext struct { backup *velerov1api.Backup backupReader io.Reader @@ -331,7 +309,8 @@ type restoreContext struct { dynamicFactory client.DynamicFactory fileSystem filesystem.Interface namespaceClient corev1.NamespaceInterface - actions []resolvedAction + restoreItemActions []framework.RestoreItemResolvedAction + itemSnapshotterActions []framework.ItemSnapshotterResolvedAction volumeSnapshotterGetter VolumeSnapshotterGetter resticRestorer restic.Restorer resticWaitGroup sync.WaitGroup @@ -713,23 +692,22 @@ func getNamespace(logger logrus.FieldLogger, path, remappedName string) *v1.Name } } -// TODO: this should be combined with DeleteItemActions at some point. -func (ctx *restoreContext) getApplicableActions(groupResource schema.GroupResource, namespace string) []resolvedAction { - var actions []resolvedAction - for _, action := range ctx.actions { - if !action.resourceIncludesExcludes.ShouldInclude(groupResource.String()) { - continue +func (ctx *restoreContext) getApplicableActions(groupResource schema.GroupResource, namespace string) []framework.RestoreItemResolvedAction { + var actions []framework.RestoreItemResolvedAction + for _, action := range ctx.restoreItemActions { + if action.ShouldUse(groupResource, namespace, nil, ctx.log) { + actions = append(actions, action) } + } + return actions +} - if namespace != "" && !action.namespaceIncludesExcludes.ShouldInclude(namespace) { - continue +func (ctx *restoreContext) getApplicableItemSnapshotters(groupResource schema.GroupResource, namespace string) []framework.ItemSnapshotterResolvedAction { + var actions []framework.ItemSnapshotterResolvedAction + for _, action := range ctx.itemSnapshotterActions { + if action.ShouldUse(groupResource, namespace, nil, ctx.log) { + actions = append(actions, action) } - - if namespace == "" && !action.namespaceIncludesExcludes.IncludeEverything() { - continue - } - - actions = append(actions, action) } return actions @@ -842,12 +820,7 @@ func (ctx *restoreContext) crdAvailable(name string, crdClient client.Dynamic) ( if err != nil { return true, err } - - // TODO: Due to upstream conversion issues in runtime.FromUnstructured, - // we use the unstructured object here. Once the upstream conversion - // functions are fixed, we should convert to the CRD types and use - // IsCRDReady. - available, err = kube.IsUnstructuredCRDReady(unstructuredCRD) + available, err = kube.IsCRDReady(unstructuredCRD) if err != nil { return true, err } @@ -1107,6 +1080,12 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso default: ctx.log.Infof("Restoring persistent volume as-is because it doesn't have a snapshot and its reclaim policy is not Delete.") + // Check to see if the claimRef.namespace field needs to be remapped, and do so if necessary. + _, err = remapClaimRefNS(ctx, obj) + if err != nil { + errs.Add(namespace, err) + return warnings, errs + } obj = resetVolumeBindingInfo(obj) // We call the pvRestorer here to clear out the PV's claimRef.UID, // so it can be re-claimed when its PVC is restored and gets a new UID. @@ -1126,13 +1105,13 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso } for _, action := range ctx.getApplicableActions(groupResource, namespace) { - if !action.selector.Matches(labels.Set(obj.GetLabels())) { + if !action.Selector.Matches(labels.Set(obj.GetLabels())) { return warnings, errs } ctx.log.Infof("Executing item action for %v", &groupResource) - executeOutput, err := action.Execute(&velero.RestoreItemActionExecuteInput{ + executeOutput, err := action.RestoreItemAction.Execute(&velero.RestoreItemActionExecuteInput{ Item: obj, ItemFromBackup: itemFromBackup, Restore: ctx.restore, @@ -1237,7 +1216,12 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso ctx.log.Infof("Attempting to restore %s: %v", obj.GroupVersionKind().Kind, name) createdObj, restoreErr := resourceClient.Create(obj) - if apierrors.IsAlreadyExists(restoreErr) { + isAlreadyExistsError, err := isAlreadyExistsError(ctx, obj, restoreErr, resourceClient) + if err != nil { + errs.Add(namespace, err) + return warnings, errs + } + if isAlreadyExistsError { fromCluster, err := resourceClient.Get(name, metav1.GetOptions{}) if err != nil { ctx.log.Infof("Error retrieving cluster version of %s: %v", kube.NamespaceAndName(obj), err) @@ -1287,7 +1271,8 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso ctx.log.Infof("ServiceAccount %s successfully updated", kube.NamespaceAndName(obj)) } default: - e := errors.Errorf("could not restore, %s. Warning: the in-cluster version is different than the backed-up version.", restoreErr) + e := errors.Errorf("could not restore, %s %q already exists. Warning: the in-cluster version is different than the backed-up version.", + obj.GetKind(), obj.GetName()) warnings.Add(namespace, e) } return warnings, errs @@ -1334,6 +1319,45 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso return warnings, errs } +func isAlreadyExistsError(ctx *restoreContext, obj *unstructured.Unstructured, err error, client client.Dynamic) (bool, error) { + if err == nil { + return false, nil + } + if apierrors.IsAlreadyExists(err) { + return true, nil + } + // The "invalid value error" or "internal error" rather than "already exists" error returns when restoring nodePort service in the following two cases: + // 1. For NodePort service, the service has nodePort preservation while the same nodePort service already exists. - Get invalid value error + // 2. For LoadBalancer service, the "healthCheckNodePort" already exists. - Get internal error + // If this is the case, the function returns true to avoid reporting error. + // Refer to https://github.com/vmware-tanzu/velero/issues/2308 for more details + if obj.GetKind() != "Service" { + return false, nil + } + statusErr, ok := err.(*apierrors.StatusError) + if !ok || statusErr.Status().Details == nil || len(statusErr.Status().Details.Causes) == 0 { + return false, nil + } + // make sure all the causes are "port allocated" error + for _, cause := range statusErr.Status().Details.Causes { + if !strings.Contains(cause.Message, "provided port is already allocated") { + return false, nil + } + } + + // the "already allocated" error may caused by other services, check whether the expected service exists or not + if _, err = client.Get(obj.GetName(), metav1.GetOptions{}); err != nil { + if apierrors.IsNotFound(err) { + ctx.log.Debugf("Service %s not found", kube.NamespaceAndName(obj)) + return false, nil + } + return false, errors.Wrapf(err, "Unable to get the service %s while checking the NodePort is already allocated error", kube.NamespaceAndName(obj)) + } + + ctx.log.Infof("Service %s exists, ignore the provided port is already allocated error", kube.NamespaceAndName(obj)) + return true, nil +} + // shouldRenamePV returns a boolean indicating whether a persistent volume should // be given a new name before being restored, or an error if this cannot be determined. // A persistent volume will be given a new name if and only if (a) a PV with the @@ -1538,14 +1562,10 @@ func resetVolumeBindingInfo(obj *unstructured.Unstructured) *unstructured.Unstru // Upon restore, this new PV will look like a statically provisioned, manually- // bound volume rather than one bound by the controller, so remove the annotation // that signals that a controller bound it. - delete(annotations, KubeAnnBindCompleted) + delete(annotations, kube.KubeAnnBindCompleted) // Remove the annotation that signals that the PV is already bound; we want // the PV(C) controller to take the two objects and bind them again. - delete(annotations, KubeAnnBoundByController) - - // Remove the provisioned-by annotation which signals that the persistent - // volume was dynamically provisioned; it is now statically provisioned. - delete(annotations, KubeAnnDynamicallyProvisioned) + delete(annotations, kube.KubeAnnBoundByController) // GetAnnotations returns a copy, so we have to set them again. obj.SetAnnotations(annotations) diff --git a/pkg/restore/restore_test.go b/pkg/restore/restore_test.go index 274397862..a81676085 100644 --- a/pkg/restore/restore_test.go +++ b/pkg/restore/restore_test.go @@ -30,6 +30,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -51,6 +52,7 @@ import ( resticmocks "github.com/vmware-tanzu/velero/pkg/restic/mocks" "github.com/vmware-tanzu/velero/pkg/test" testutil "github.com/vmware-tanzu/velero/pkg/test" + "github.com/vmware-tanzu/velero/pkg/util/kube" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/volume" ) @@ -544,7 +546,7 @@ func TestRestoreResourceFiltering(t *testing.T) { } warnings, errs := h.restorer.Restore( data, - nil, // actions + nil, // restoreItemActions nil, // snapshot location lister nil, // volume snapshotter getter ) @@ -625,7 +627,7 @@ func TestRestoreNamespaceMapping(t *testing.T) { } warnings, errs := h.restorer.Restore( data, - nil, // actions + nil, // restoreItemActions nil, // snapshot location lister nil, // volume snapshotter getter ) @@ -707,7 +709,7 @@ func TestRestoreResourcePriorities(t *testing.T) { } warnings, errs := h.restorer.Restore( data, - nil, // actions + nil, // restoreItemActions nil, // snapshot location lister nil, // volume snapshotter getter ) @@ -784,7 +786,7 @@ func TestInvalidTarballContents(t *testing.T) { } warnings, errs := h.restorer.Restore( data, - nil, // actions + nil, // restoreItemActions nil, // snapshot location lister nil, // volume snapshotter getter ) @@ -999,7 +1001,7 @@ func TestRestoreItems(t *testing.T) { } warnings, errs := h.restorer.Restore( data, - nil, // actions + nil, // restoreItemActions nil, // snapshot location lister nil, // volume snapshotter getter ) @@ -1811,6 +1813,8 @@ func TestRestorePersistentVolumes(t *testing.T) { volumeSnapshotLocations []*velerov1api.VolumeSnapshotLocation volumeSnapshotterGetter volumeSnapshotterGetter want []*test.APIResource + wantError bool + wantWarning bool }{ { name: "when a PV with a reclaim policy of delete has no snapshot and does not exist in-cluster, it does not get restored, and its PVC gets reset for dynamic provisioning", @@ -2191,6 +2195,95 @@ func TestRestorePersistentVolumes(t *testing.T) { ), }, }, + { + name: "when a PV without a snapshot is used by a PVC in a namespace that's being remapped, and the original PV exists in-cluster, the PV is not replaced and there is a restore warning", + restore: defaultRestore().NamespaceMappings("source-ns", "target-ns").Result(), + backup: defaultBackup().Result(), + tarball: test.NewTarWriter(t). + AddItems( + "persistentvolumes", + builder.ForPersistentVolume("source-pv"). + //ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain). + AWSEBSVolumeID("source-volume"). + ClaimRef("source-ns", "pvc-1"). + Result(), + ). + AddItems( + "persistentvolumeclaims", + builder.ForPersistentVolumeClaim("source-ns", "pvc-1").VolumeName("source-pv").Result(), + ). + Done(), + apiResources: []*test.APIResource{ + test.PVs( + builder.ForPersistentVolume("source-pv"). + //ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain). + AWSEBSVolumeID("source-volume"). + ClaimRef("source-ns", "pvc-1"). + Result(), + ), + test.PVCs(), + }, + want: []*test.APIResource{ + test.PVs( + builder.ForPersistentVolume("source-pv"). + AWSEBSVolumeID("source-volume"). + ClaimRef("source-ns", "pvc-1"). + Result(), + ), + test.PVCs( + builder.ForPersistentVolumeClaim("target-ns", "pvc-1"). + ObjectMeta( + builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1"), + ). + VolumeName("source-pv"). + Result(), + ), + }, + wantWarning: true, + }, + { + name: "when a PV without a snapshot is used by a PVC in a namespace that's being remapped, and the original PV does not exist in-cluster, the PV is not renamed", + restore: defaultRestore().NamespaceMappings("source-ns", "target-ns").Result(), + backup: defaultBackup().Result(), + tarball: test.NewTarWriter(t). + AddItems( + "persistentvolumes", + builder.ForPersistentVolume("source-pv"). + AWSEBSVolumeID("source-volume"). + ClaimRef("source-ns", "pvc-1"). + Result(), + ). + AddItems( + "persistentvolumeclaims", + builder.ForPersistentVolumeClaim("source-ns", "pvc-1").VolumeName("source-pv").Result(), + ). + Done(), + apiResources: []*test.APIResource{ + test.PVs(), + test.PVCs(), + }, + want: []*test.APIResource{ + test.PVs( + builder.ForPersistentVolume("source-pv"). + //ReclaimPolicy(corev1api.PersistentVolumeReclaimRetain). + ObjectMeta( + builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1"), + ). + // the namespace for this PV's claimRef should be the one that the PVC was remapped into. + ClaimRef("target-ns", "pvc-1"). + AWSEBSVolumeID("source-volume"). + Result(), + ), + test.PVCs( + builder.ForPersistentVolumeClaim("target-ns", "pvc-1"). + ObjectMeta( + builder.WithLabels("velero.io/backup-name", "backup-1", "velero.io/restore-name", "restore-1"), + ). + VolumeName("source-pv"). + Result(), + ), + }, + }, { name: "when a PV is renamed and the original PV does not exist in-cluster, the PV should be renamed", restore: defaultRestore().NamespaceMappings("source-ns", "target-ns").Result(), @@ -2418,12 +2511,21 @@ func TestRestorePersistentVolumes(t *testing.T) { } warnings, errs := h.restorer.Restore( data, - nil, // actions + nil, // restoreItemActions vslInformer.Lister(), tc.volumeSnapshotterGetter, ) - assertEmptyResults(t, warnings, errs) + if tc.wantWarning { + assertNonEmptyResults(t, "warning", warnings) + } else { + assertEmptyResults(t, warnings) + } + if tc.wantError { + assertNonEmptyResults(t, "error", errs) + } else { + assertEmptyResults(t, errs) + } assertAPIContents(t, h, wantIDs) assertRestoredItems(t, h, tc.want) }) @@ -2545,7 +2647,7 @@ func TestRestoreWithRestic(t *testing.T) { warnings, errs := h.restorer.Restore( data, - nil, // actions + nil, // restoreItemActions nil, // snapshot location lister nil, // volume snapshotter getter ) @@ -2804,6 +2906,17 @@ func assertEmptyResults(t *testing.T, res ...Result) { } } +func assertNonEmptyResults(t *testing.T, typeMsg string, res ...Result) { + t.Helper() + total := 0 + for _, r := range res { + total += len(r.Cluster) + total += len(r.Namespaces) + total += len(r.Velero) + } + assert.Greater(t, total, 0, "Expected at least one "+typeMsg) +} + type harness struct { *test.APIServer @@ -2875,9 +2988,9 @@ func Test_resetVolumeBindingInfo(t *testing.T) { name: "PVs that are bound have their binding and dynamic provisioning annotations removed", obj: NewTestUnstructured().WithMetadataField("kind", "persistentVolume"). WithName("pv-1").WithAnnotations( - KubeAnnBindCompleted, - KubeAnnBoundByController, - KubeAnnDynamicallyProvisioned, + kube.KubeAnnBindCompleted, + kube.KubeAnnBoundByController, + kube.KubeAnnDynamicallyProvisioned, ).WithSpecField("claimRef", map[string]interface{}{ "namespace": "ns-1", "name": "pvc-1", @@ -2885,7 +2998,7 @@ func Test_resetVolumeBindingInfo(t *testing.T) { "resourceVersion": "1"}).Unstructured, expected: NewTestUnstructured().WithMetadataField("kind", "persistentVolume"). WithName("pv-1"). - WithAnnotations(). + WithAnnotations(kube.KubeAnnDynamicallyProvisioned). WithSpecField("claimRef", map[string]interface{}{ "namespace": "ns-1", "name": "pvc-1"}).Unstructured, }, @@ -2893,9 +3006,8 @@ func Test_resetVolumeBindingInfo(t *testing.T) { name: "PVCs that are bound have their binding annotations removed, but the volume name stays", obj: NewTestUnstructured().WithMetadataField("kind", "persistentVolumeClaim"). WithName("pvc-1").WithAnnotations( - KubeAnnBindCompleted, - KubeAnnBoundByController, - KubeAnnDynamicallyProvisioned, + kube.KubeAnnBindCompleted, + kube.KubeAnnBoundByController, ).WithSpecField("volumeName", "pv-1").Unstructured, expected: NewTestUnstructured().WithMetadataField("kind", "persistentVolumeClaim"). WithName("pvc-1").WithAnnotations(). @@ -2910,3 +3022,138 @@ func Test_resetVolumeBindingInfo(t *testing.T) { }) } } + +func TestIsAlreadyExistsError(t *testing.T) { + tests := []struct { + name string + apiResource *test.APIResource + obj *unstructured.Unstructured + err error + expected bool + }{ + { + name: "The input error is IsAlreadyExists error", + err: apierrors.NewAlreadyExists(schema.GroupResource{}, ""), + expected: true, + }, + { + name: "The input obj isn't service", + obj: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "kind": "Pod", + }, + }, + expected: false, + }, + { + name: "The StatusError contains no causes", + obj: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "kind": "Service", + }, + }, + err: &apierrors.StatusError{ + ErrStatus: metav1.Status{ + Reason: metav1.StatusReasonInvalid, + }, + }, + expected: false, + }, + { + name: "The causes contains not only port already allocated error", + obj: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "kind": "Service", + }, + }, + err: &apierrors.StatusError{ + ErrStatus: metav1.Status{ + Reason: metav1.StatusReasonInvalid, + Details: &metav1.StatusDetails{ + Causes: []metav1.StatusCause{ + {Message: "provided port is already allocated"}, + {Message: "other error"}, + }, + }, + }, + }, + expected: false, + }, + { + name: "Get already allocated error but the service doesn't exist", + obj: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "kind": "Service", + "metadata": map[string]interface{}{ + "namespace": "default", + "name": "test", + }, + }, + }, + err: &apierrors.StatusError{ + ErrStatus: metav1.Status{ + Reason: metav1.StatusReasonInvalid, + Details: &metav1.StatusDetails{ + Causes: []metav1.StatusCause{ + {Message: "provided port is already allocated"}, + }, + }, + }, + }, + expected: false, + }, + { + name: "Get already allocated error and the service exists", + apiResource: test.Services( + builder.ForService("default", "test").Result(), + ), + obj: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "kind": "Service", + "metadata": map[string]interface{}{ + "namespace": "default", + "name": "test", + }, + }, + }, + err: &apierrors.StatusError{ + ErrStatus: metav1.Status{ + Reason: metav1.StatusReasonInvalid, + Details: &metav1.StatusDetails{ + Causes: []metav1.StatusCause{ + {Message: "provided port is already allocated"}, + }, + }, + }, + }, + expected: true, + }, + } + for _, test := range tests { + h := newHarness(t) + + ctx := &restoreContext{ + log: h.log, + dynamicFactory: client.NewDynamicFactory(h.DynamicClient), + namespaceClient: h.KubeClient.CoreV1().Namespaces(), + } + + if test.apiResource != nil { + h.AddItems(t, test.apiResource) + } + + client, err := ctx.dynamicFactory.ClientForGroupVersionResource( + schema.GroupVersion{Group: "", Version: "v1"}, + metav1.APIResource{Name: "services"}, + "default", + ) + require.NoError(t, err) + + t.Run(test.name, func(t *testing.T) { + result, err := isAlreadyExistsError(ctx, test.obj, test.err, client) + require.NoError(t, err) + + assert.Equal(t, test.expected, result) + }) + } +} diff --git a/pkg/restore/service_action.go b/pkg/restore/service_action.go index e8b58f062..0b22cf095 100644 --- a/pkg/restore/service_action.go +++ b/pkg/restore/service_action.go @@ -55,6 +55,7 @@ func (a *ServiceAction) Execute(input *velero.RestoreItemActionExecuteInput) (*v if service.Spec.ClusterIP != "None" { service.Spec.ClusterIP = "" + service.Spec.ClusterIPs = nil } /* Do not delete NodePorts if restore triggered with "--preserve-nodeports" flag */ @@ -83,6 +84,7 @@ func deleteNodePorts(service *corev1api.Service) error { // to the last-applied-config annotation. We'll retain these values, and // clear out any other (presumably auto-assigned) NodePort values. explicitNodePorts := sets.NewString() + unnamedPortInts := sets.NewInt() lastAppliedConfig, ok := service.Annotations[annotationLastAppliedConfig] if ok { appliedServiceUnstructured := new(map[string]interface{}) @@ -123,7 +125,7 @@ func deleteNodePorts(service *corev1api.Service) error { portName, ok := p["name"] if !ok { // unnamed port - explicitNodePorts.Insert("") + unnamedPortInts.Insert(nodePortInt) } else { explicitNodePorts.Insert(portName.(string)) } @@ -135,8 +137,14 @@ func deleteNodePorts(service *corev1api.Service) error { } for i, port := range service.Spec.Ports { - if !explicitNodePorts.Has(port.Name) { - service.Spec.Ports[i].NodePort = 0 + if port.Name != "" { + if !explicitNodePorts.Has(port.Name) { + service.Spec.Ports[i].NodePort = 0 + } + } else { + if !unnamedPortInts.Has(int(port.NodePort)) { + service.Spec.Ports[i].NodePort = 0 + } } } diff --git a/pkg/restore/service_action_test.go b/pkg/restore/service_action_test.go index e6b0b1530..59fef00a1 100644 --- a/pkg/restore/service_action_test.go +++ b/pkg/restore/service_action_test.go @@ -73,13 +73,14 @@ func TestServiceActionExecute(t *testing.T) { expectedRes corev1api.Service }{ { - name: "clusterIP (only) should be deleted from spec", + name: "clusterIP/clusterIPs should be deleted from spec", obj: corev1api.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "svc-1", }, Spec: corev1api.ServiceSpec{ ClusterIP: "should-be-removed", + ClusterIPs: []string{"should-be-removed"}, LoadBalancerIP: "should-be-kept", }, }, @@ -196,6 +197,9 @@ func TestServiceActionExecute(t *testing.T) { { NodePort: 8080, }, + { + NodePort: 9090, + }, }, }, }, @@ -212,6 +216,7 @@ func TestServiceActionExecute(t *testing.T) { { NodePort: 8080, }, + {}, }, }, }, diff --git a/pkg/test/api_server.go b/pkg/test/api_server.go index a18fbd2d0..06439bca1 100644 --- a/pkg/test/api_server.go +++ b/pkg/test/api_server.go @@ -57,6 +57,7 @@ func NewAPIServer(t *testing.T) *APIServer { {Group: "apiextensions.k8s.io", Version: "v1beta1", Resource: "customresourcedefinitions"}: "CRDList", {Group: "velero.io", Version: "v1", Resource: "volumesnapshotlocations"}: "VSLList", {Group: "extensions", Version: "v1", Resource: "deployments"}: "ExtDeploymentsList", + {Group: "velero.io", Version: "v1", Resource: "deployments"}: "VeleroDeploymentsList", }) discoveryClient = &DiscoveryClient{FakeDiscovery: kubeClient.Discovery().(*discoveryfake.FakeDiscovery)} ) diff --git a/pkg/test/fake_mapper.go b/pkg/test/fake_mapper.go index 1b430c5aa..210d25747 100644 --- a/pkg/test/fake_mapper.go +++ b/pkg/test/fake_mapper.go @@ -43,6 +43,16 @@ func (m *FakeMapper) ResourceFor(input schema.GroupVersionResource) (schema.Grou if gr, found := m.Resources[input]; found { return gr, nil } + if input.Version == "" { + input.Version = "v1" + if gr, found := m.Resources[input]; found { + return gr, nil + } + input.Version = "v1beta1" + if gr, found := m.Resources[input]; found { + return gr, nil + } + } return schema.GroupVersionResource{}, errors.Errorf("invalid resource %q", input.String()) } diff --git a/pkg/test/fake_namespace.go b/pkg/test/fake_namespace.go index 686e2a43a..0565151b4 100644 --- a/pkg/test/fake_namespace.go +++ b/pkg/test/fake_namespace.go @@ -24,6 +24,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/watch" + v1 "k8s.io/client-go/applyconfigurations/core/v1" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" ) @@ -77,3 +78,13 @@ func (c *FakeNamespaceClient) UpdateStatus(ctx context.Context, namespace *corev args := c.Called(namespace) return args.Get(0).(*corev1api.Namespace), args.Error(1) } + +func (c *FakeNamespaceClient) Apply(ctx context.Context, namespace *v1.NamespaceApplyConfiguration, opts metav1.ApplyOptions) (result *corev1api.Namespace, err error) { + args := c.Called(namespace) + return args.Get(0).(*corev1api.Namespace), args.Error(1) +} + +func (c *FakeNamespaceClient) ApplyStatus(ctx context.Context, namespace *v1.NamespaceApplyConfiguration, opts metav1.ApplyOptions) (result *corev1api.Namespace, err error) { + args := c.Called(namespace) + return args.Get(0).(*corev1api.Namespace), args.Error(1) +} diff --git a/pkg/test/resources.go b/pkg/test/resources.go index e83f171d4..9e09fccdf 100644 --- a/pkg/test/resources.go +++ b/pkg/test/resources.go @@ -108,6 +108,18 @@ func ExtensionsDeployments(items ...metav1.Object) *APIResource { } } +// test CRD +func VeleroDeployments(items ...metav1.Object) *APIResource { + return &APIResource{ + Group: "velero.io", + Version: "v1", + Name: "deployments", + ShortName: "deploy", + Namespaced: true, + Items: items, + } +} + func Namespaces(items ...metav1.Object) *APIResource { return &APIResource{ Group: "", @@ -150,3 +162,14 @@ func VSLs(items ...metav1.Object) *APIResource { Items: items, } } + +func Services(items ...metav1.Object) *APIResource { + return &APIResource{ + Group: "", + Version: "v1", + Name: "services", + ShortName: "svc", + Namespaced: true, + Items: items, + } +} diff --git a/pkg/util/collections/includes_excludes.go b/pkg/util/collections/includes_excludes.go index c113f0d95..231689be0 100644 --- a/pkg/util/collections/includes_excludes.go +++ b/pkg/util/collections/includes_excludes.go @@ -1,5 +1,5 @@ /* -Copyright 2020 the Velero contributors. +Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import ( "github.com/gobwas/glob" "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/api/validation" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" @@ -154,6 +155,51 @@ func ValidateIncludesExcludes(includesList, excludesList []string) []error { return errs } +// ValidateNamespaceIncludesExcludes checks provided lists of included and +// excluded namespaces to ensure they are a valid set of IncludesExcludes data. +func ValidateNamespaceIncludesExcludes(includesList, excludesList []string) []error { + errs := ValidateIncludesExcludes(includesList, excludesList) + + includes := sets.NewString(includesList...) + excludes := sets.NewString(excludesList...) + + for _, itm := range includes.List() { + if nsErrs := validateNamespaceName(itm); nsErrs != nil { + errs = append(errs, nsErrs...) + } + } + for _, itm := range excludes.List() { + if nsErrs := validateNamespaceName(itm); nsErrs != nil { + errs = append(errs, nsErrs...) + } + } + + return errs +} + +func validateNamespaceName(ns string) []error { + var errs []error + + // Velero interprets empty string as "no namespace", so allow it even though + // it is not a valid Kubernetes name. + if ns == "" { + return nil + } + + // Kubernetes does not allow asterisks in namespaces but Velero uses them as + // wildcards. Replace asterisks with an arbitrary letter to pass Kubernetes + // validation. + tmpNamespace := strings.ReplaceAll(ns, "*", "x") + + if errMsgs := validation.ValidateNamespaceName(tmpNamespace, false); errMsgs != nil { + for _, msg := range errMsgs { + errs = append(errs, errors.Errorf("invalid namespace %q: %s", ns, msg)) + } + } + + return errs +} + // GenerateIncludesExcludes constructs an IncludesExcludes struct by taking the provided // include/exclude slices, applying the specified mapping function to each item in them, // and adding the output of the function to the new struct. If the mapping function returns diff --git a/pkg/util/collections/includes_excludes_test.go b/pkg/util/collections/includes_excludes_test.go index 9f08a7012..672a309d2 100644 --- a/pkg/util/collections/includes_excludes_test.go +++ b/pkg/util/collections/includes_excludes_test.go @@ -1,5 +1,5 @@ /* -Copyright 2017 the Velero contributors. +Copyright The Velero Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -29,86 +29,87 @@ func TestShouldInclude(t *testing.T) { name string includes []string excludes []string - check string - should bool + item string + want bool }{ { - name: "empty - include everything", - check: "foo", - should: true, + name: "empty string should include every item", + item: "foo", + want: true, }, { - name: "include *", + name: "include * should include every item", includes: []string{"*"}, - check: "foo", - should: true, + item: "foo", + want: true, }, { - name: "include specific - found", + name: "item in includes list should include item", includes: []string{"foo", "bar", "baz"}, - check: "foo", - should: true, + item: "foo", + want: true, }, { - name: "include specific - not found", + name: "item not in includes list should not include item", includes: []string{"foo", "baz"}, - check: "bar", - should: false, + item: "bar", + want: false, }, { - name: "include *, exclude foo", + name: "include *, excluded item should not include item", includes: []string{"*"}, excludes: []string{"foo"}, - check: "foo", - should: false, + item: "foo", + want: false, }, { - name: "include *, exclude foo, check bar", + name: "include *, exclude foo, bar should be included", includes: []string{"*"}, excludes: []string{"foo"}, - check: "bar", - should: true, + item: "bar", + want: true, }, { - name: "both include and exclude foo", + name: "an item both included and excluded should not be included", includes: []string{"foo"}, excludes: []string{"foo"}, - check: "foo", - should: false, + item: "foo", + want: false, }, { - name: "wildcard include", + name: "wildcard should include item", includes: []string{"*.bar"}, - check: "foo.bar", - should: true, + item: "foo.bar", + want: true, }, { - name: "wildcard include fail", + name: "wildcard mismatch should not include item", includes: []string{"*.bar"}, - check: "bar.foo", - should: false, + item: "bar.foo", + want: false, }, { - name: "wildcard exclude", + name: "wildcard exclude should not include item", includes: []string{"*"}, excludes: []string{"*.bar"}, - check: "foo.bar", - should: false, + item: "foo.bar", + want: false, }, { - name: "wildcard exclude fail", + name: "wildcard mismatch should include item", includes: []string{"*"}, excludes: []string{"*.bar"}, - check: "bar.foo", - should: true, + item: "bar.foo", + want: true, }, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - i := NewIncludesExcludes().Includes(test.includes...).Excludes(test.excludes...) - if e, a := test.should, i.ShouldInclude(test.check); e != a { - t.Errorf("expected %t, got %t", e, a) + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + includesExcludes := NewIncludesExcludes().Includes(tc.includes...).Excludes(tc.excludes...) + + if got := includesExcludes.ShouldInclude((tc.item)); got != tc.want { + t.Errorf("want %t, got %t", tc.want, got) } }) } @@ -119,7 +120,7 @@ func TestValidateIncludesExcludes(t *testing.T) { name string includes []string excludes []string - expected []error + want []error }{ { name: "empty includes (everything) is allowed", @@ -132,30 +133,30 @@ func TestValidateIncludesExcludes(t *testing.T) { { name: "include everything not allowed with other includes", includes: []string{"*", "foo"}, - expected: []error{errors.New("includes list must either contain '*' only, or a non-empty list of items")}, + want: []error{errors.New("includes list must either contain '*' only, or a non-empty list of items")}, }, { name: "exclude everything not allowed", includes: []string{"foo"}, excludes: []string{"*"}, - expected: []error{errors.New("excludes list cannot contain '*'")}, + want: []error{errors.New("excludes list cannot contain '*'")}, }, { name: "excludes cannot contain items in includes", includes: []string{"foo", "bar"}, excludes: []string{"bar"}, - expected: []error{errors.New("excludes list cannot contain an item in the includes list: bar")}, + want: []error{errors.New("excludes list cannot contain an item in the includes list: bar")}, }, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - res := ValidateIncludesExcludes(test.includes, test.excludes) + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + errs := ValidateIncludesExcludes(tc.includes, tc.excludes) - require.Equal(t, len(test.expected), len(res)) + require.Equal(t, len(tc.want), len(errs)) - for i := 0; i < len(test.expected); i++ { - assert.Equal(t, test.expected[i].Error(), res[i].Error()) + for i := 0; i < len(tc.want); i++ { + assert.Equal(t, tc.want[i].Error(), errs[i].Error()) } }) } @@ -163,33 +164,134 @@ func TestValidateIncludesExcludes(t *testing.T) { func TestIncludeExcludeString(t *testing.T) { tests := []struct { - name string - includes []string - excludes []string - expectedIncludes string - expectedExcludes string + name string + includes []string + excludes []string + wantIncludes string + wantExcludes string }{ { - name: "unspecified includes/excludes should return '*'/''", - includes: nil, - excludes: nil, - expectedIncludes: "*", - expectedExcludes: "", + name: "unspecified includes/excludes should return '*'/''", + includes: nil, + excludes: nil, + wantIncludes: "*", + wantExcludes: "", }, { - name: "specific resources should result in sorted joined string", - includes: []string{"foo", "bar"}, - excludes: []string{"baz", "xyz"}, - expectedIncludes: "bar, foo", - expectedExcludes: "baz, xyz", + name: "specific resources should result in sorted joined string", + includes: []string{"foo", "bar"}, + excludes: []string{"baz", "xyz"}, + wantIncludes: "bar, foo", + wantExcludes: "baz, xyz", }, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - ie := NewIncludesExcludes().Includes(test.includes...).Excludes(test.excludes...) - assert.Equal(t, test.expectedIncludes, ie.IncludesString()) - assert.Equal(t, test.expectedExcludes, ie.ExcludesString()) + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + includesExcludes := NewIncludesExcludes().Includes(tc.includes...).Excludes(tc.excludes...) + assert.Equal(t, tc.wantIncludes, includesExcludes.IncludesString()) + assert.Equal(t, tc.wantExcludes, includesExcludes.ExcludesString()) + }) + } +} + +func TestValidateNamespaceIncludesExcludes(t *testing.T) { + tests := []struct { + name string + includes []string + excludes []string + wantErr bool + }{ + { + name: "empty slice doesn't return error", + includes: []string{}, + wantErr: false, + }, + { + name: "asterisk by itself is valid", + includes: []string{"*"}, + wantErr: false, + }, + { + name: "alphanumeric names with optional dash inside are valid", + includes: []string{"foobar", "bar-321", "foo123bar"}, + excludes: []string{"123bar", "barfoo", "foo-321", "bar123foo"}, + wantErr: false, + }, + { + name: "not starting or ending with an alphanumeric character is invalid", + includes: []string{"-123foo"}, + excludes: []string{"foo321-", "foo321-"}, + wantErr: true, + }, + { + name: "special characters in name is invalid", + includes: []string{"foo?", "foo.bar", "bar_321"}, + excludes: []string{"$foo", "foo>bar", "bar=321"}, + wantErr: true, + }, + { + name: "empty includes (everything) is valid", + includes: []string{}, + wantErr: false, + }, + { + name: "empty string includes is valid (includes nothing)", + includes: []string{""}, + wantErr: false, + }, + { + name: "empty string excludes is valid (excludes nothing)", + excludes: []string{""}, + wantErr: false, + }, + { + name: "include everything using asterisk is valid", + includes: []string{"*"}, + wantErr: false, + }, + { + name: "excludes can contain wildcard", + includes: []string{"foo", "bar"}, + excludes: []string{"nginx-ingress-*", "*-bar", "*-ingress-*"}, + wantErr: false, + }, + { + name: "includes can contain wildcard", + includes: []string{"*-foo", "kube-*", "*kube*"}, + excludes: []string{"bar"}, + wantErr: false, + }, + { + name: "include everything not allowed with other includes", + includes: []string{"*", "foo"}, + wantErr: true, + }, + { + name: "exclude everything not allowed", + includes: []string{"foo"}, + excludes: []string{"*"}, + wantErr: true, + }, + { + name: "excludes cannot contain items in includes", + includes: []string{"foo", "bar"}, + excludes: []string{"bar"}, + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + errs := ValidateNamespaceIncludesExcludes(tc.includes, tc.excludes) + + if tc.wantErr && len(errs) == 0 { + t.Errorf("%s: wanted errors but got none", tc.name) + } + + if !tc.wantErr && len(errs) != 0 { + t.Errorf("%s: wanted no errors but got: %v", tc.name, errs) + } }) } } diff --git a/pkg/util/kube/utils.go b/pkg/util/kube/utils.go index a80f864de..ccda34ad3 100644 --- a/pkg/util/kube/utils.go +++ b/pkg/util/kube/utils.go @@ -21,18 +21,31 @@ import ( "fmt" "time" + "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" + storagev1api "k8s.io/api/storage/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" corev1listers "k8s.io/client-go/listers/core/v1" ) +// These annotations are taken from the Kubernetes persistent volume/persistent volume claim controller. +// They cannot be directly importing because they are part of the kubernetes/kubernetes package, and importing that package is unsupported. +// Their values are well-known and slow changing. They're duplicated here as constants to provide compile-time checking. +// Originals can be found in kubernetes/kubernetes/pkg/controller/volume/persistentvolume/util/util.go. +const KubeAnnBindCompleted = "pv.kubernetes.io/bind-completed" +const KubeAnnBoundByController = "pv.kubernetes.io/bound-by-controller" +const KubeAnnDynamicallyProvisioned = "pv.kubernetes.io/provisioned-by" + // NamespaceAndName returns a string in the format / func NamespaceAndName(objMeta metav1.Object) string { if objMeta.GetNamespace() == "" { @@ -104,7 +117,8 @@ func EnsureNamespaceExistsAndIsReady(namespace *corev1api.Namespace, client core // GetVolumeDirectory gets the name of the directory on the host, under /var/lib/kubelet/pods//volumes/, // where the specified volume lives. // For volumes with a CSIVolumeSource, append "/mount" to the directory name. -func GetVolumeDirectory(pod *corev1api.Pod, volumeName string, pvcLister corev1listers.PersistentVolumeClaimLister, pvLister corev1listers.PersistentVolumeLister) (string, error) { +func GetVolumeDirectory(log logrus.FieldLogger, pod *corev1api.Pod, volumeName string, pvcLister corev1listers.PersistentVolumeClaimLister, + pvLister corev1listers.PersistentVolumeLister, client client.Client) (string, error) { var volume *corev1api.Volume for _, item := range pod.Spec.Volumes { @@ -139,13 +153,41 @@ func GetVolumeDirectory(pod *corev1api.Pod, volumeName string, pvcLister corev1l } // PV's been created with a CSI source. - if pv.Spec.CSI != nil { + isProvisionedByCSI, err := isProvisionedByCSI(log, pv, client) + if err != nil { + return "", errors.WithStack(err) + } + if isProvisionedByCSI { return pvc.Spec.VolumeName + "/mount", nil } return pvc.Spec.VolumeName, nil } +func isProvisionedByCSI(log logrus.FieldLogger, pv *corev1api.PersistentVolume, kbClient client.Client) (bool, error) { + if pv.Spec.CSI != nil { + return true, nil + } + // Although the pv.Spec.CSI is nil, the volume could be provisioned by a CSI driver when enabling the CSI migration + // Refer to https://github.com/vmware-tanzu/velero/issues/4496 for more details + if pv.Annotations != nil { + driverName := pv.Annotations[KubeAnnDynamicallyProvisioned] + if len(driverName) > 0 { + list := &storagev1api.CSIDriverList{} + if err := kbClient.List(context.TODO(), list); err != nil { + return false, err + } + for _, driver := range list.Items { + if driverName == driver.Name { + log.Debugf("the annotation %s=%s indicates the volume is provisioned by a CSI driver", KubeAnnDynamicallyProvisioned, driverName) + return true, nil + } + } + } + } + return false, nil +} + // IsV1CRDReady checks a v1 CRD to see if it's ready, with both the Established and NamesAccepted conditions. func IsV1CRDReady(crd *apiextv1.CustomResourceDefinition) bool { var isEstablished, namesAccepted bool @@ -176,61 +218,25 @@ func IsV1Beta1CRDReady(crd *apiextv1beta1.CustomResourceDefinition) bool { return (isEstablished && namesAccepted) } -// IsUnstructuredCRDReady checks an unstructured CRD to see if it's ready, with both the Established and NamesAccepted conditions. -// TODO: Delete this function and use IsV1CRDReady/IsV1Beta1CRDReady when the upstream runtime.FromUnstructured function properly handles int64 field conversions. -// Duplicated function because the velero install package uses IsV1CRDReady/IsV1Beta1CRDReady with instances of v1/v1beta1 types. -// See https://github.com/kubernetes/kubernetes/issues/87675 -// This is different from the fix for https://github.com/vmware-tanzu/velero/issues/2319 because here, -// we need to account for *both* v1beta1 and v1 CRDs, so doing marshalling into JSON to convert to a Go type may not be as useful here, unless we do -// type switching. -func IsUnstructuredCRDReady(crd *unstructured.Unstructured) (bool, error) { - var isEstablished, namesAccepted bool - - conditions, ok, err := unstructured.NestedSlice(crd.UnstructuredContent(), "status", "conditions") - if !ok { - return false, nil - } - if err != nil { - return false, errors.Wrap(err, "unable to access CRD's conditions") - } - - for _, c := range conditions { - // Unlike the typed version of this function, we need to cast the Condition since it's an interface{} here, - // then we fetch the type and status of the Condition before inspecting them for relevant values - cond, ok := c.(map[string]interface{}) - if !ok { - return false, errors.New("unable to convert condition to map[string]interface{}") - } - conditionType, ok, err := unstructured.NestedString(cond, "type") - if !ok { - // This should never happen unless someone manually edits the serialized data. - return false, errors.New("condition missing a type") - } - +// IsCRDReady triggers IsV1Beta1CRDReady/IsV1CRDReady according to the version of the input param +func IsCRDReady(crd *unstructured.Unstructured) (bool, error) { + ver := crd.GroupVersionKind().Version + switch ver { + case "v1beta1": + v1beta1crd := &apiextv1beta1.CustomResourceDefinition{} + err := runtime.DefaultUnstructuredConverter.FromUnstructured(crd.Object, v1beta1crd) if err != nil { - return false, errors.Wrap(err, "unable to access condition's type") + return false, err } - - status, ok, err := unstructured.NestedString(cond, "status") - if !ok { - // This should never happen unless someone manually edits the serialized data. - return false, errors.New("condition missing a status") - } - + return IsV1Beta1CRDReady(v1beta1crd), nil + case "v1": + v1crd := &apiextv1.CustomResourceDefinition{} + err := runtime.DefaultUnstructuredConverter.FromUnstructured(crd.Object, v1crd) if err != nil { - return false, errors.Wrap(err, "unable to access condition's status") - } - - // Here is the actual logic of the function - // Cast the API's types into strings since we're pulling strings out of the unstructured data. - // We are using the v1beta1 constants here but they are the same as the v1 constants. - if conditionType == string(apiextv1beta1.Established) && status == string(apiextv1beta1.ConditionTrue) { - isEstablished = true - } - if conditionType == string(apiextv1beta1.NamesAccepted) && status == string(apiextv1beta1.ConditionTrue) { - namesAccepted = true + return false, err } + return IsV1CRDReady(v1crd), nil + default: + return false, fmt.Errorf("unable to handle CRD with version %s", ver) } - - return (isEstablished && namesAccepted), nil } diff --git a/pkg/util/kube/utils_test.go b/pkg/util/kube/utils_test.go index d5966256d..eea77b110 100644 --- a/pkg/util/kube/utils_test.go +++ b/pkg/util/kube/utils_test.go @@ -17,6 +17,7 @@ limitations under the License. package kube import ( + "encoding/json" "testing" "time" @@ -24,6 +25,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + storagev1api "k8s.io/api/storage/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -32,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" kubeinformers "k8s.io/client-go/informers" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/test" @@ -187,8 +190,19 @@ func TestGetVolumeDirectorySuccess(t *testing.T) { pod: builder.ForPod("ns-1", "my-pod").Volumes(builder.ForVolume("my-vol").Result()).Result(), want: "my-vol", }, + { + name: "Volume with CSI annotation appends '/mount' to the volume name", + pod: builder.ForPod("ns-1", "my-pod").Volumes(builder.ForVolume("my-vol").PersistentVolumeClaimSource("my-pvc").Result()).Result(), + pvc: builder.ForPersistentVolumeClaim("ns-1", "my-pvc").VolumeName("a-pv").Result(), + pv: builder.ForPersistentVolume("a-pv").ObjectMeta(builder.WithAnnotations(KubeAnnDynamicallyProvisioned, "csi.test.com")).Result(), + want: "a-pv/mount", + }, } + csiDriver := storagev1api.CSIDriver{ + ObjectMeta: metav1.ObjectMeta{Name: "csi.test.com"}, + } + kbClient := fake.NewClientBuilder().WithLists(&storagev1api.CSIDriverList{Items: []storagev1api.CSIDriver{csiDriver}}).Build() for _, tc := range tests { h := newHarness(t) @@ -203,7 +217,7 @@ func TestGetVolumeDirectorySuccess(t *testing.T) { } // Function under test - dir, err := GetVolumeDirectory(tc.pod, tc.pod.Spec.Volumes[0].Name, pvcInformer.Lister(), pvInformer.Lister()) + dir, err := GetVolumeDirectory(logrus.StandardLogger(), tc.pod, tc.pod.Spec.Volumes[0].Name, pvcInformer.Lister(), pvInformer.Lister(), kbClient) require.NoError(t, err) assert.Equal(t, tc.want, dir) @@ -288,31 +302,31 @@ func TestIsV1CRDReady(t *testing.T) { } } -func TestIsUnstructuredCRDReady(t *testing.T) { - tests := []struct { +func TestIsCRDReady(t *testing.T) { + v1beta1tests := []struct { name string crd *apiextv1beta1.CustomResourceDefinition want bool }{ { - name: "CRD is not established & not accepting names - not ready", + name: "v1beta1CRD is not established & not accepting names - not ready", crd: builder.ForCustomResourceDefinitionV1Beta1("MyCRD").Result(), want: false, }, { - name: "CRD is established & not accepting names - not ready", + name: "v1beta1CRD is established & not accepting names - not ready", crd: builder.ForCustomResourceDefinitionV1Beta1("MyCRD"). Condition(builder.ForCustomResourceDefinitionV1Beta1Condition().Type(apiextv1beta1.Established).Status(apiextv1beta1.ConditionTrue).Result()).Result(), want: false, }, { - name: "CRD is not established & accepting names - not ready", + name: "v1beta1CRD is not established & accepting names - not ready", crd: builder.ForCustomResourceDefinitionV1Beta1("MyCRD"). Condition(builder.ForCustomResourceDefinitionV1Beta1Condition().Type(apiextv1beta1.NamesAccepted).Status(apiextv1beta1.ConditionTrue).Result()).Result(), want: false, }, { - name: "CRD is established & accepting names - ready", + name: "v1beta1CRD is established & accepting names - ready", crd: builder.ForCustomResourceDefinitionV1Beta1("MyCRD"). Condition(builder.ForCustomResourceDefinitionV1Beta1Condition().Type(apiextv1beta1.Established).Status(apiextv1beta1.ConditionTrue).Result()). Condition(builder.ForCustomResourceDefinitionV1Beta1Condition().Type(apiextv1beta1.NamesAccepted).Status(apiextv1beta1.ConditionTrue).Result()). @@ -321,66 +335,97 @@ func TestIsUnstructuredCRDReady(t *testing.T) { }, } - for _, tc := range tests { + for _, tc := range v1beta1tests { m, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.crd) require.NoError(t, err) - result, err := IsUnstructuredCRDReady(&unstructured.Unstructured{Object: m}) + result, err := IsCRDReady(&unstructured.Unstructured{Object: m}) require.NoError(t, err) assert.Equal(t, tc.want, result) } -} -// TestFromUnstructuredIntToFloatBug tests for a bug where runtime.DefaultUnstructuredConverter.FromUnstructured can't take a whole number into a float. -// This test should fail when https://github.com/kubernetes/kubernetes/issues/87675 is fixed upstream, letting us know we can remove the IsUnstructuredCRDReady function. -/* -func TestFromUnstructuredIntToFloatBug(t *testing.T) { - b := []byte(` + v1tests := []struct { + name string + crd *apiextv1.CustomResourceDefinition + want bool + }{ + { + name: "v1CRD is not established & not accepting names - not ready", + crd: builder.ForV1CustomResourceDefinition("MyCRD").Result(), + want: false, + }, + { + name: "v1CRD is established & not accepting names - not ready", + crd: builder.ForV1CustomResourceDefinition("MyCRD"). + Condition(builder.ForV1CustomResourceDefinitionCondition().Type(apiextv1.Established).Status(apiextv1.ConditionTrue).Result()).Result(), + want: false, + }, + { + name: "v1CRD is not established & accepting names - not ready", + crd: builder.ForV1CustomResourceDefinition("MyCRD"). + Condition(builder.ForV1CustomResourceDefinitionCondition().Type(apiextv1.NamesAccepted).Status(apiextv1.ConditionTrue).Result()).Result(), + want: false, + }, + { + name: "v1CRD is established & accepting names - ready", + crd: builder.ForV1CustomResourceDefinition("MyCRD"). + Condition(builder.ForV1CustomResourceDefinitionCondition().Type(apiextv1.Established).Status(apiextv1.ConditionTrue).Result()). + Condition(builder.ForV1CustomResourceDefinitionCondition().Type(apiextv1.NamesAccepted).Status(apiextv1.ConditionTrue).Result()). + Result(), + want: true, + }, + } + + for _, tc := range v1tests { + m, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.crd) + require.NoError(t, err) + result, err := IsCRDReady(&unstructured.Unstructured{Object: m}) + require.NoError(t, err) + assert.Equal(t, tc.want, result) + } + + // input param is unrecognized + resBytes := []byte(` { - "apiVersion": "apiextensions.k8s.io/v1beta1", + "apiVersion": "apiextensions.k8s.io/v9", "kind": "CustomResourceDefinition", "metadata": { - "name": "foos.example.foo.com" + "name": "foos.example.foo.com" }, "spec": { - "group": "example.foo.com", - "version": "v1alpha1", - "scope": "Namespaced", - "names": { - "plural": "foos", - "singular": "foo", - "kind": "Foo" - }, - "validation": { - "openAPIV3Schema": { - "required": [ - "spec" - ], - "properties": { - "spec": { - "required": [ - "bar" - ], - "properties": { - "bar": { - "type": "integer", - "minimum": 1 + "group": "example.foo.com", + "version": "v1alpha1", + "scope": "Namespaced", + "names": { + "plural": "foos", + "singular": "foo", + "kind": "Foo" + }, + "validation": { + "openAPIV3Schema": { + "required": [ + "spec" + ], + "properties": { + "spec": { + "required": [ + "bar" + ], + "properties": { + "bar": { + "type": "integer", + "minimum": 1 + } + } + } } - } } - } } - } } - } -`) - - var obj unstructured.Unstructured - err := json.Unmarshal(b, &obj) - require.NoError(t, err) - - var newCRD apiextv1beta1.CustomResourceDefinition - err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &newCRD) - // If there's no error, then the upstream issue is fixed, and we need to remove our workarounds. - require.Error(t, err) } -*/ +`) + obj := &unstructured.Unstructured{} + err := json.Unmarshal(resBytes, obj) + require.NoError(t, err) + _, err = IsCRDReady(obj) + assert.NotNil(t, err) +} diff --git a/pkg/volume/item_snapshot.go b/pkg/volume/item_snapshot.go new file mode 100644 index 000000000..909c21994 --- /dev/null +++ b/pkg/volume/item_snapshot.go @@ -0,0 +1,57 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volume + +import isv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/item_snapshotter/v1" + +// ItemSnapshot stores information about an item snapshot (includes volumes and other Astrolabe objects) taken as +// part of a Velero backup. +type ItemSnapshot struct { + Spec ItemSnapshotSpec `json:"spec"` + + Status ItemSnapshotStatus `json:"status"` +} + +type ItemSnapshotSpec struct { + // ItemSnapshotter is the name of the ItemSnapshotter plugin that took the snapshot + ItemSnapshotter string `json:"itemSnapshotter"` + + // BackupName is the name of the Velero backup this snapshot + // is associated with. + BackupName string `json:"backupName"` + + // BackupUID is the UID of the Velero backup this snapshot + // is associated with. + BackupUID string `json:"backupUID"` + + // Location is the name of the location where this snapshot is stored. + Location string `json:"location"` + + // Kubernetes resource identifier for the item + ResourceIdentifier string "json:resourceIdentifier" +} + +type ItemSnapshotStatus struct { + // ProviderSnapshotID is the ID of the snapshot taken by the ItemSnapshotter + ProviderSnapshotID string `json:"providerSnapshotID,omitempty"` + + // Metadata is the metadata returned with the snapshot to be returned to the ItemSnapshotter at restore time + Metadata map[string]string `json:"metadata,omitempty"` + + // Phase is the current state of the ItemSnapshot. + Phase isv1.SnapshotPhase `json:"phase,omitempty"` +} diff --git a/site/config.yaml b/site/config.yaml index 0697054ad..1a9e926e4 100644 --- a/site/config.yaml +++ b/site/config.yaml @@ -12,9 +12,11 @@ params: hero: backgroundColor: med-blue versioning: true - latest: v1.6 + latest: v1.8 versions: - main + - v1.8 + - v1.7 - v1.6 - v1.5 - v1.4 diff --git a/site/content/community/_index.md b/site/content/community/_index.md index b6740d946..bc57977b5 100644 --- a/site/content/community/_index.md +++ b/site/content/community/_index.md @@ -20,4 +20,3 @@ You can follow the work we do, see our milestones, and our backlog on our [GitHu * Read and comment on the [meeting notes](https://hackmd.io/Jq6F5zqZR7S80CeDWUklkA?view) * See previous community meetings on our [YouTube Channel](https://www.youtube.com/playlist?list=PL7bmigfV0EqQRysvqvqOtRNk4L5S7uqwM) * Have a question to discuss in the community meeting? Please add it to our [Q&A Discussion board](https://github.com/vmware-tanzu/velero/discussions/categories/community-support-q-a) -* Have a question about "how do I configure Velero to do X?", or "how can I backup X with Velero?", or "how do I start to contribute to Velero?" and would like to chat with us face to face, come join the [Velero Office Hours](https://hackmd.io/I3u1x0u9T46KhuYZN4LX-A?view) that's held twice per month. diff --git a/site/content/contributors/01-dave-smith-uchida.md b/site/content/contributors/00-dave-smith-uchida.md similarity index 88% rename from site/content/contributors/01-dave-smith-uchida.md rename to site/content/contributors/00-dave-smith-uchida.md index 15e9ea004..7f83b13ad 100644 --- a/site/content/contributors/01-dave-smith-uchida.md +++ b/site/content/contributors/00-dave-smith-uchida.md @@ -4,5 +4,5 @@ last_name: Smith-Uchida image: /img/contributors/dave.png github_handle: dsu-igeek --- -Technical Lead +Architect diff --git a/site/content/contributors/01-daniel-jiang.md b/site/content/contributors/01-daniel-jiang.md new file mode 100644 index 000000000..28cd7883c --- /dev/null +++ b/site/content/contributors/01-daniel-jiang.md @@ -0,0 +1,7 @@ +--- +first_name: Daniel +last_name: Jiang +image: /img/contributors/daniel-jiang.png +github_handle: reasonerjt +--- +Technical Lead \ No newline at end of file diff --git a/site/content/contributors/02-bridget-mcerlean.md b/site/content/contributors/02-bridget-mcerlean.md deleted file mode 100644 index fda4e4532..000000000 --- a/site/content/contributors/02-bridget-mcerlean.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -first_name: Bridget -last_name: McErlean -image: /img/contributors/bridget-mcerlean.png -github_handle: zubron ---- -Engineer diff --git a/site/content/contributors/02-wenkai-yin.md b/site/content/contributors/02-wenkai-yin.md new file mode 100644 index 000000000..87eea9adc --- /dev/null +++ b/site/content/contributors/02-wenkai-yin.md @@ -0,0 +1,7 @@ +--- +first_name: Wenkai +last_name: Yin +image: /img/contributors/wenkai-yin.png +github_handle: ywk253100 +--- +Engineer \ No newline at end of file diff --git a/site/content/contributors/02-xun-jiang.md b/site/content/contributors/02-xun-jiang.md new file mode 100644 index 000000000..0afc2dec7 --- /dev/null +++ b/site/content/contributors/02-xun-jiang.md @@ -0,0 +1,7 @@ +--- +first_name: Xun +last_name: Jiang +image: /img/contributors/xun-jiang.png +github_handle: blackpiglet +--- +Engineer \ No newline at end of file diff --git a/site/content/docs/main/api-types/schedule.md b/site/content/docs/main/api-types/schedule.md index 3c98bef16..9c5dc26ce 100644 --- a/site/content/docs/main/api-types/schedule.md +++ b/site/content/docs/main/api-types/schedule.md @@ -78,6 +78,8 @@ spec: # a default value of 30 days will be used. The default can be configured on the velero server # by passing the flag --default-backup-ttl. ttl: 24h0m0s + # Whether restic should be used to take a backup of all pod volumes by default. + defaultVolumesToRestic: true # The labels you want on backup objects, created from this schedule (instead of copying the labels you have on schedule object itself). # When this field is set, the labels from the Schedule resource are not copied to the Backup resource. metadata: diff --git a/site/content/docs/main/basic-install.md b/site/content/docs/main/basic-install.md index 8e3bcd48b..080b27b2f 100644 --- a/site/content/docs/main/basic-install.md +++ b/site/content/docs/main/basic-install.md @@ -8,7 +8,7 @@ Refer [this document](customize-installation.md) to customize your installation. ## Prerequisites -- Access to a Kubernetes cluster, v1.12 or later, with DNS and container networking enabled. +- Access to a Kubernetes cluster, v1.16 or later, with DNS and container networking enabled. For more information on supported Kubernetes versions, see the Velero [compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatabilty-matrix). - `kubectl` installed locally Velero uses object storage to store backups and associated artifacts. It also optionally integrates with supported block storage systems to snapshot your persistent volumes. Before beginning the installation process, you should identify the object storage provider and optional block storage provider(s) you'll be using from the list of [compatible providers][0]. @@ -23,7 +23,7 @@ If you want to perform your own testing of Velero on Windows, you must deploy Ve ## Install the CLI -### Option 1: macOS - Homebrew +### Option 1: MacOS - Homebrew On macOS, you can use [Homebrew](https://brew.sh) to install the `velero` client: @@ -70,4 +70,4 @@ Please refer to [this part of the documentation][5]. [2]: on-premises.md [3]: overview-plugins.md [4]: customize-installation.md#install-an-additional-volume-snapshot-provider -[5]: customize-installation.md#optional-velero-cli-configurations \ No newline at end of file +[5]: customize-installation.md#optional-velero-cli-configurations diff --git a/site/content/docs/main/code-standards.md b/site/content/docs/main/code-standards.md index 6e18f0c6d..a6dfa12c6 100644 --- a/site/content/docs/main/code-standards.md +++ b/site/content/docs/main/code-standards.md @@ -20,11 +20,11 @@ changelog. Add that to the PR. -If a PR does not warrant a changelog, the CI check for a changelog can be skipped by applying a `changelog-not-required` label on the PR. +If a PR does not warrant a changelog, the CI check for a changelog can be skipped by applying a `changelog-not-required` label on the PR. If you are making a PR on a release branch, you should still make a new file in the `changelogs/unreleased` folder on the release branch for your change. -## Copyright header +## Copyright header -Whenever a source code file is being modified, the copyright notice should be updated to our standard copyright notice. That is, it should read “Copyright the Velero contributors.” +Whenever a source code file is being modified, the copyright notice should be updated to our standard copyright notice. That is, it should read “Copyright the Velero contributors.” For new files, the entire copyright and license header must be added. @@ -61,7 +61,7 @@ Example: metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" corev1listers "k8s.io/client-go/listers/core/v1" - + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" ) diff --git a/site/content/docs/main/contributions/minio.md b/site/content/docs/main/contributions/minio.md index 2b93ee6c6..3c683473b 100644 --- a/site/content/docs/main/contributions/minio.md +++ b/site/content/docs/main/contributions/minio.md @@ -22,7 +22,17 @@ If you encounter issues with installing or configuring, see [Debugging Installat * Sufficient disk space to store backups in Minio. You will need sufficient disk space available to handle any backups plus at least 1GB additional. Minio will not operate if less than 1GB of free disk space is available. -## Download Velero +## Install the CLI + +### Option 1: MacOS - Homebrew + +On macOS, you can use [Homebrew](https://brew.sh) to install the `velero` client: + +```bash +brew install velero +``` + +### Option 2: GitHub release 1. Download the [latest official release's](https://github.com/vmware-tanzu/velero/releases) tarball for your client platform. @@ -40,19 +50,11 @@ of the Velero repository is under active development and is not guaranteed to be 1. Move the `velero` binary from the Velero directory to somewhere in your PATH. -### MacOS Installation - -On Mac, you can use [HomeBrew](https://brew.sh) to install the `velero` client: - -```bash -brew install velero -``` - ## Set up server -These instructions start the Velero server and a Minio instance that is accessible from within the cluster only. See [Expose Minio outside your cluster][31] for information about configuring your cluster for outside access to Minio. Outside access is required to access logs and run `velero describe` commands. +These instructions start the Velero server and a Minio instance that is accessible from within the cluster only. See [Expose Minio outside your cluster](#expose-minio-outside-your-cluster-with-a-service) for information about configuring your cluster for outside access to Minio. Outside access is required to access logs and run `velero describe` commands. -1. Create a Velero-specific credentials file (`credentials-velero`) in your local directory: +1. Create a Velero-specific credentials file (`credentials-velero`) in your Velero directory: ``` [default] @@ -83,6 +85,8 @@ These instructions start the Velero server and a Minio instance that is accessib Additionally, you can specify `--use-restic` to enable restic support, and `--wait` to wait for the deployment to be ready. + This example also assumes you have named your Minio bucket "velero". + 1. Deploy the example nginx application: diff --git a/site/content/docs/main/contributions/tencent-config.md b/site/content/docs/main/contributions/tencent-config.md index 50edfce49..11b0762c0 100644 --- a/site/content/docs/main/contributions/tencent-config.md +++ b/site/content/docs/main/contributions/tencent-config.md @@ -11,7 +11,7 @@ You can deploy Velero on Tencent [TKE](https://cloud.tencent.com/document/produc - Registered [Tencent Cloud Account](https://cloud.tencent.com/register). - [Tencent Cloud COS](https://console.cloud.tencent.com/cos) service, referred to as COS, has been launched -- A Kubernetes cluster has been created, cluster version v1.12 or later, and the cluster can use DNS and Internet services normally. If you need to create a TKE cluster, refer to the Tencent [create a cluster](https://cloud.tencent.com/document/product/457/32189) documentation. +- A Kubernetes cluster has been created, cluster version v1.16 or later, and the cluster can use DNS and Internet services normally. If you need to create a TKE cluster, refer to the Tencent [create a cluster](https://cloud.tencent.com/document/product/457/32189) documentation. ## Create a Tencent Cloud COS bucket @@ -21,7 +21,7 @@ Set access to the bucket through the object storage console, the bucket needs to ## Get bucket access credentials -Velero uses an AWS S3-compatible API to access Tencent Cloud COS storage, which requires authentication using a pair of access key IDs and key-created signatures. +Velero uses an AWS S3-compatible API to access Tencent Cloud COS storage, which requires authentication using a pair of access key IDs and key-created signatures. In the S3 API parameter, the "access_key_id" field is the access key ID and the "secret_access_key" field is the key. @@ -116,7 +116,7 @@ After deleting the MinIO resource, use your backup to restore the deleted MinIO kubectl patch backupstoragelocation default --namespace velero \ --type merge \ --patch '{"spec":{"accessMode":"ReadOnly"}}' - + ``` Modifying access to Velero's storage location is "ReadOnly," as shown in the following image: diff --git a/site/content/docs/main/csi.md b/site/content/docs/main/csi.md index 49f4f77e9..5c2950b35 100644 --- a/site/content/docs/main/csi.md +++ b/site/content/docs/main/csi.md @@ -1,26 +1,25 @@ --- -title: "Container Storage Interface Snapshot Support in Velero" +title: "Beta Container Storage Interface Snapshot Support in Velero" layout: docs --- -_This feature is under development. Documentation may not be up-to-date and features may not work as expected._ +Integrating Container Storage Interface (CSI) snapshot support into Velero enables Velero to backup and restore CSI-backed volumes using the [Kubernetes CSI Snapshot APIs](https://kubernetes.io/docs/concepts/storage/volume-snapshots/). -Integrating Container Storage Interface (CSI) snapshot support into Velero enables Velero to backup and restore CSI-backed volumes using the [Kubernetes CSI Snapshot Beta APIs](https://kubernetes.io/docs/concepts/storage/volume-snapshots/). +By supporting CSI snapshot APIs, Velero can support any volume provider that has a CSI driver, without requiring a Velero-specific plugin to be available. This page gives an overview of how to add support for CSI snapshots to Velero through CSI plugins. For more information about specific components, see the [plugin repo](https://github.com/vmware-tanzu/velero-plugin-for-csi/). -By supporting CSI snapshot APIs, Velero can support any volume provider that has a CSI driver, without requiring a Velero-specific plugin to be available. +The Velero Container Storage Interface (CSI) plugins are in beta. It's not recommended for use in a production environment and the `EnableCSI` feature flag is disabled by default. The Velero team plans to follow the upstream Kubernetes support level for CSI volume snapshotting, and the Velero CSI plugins will reach general availability sometime after volume snapshotting is GA in upstream Kubernetes. + +**Note:** The AWS, Microsoft Azure, and Google Cloud Platform (GCP) Velero plugins version 1.4 and later are able to snapshot and restore persistent volumes provisioned by a CSI driver via the APIs of the cloud provider, without having to install Velero CSI plugins. See the [AWS](https://github.com/vmware-tanzu/velero-plugin-for-aws), [Microsoft Azure](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure), and [Google Cloud Platform (GCP)](https://github.com/vmware-tanzu/velero-plugin-for-gcp) Velero plugin repo for more information on supported CSI drivers. ## Prerequisites -The following are the prerequisites for using Velero to take Container Storage Interface (CSI) snapshots: - - 1. The cluster is Kubernetes version 1.17 or greater. - 1. The cluster is running a CSI driver capable of support volume snapshots at the [v1beta1 API level](https://kubernetes.io/blog/2019/12/09/kubernetes-1-17-feature-cis-volume-snapshot-beta/). - 1. When restoring CSI volumesnapshots across clusters, the name of the CSI driver in the destination cluster is the same as that on the source cluster to ensure cross cluster portability of CSI volumesnapshots + 1. Your cluster is Kubernetes version 1.17 or greater. + 1. Your cluster is running a CSI driver capable of support volume snapshots at the [v1beta1 API level](https://kubernetes.io/blog/2019/12/09/kubernetes-1-17-feature-cis-volume-snapshot-beta/). + 1. When restoring CSI VolumeSnapshots across clusters, the name of the CSI driver in the destination cluster is the same as that on the source cluster to ensure cross cluster portability of CSI VolumeSnapshots ## Installing Velero with CSI support -Ensure that the Velero server is running with the `EnableCSI` feature flag. See [Enabling Features][1] for more information. -Also, the Velero [CSI plugin][2] ([Docker Hub][3]) is necessary to integrate with the CSI volume snapshot APIs. +To integrate Velero with the CSI volume snapshot APIs, you must enable the `EnableCSI` feature flag and install the Velero [CSI plugins][2] on the Velero server. Both of these can be added with the `velero install` command. @@ -32,24 +31,19 @@ velero install \ ``` To include the status of CSI objects associated with a Velero backup in `velero backup describe` output, run `velero client config set features=EnableCSI`. -See [Enabling Features][1] for more information about managing client-side feature flags. +See [Enabling Features][1] for more information about managing client-side feature flags. You can also view the image on [Docker Hub][3]. ## Implementation Choices -This section documents some of the choices made during implementation of the Velero [CSI plugin][2]: +This section documents some of the choices made during implementation of the Velero [CSI plugins][2]: -1. Volumesnapshots created by the plugin will be retained only for the lifetime of the backup even if the `DeletionPolicy` on the volumesnapshotclass is set to `Retain`. To accomplish this, during deletion of the backup the prior to deleting the volumesnapshot, volumesnapshotcontent object will be patched to set its `DeletionPolicy` to `Delete`. Thus deleting volumesnapshot object will result in cascade delete of the volumesnapshotcontent and the snapshot in the storage provider. -1. Volumesnapshotcontent objects created during a velero backup that are dangling, unbound to a volumesnapshot object, will also be discovered, through labels, and deleted on backup deletion. -1. The Velero CSI plugin, to backup CSI backed PVCs, will choose the VolumeSnapshotClass in the cluster that has the same driver name and also has the `velero.io/csi-volumesnapshot-class` label set on it, like +1. VolumeSnapshots created by the Velero CSI plugins are retained only for the lifetime of the backup even if the `DeletionPolicy` on the VolumeSnapshotClass is set to `Retain`. To accomplish this, during deletion of the backup the prior to deleting the VolumeSnapshot, VolumeSnapshotContent object is patched to set its `DeletionPolicy` to `Delete`. Deleting the VolumeSnapshot object will result in cascade delete of the VolumeSnapshotContent and the snapshot in the storage provider. +1. VolumeSnapshotContent objects created during a `velero backup` that are dangling, unbound to a VolumeSnapshot object, will be discovered, using labels, and deleted on backup deletion. +1. The Velero CSI plugins, to backup CSI backed PVCs, will choose the VolumeSnapshotClass in the cluster that has the same driver name and also has the `velero.io/csi-volumesnapshot-class` label set on it, like ```yaml velero.io/csi-volumesnapshot-class: "true" ``` -## Roadmap - -Velero's support level for CSI volume snapshotting will follow upstream Kubernetes support for the feature, and will reach general availability sometime -after volume snapshotting is GA in upstream Kubernetes. Beta support is expected to launch in Velero v1.4. - ## How it Works - Overview Velero's CSI support does not rely on the Velero VolumeSnapshotter plugin interface. @@ -64,6 +58,9 @@ The external-snapshotter plugin will call the CSI driver's snapshot method, and Once an ID is generated and the storage system marks the snapshot as usable for restore, the VolumeSnapshotContent object will be updated with a `status.snapshotHandle` and the `status.readyToUse` field will be set. Velero will include the generated VolumeSnapshot and VolumeSnapshotContent objects in the backup tarball, as well as upload all VolumeSnapshots and VolumeSnapshotContents objects in a JSON file to the object storage system. + +**NOTE:** Not all cloud provider's CSI drivers guarantee snapshot durability, meaning that the VolumeSnapshot and VolumeSnapshotContent objects may be stored in the same object storage system location as the original PersistentVolume and may be vulnerable to data loss. You should refer to your cloud provider's documentation for more information on configuring snapshot durability. + When Velero synchronizes backups into a new cluster, VolumeSnapshotContent objects will be synced into the cluster as well, so that Velero can manage backup expiration appropriately. The `DeletionPolicy` on the VolumeSnapshotContent will be the same as the `DeletionPolicy` on the VolumeSnapshotClass that was used to create the VolumeSnapshot. Setting a `DeletionPolicy` of `Retain` on the VolumeSnapshotClass will preserve the volume snapshot in the storage system for the lifetime of the Velero backup and will prevent the deletion of the volume snapshot, in the storage system, in the event of a disaster where the namespace with the VolumeSnapshot object may be lost. diff --git a/site/content/docs/main/custom-plugins.md b/site/content/docs/main/custom-plugins.md index 1789f68f9..c0937c20c 100644 --- a/site/content/docs/main/custom-plugins.md +++ b/site/content/docs/main/custom-plugins.md @@ -99,8 +99,7 @@ data: # add your configuration data here as key-value pairs ``` -Then, in your plugin's implementation, you can read this ConfigMap to fetch the necessary configuration. See the [restic restore action][3] -for an example of this -- in particular, the `getPluginConfig(...)` function. +Then, in your plugin's implementation, you can read this ConfigMap to fetch the necessary configuration. ## Feature Flags @@ -114,4 +113,3 @@ Velero adds the `LD_LIBRARY_PATH` into the list of environment variables to prov [1]: https://github.com/vmware-tanzu/velero-plugin-example [2]: https://github.com/vmware-tanzu/velero/blob/main/pkg/plugin/logger.go -[3]: https://github.com/vmware-tanzu/velero/blob/main/pkg/restore/restic_restore_action.go diff --git a/site/content/docs/main/migration-case.md b/site/content/docs/main/migration-case.md index c547ebbb4..9ad7b809a 100644 --- a/site/content/docs/main/migration-case.md +++ b/site/content/docs/main/migration-case.md @@ -53,6 +53,7 @@ If you encounter issues, make sure that Velero is running in the same namespace ## Migrating Workloads Across Different Kubernetes Versions Migration across clusters that are not running the same version of Kubernetes might be possible, but some factors need to be considered: compatibility of API groups between clusters for each custom resource, and if a Kubernetes version upgrade breaks the compatibility of core/native API groups. For more information about API group versions, please see [EnableAPIGroupVersions](enable-api-group-versions-feature.md). +**Note:** Velero doesn't support restoring into a cluster with a lower Kubernetes version than where the backup was taken. [1]: how-velero-works.md#set-a-backup-to-expire [2]: restic.md diff --git a/site/content/docs/main/release-instructions.md b/site/content/docs/main/release-instructions.md index 0d59a60a9..29dc8e0ce 100644 --- a/site/content/docs/main/release-instructions.md +++ b/site/content/docs/main/release-instructions.md @@ -82,6 +82,12 @@ For each major or minor release, create and publish a blog post to let folks kno - Do a review of the diffs, and/or run `make serve-docs` and review the site. - Submit a PR containing the changelog and the version-tagged docs. +### Pin the base image +The image of velero is built based on [Distroless docker image](https://github.com/GoogleContainerTools/distroless). +For the reproducibility of the release, before the release candidate is tagged, we need to make sure the in the Dockerfile +on the release branch, the base image is referenced by digest, such as +https://github.com/vmware-tanzu/velero/blob/release-1.7/Dockerfile#L53-L54 + ## Velero release ### Notes - Pre-requisite: PR with the changelog and docs is merged, so that it's included in the release tag. diff --git a/site/content/docs/main/resource-filtering.md b/site/content/docs/main/resource-filtering.md index 6c1470e04..ddcd97e4a 100644 --- a/site/content/docs/main/resource-filtering.md +++ b/site/content/docs/main/resource-filtering.md @@ -88,6 +88,12 @@ Wildcard takes precedence when both a wildcard and specific resource are include ```bash velero backup create --selector = ``` +* Include resources that are not matching the selector + ```bash + velero backup create --selector != + ``` + +For more information read the [Kubernetes label selector documentation](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors) ## Excludes diff --git a/site/content/docs/main/restic.md b/site/content/docs/main/restic.md index 096abda6e..ac75f0f9d 100644 --- a/site/content/docs/main/restic.md +++ b/site/content/docs/main/restic.md @@ -10,41 +10,41 @@ the supported cloud providers’ block storage offerings (Amazon EBS Volumes, Az It also provides a plugin model that enables anyone to implement additional object and block storage backends, outside the main Velero repository. -The restic integration was added to give you an out-of-the-box solution for backing up and restoring almost any type of Kubernetes volume. This integration is an addition to Velero's capabilities, not a replacement for existing functionality. If you're running on AWS, and taking EBS snapshots as part of your regular Velero backups, there's no need to switch to using restic. However, if you need a volume snapshot plugin for your storage platform, or if you're using EFS, AzureFile, NFS, emptyDir, -local, or any other volume type that doesn't have a native snapshot concept, restic might be for you. +Velero's Restic integration was added to give you an out-of-the-box solution for backing up and restoring almost any type of Kubernetes volume. This integration is an addition to Velero's capabilities, not a replacement for existing functionality. If you're running on AWS, and taking EBS snapshots as part of your regular Velero backups, there's no need to switch to using Restic. However, if you need a volume snapshot plugin for your storage platform, or if you're using EFS, AzureFile, NFS, emptyDir, +local, or any other volume type that doesn't have a native snapshot concept, Restic might be for you. Restic is not tied to a specific storage platform, which means that this integration also paves the way for future work to enable cross-volume-type data migrations. **NOTE:** hostPath volumes are not supported, but the [local volume type][4] is supported. -## Setup restic +## Setup Restic ### Prerequisites -- Understand how Velero performs [backups with the restic integration](#how-backup-and-restore-work-with-restic). +- Understand how Velero performs [backups with the Restic integration](#how-backup-and-restore-work-with-restic). - [Download][3] the latest Velero release. -- Kubernetes v1.12.0 and later. Velero's restic integration requires the Kubernetes [MountPropagation feature][6], which is enabled by default in Kubernetes v1.12.0 and later. +- Kubernetes v1.16.0 and later. Velero's Restic integration requires the Kubernetes [MountPropagation feature][6]. -### Install restic +### Install Restic -To install restic, use the `--use-restic` flag in the `velero install` command. See the [install overview][2] for more details on other flags for the install command. +To install Restic, use the `--use-restic` flag in the `velero install` command. See the [install overview][2] for more details on other flags for the install command. ``` velero install --use-restic ``` -When using restic on a storage provider that doesn't have Velero support for snapshots, the `--use-volume-snapshots=false` flag prevents an unused `VolumeSnapshotLocation` from being created on installation. +When using Restic on a storage provider that doesn't have Velero support for snapshots, the `--use-volume-snapshots=false` flag prevents an unused `VolumeSnapshotLocation` from being created on installation. -### Configure restic DaemonSet spec +### Configure Restic DaemonSet spec -After installation, some PaaS/CaaS platforms based on Kubernetes also require modifications the restic DaemonSet spec. The steps in this section are only needed if you are installing on RancherOS, OpenShift, VMware Tanzu Kubernetes Grid Integrated Edition (formerly VMware Enterprise PKS), or Microsoft Azure. +After installation, some PaaS/CaaS platforms based on Kubernetes also require modifications the Restic DaemonSet spec. The steps in this section are only needed if you are installing on RancherOS, OpenShift, VMware Tanzu Kubernetes Grid Integrated Edition (formerly VMware Enterprise PKS), or Microsoft Azure. **RancherOS** -Update the host path for volumes in the restic DaemonSet in the Velero namespace from `/var/lib/kubelet/pods` to `/opt/rke/var/lib/kubelet/pods`. +Update the host path for volumes in the Restic DaemonSet in the Velero namespace from `/var/lib/kubelet/pods` to `/opt/rke/var/lib/kubelet/pods`. ```yaml hostPath: @@ -62,7 +62,7 @@ hostPath: **OpenShift** -To mount the correct hostpath to pods volumes, run the restic pod in `privileged` mode. +To mount the correct hostpath to pods volumes, run the Restic pod in `privileged` mode. 1. Add the `velero` ServiceAccount to the `privileged` SCC: @@ -125,7 +125,7 @@ To mount the correct hostpath to pods volumes, run the restic pod in `privileged ``` -If restic is not running in a privileged mode, it will not be able to access pods volumes within the mounted hostpath directory because of the default enforced SELinux mode configured in the host system level. You can [create a custom SCC](https://docs.openshift.com/container-platform/3.11/admin_guide/manage_scc.html) to relax the security in your cluster so that restic pods are allowed to use the hostPath volume plug-in without granting them access to the `privileged` SCC. +If Restic is not running in a privileged mode, it will not be able to access pods volumes within the mounted hostpath directory because of the default enforced SELinux mode configured in the host system level. You can [create a custom SCC](https://docs.openshift.com/container-platform/3.11/admin_guide/manage_scc.html) to relax the security in your cluster so that Restic pods are allowed to use the hostPath volume plug-in without granting them access to the `privileged` SCC. By default a userland openshift namespace will not schedule pods on all nodes in the cluster. @@ -147,7 +147,7 @@ oc create -n -f ds.yaml **VMware Tanzu Kubernetes Grid Integrated Edition (formerly VMware Enterprise PKS)** -You need to enable the `Allow Privileged` option in your plan configuration so that restic is able to mount the hostpath. +You need to enable the `Allow Privileged` option in your plan configuration so that Restic is able to mount the hostpath. The hostPath should be changed from `/var/lib/kubelet/pods` to `/var/vcap/data/kubelet/pods` @@ -172,16 +172,16 @@ kubectl patch storageclass/ \ ## To back up -Velero supports two approaches of discovering pod volumes that need to be backed up using restic: +Velero supports two approaches of discovering pod volumes that need to be backed up using Restic: -- Opt-in approach: Where every pod containing a volume to be backed up using restic must be annotated with the volume's name. -- Opt-out approach: Where all pod volumes are backed up using restic, with the ability to opt-out any volumes that should not be backed up. +- Opt-in approach: Where every pod containing a volume to be backed up using Restic must be annotated with the volume's name. +- Opt-out approach: Where all pod volumes are backed up using Restic, with the ability to opt-out any volumes that should not be backed up. The following sections provide more details on the two approaches. ### Using the opt-out approach -In this approach, Velero will back up all pod volumes using restic with the exception of: +In this approach, Velero will back up all pod volumes using Restic with the exception of: - Volumes mounting the default service account token, kubernetes secrets, and config maps - Hostpath volumes @@ -190,7 +190,7 @@ It is possible to exclude volumes from being backed up using the `backup.velero. Instructions to back up using this approach are as follows: -1. Run the following command on each pod that contains volumes that should **not** be backed up using restic +1. Run the following command on each pod that contains volumes that should **not** be backed up using Restic ```bash kubectl -n YOUR_POD_NAMESPACE annotate pod/YOUR_POD_NAME backup.velero.io/backup-volumes-excludes=YOUR_VOLUME_NAME_1,YOUR_VOLUME_NAME_2,... @@ -221,7 +221,7 @@ Instructions to back up using this approach are as follows: - name: pvc2-vm claimName: pvc2 ``` - to exclude restic backup of volume `pvc1-vm`, you would run: + to exclude Restic backup of volume `pvc1-vm`, you would run: ```bash kubectl -n sample annotate pod/app1 backup.velero.io/backup-volumes-excludes=pvc1-vm @@ -248,7 +248,7 @@ Instructions to back up using this approach are as follows: ### Using opt-in pod volume backup -Velero, by default, uses this approach to discover pod volumes that need to be backed up using restic, where every pod containing a volume to be backed up using restic must be annotated with the volume's name. +Velero, by default, uses this approach to discover pod volumes that need to be backed up using Restic, where every pod containing a volume to be backed up using Restic must be annotated with the volume's name. Instructions to back up using this approach are as follows: @@ -310,7 +310,7 @@ Instructions to back up using this approach are as follows: ## To restore -Regardless of how volumes are discovered for backup using restic, the process of restoring remains the same. +Regardless of how volumes are discovered for backup using Restic, the process of restoring remains the same. 1. Restore from your Velero backup: @@ -331,20 +331,20 @@ Regardless of how volumes are discovered for backup using restic, the process of - `hostPath` volumes are not supported. [Local persistent volumes][4] are supported. - Those of you familiar with [restic][1] may know that it encrypts all of its data. Velero uses a static, -common encryption key for all restic repositories it creates. **This means that anyone who has access to your -bucket can decrypt your restic backup data**. Make sure that you limit access to the restic bucket +common encryption key for all Restic repositories it creates. **This means that anyone who has access to your +bucket can decrypt your Restic backup data**. Make sure that you limit access to the Restic bucket appropriately. - An incremental backup chain will be maintained across pod reschedules for PVCs. However, for pod volumes that are *not* PVCs, such as `emptyDir` volumes, when a pod is deleted/recreated (for example, by a ReplicaSet/Deployment), the next backup of those volumes will be full rather than incremental, because the pod volume's lifecycle is assumed to be defined by its pod. - Restic scans each file in a single thread. This means that large files (such as ones storing a database) will take a long time to scan for data deduplication, even if the actual difference is small. -- If you plan to use the Velero restic integration to backup 100GB of data or more, you may need to [customize the resource limits](/docs/main/customize-installation/#customize-resource-requests-and-limits) to make sure backups complete successfully. -- Velero's restic integration backs up data from volumes by accessing the node's filesystem, on which the pod is running. For this reason, restic integration can only backup volumes that are mounted by a pod and not directly from the PVC. +- If you plan to use Velero's Restic integration to backup 100GB of data or more, you may need to [customize the resource limits](/docs/main/customize-installation/#customize-resource-requests-and-limits) to make sure backups complete successfully. +- Velero's Restic integration backs up data from volumes by accessing the node's filesystem, on which the pod is running. For this reason, Velero's Restic integration can only backup volumes that are mounted by a pod and not directly from the PVC. For orphan PVC/PV pairs (without running pods), some Velero users overcame this limitation running a staging pod (i.e. a busybox or alpine container with an infinite sleep) to mount these PVC/PV pairs prior taking a Velero backup. ## Customize Restore Helper Container -Velero uses a helper init container when performing a restic restore. By default, the image for this container is `velero/velero-restic-restore-helper:`, +Velero uses a helper init container when performing a Restic restore. By default, the image for this container is `velero/velero-restic-restore-helper:`, where `VERSION` matches the version/tag of the main Velero image. You can customize the image that is used for this helper by creating a ConfigMap in the Velero namespace with the alternate image. @@ -426,7 +426,7 @@ Are your Velero server and daemonset pods running? kubectl get pods -n velero ``` -Does your restic repository exist, and is it ready? +Does your Restic repository exist, and is it ready? ```bash velero restic repo get @@ -462,31 +462,31 @@ kubectl -n velero logs DAEMON_POD_NAME **NOTE**: You can increase the verbosity of the pod logs by adding `--log-level=debug` as an argument to the container command in the deployment/daemonset pod template spec. -## How backup and restore work with restic +## How backup and restore work with Restic Velero has three custom resource definitions and associated controllers: - `ResticRepository` - represents/manages the lifecycle of Velero's [restic repositories][5]. Velero creates -a restic repository per namespace when the first restic backup for a namespace is requested. The controller -for this custom resource executes restic repository lifecycle commands -- `restic init`, `restic check`, +a Restic repository per namespace when the first Restic backup for a namespace is requested. The controller +for this custom resource executes Restic repository lifecycle commands -- `restic init`, `restic check`, and `restic prune`. - You can see information about your Velero restic repositories by running `velero restic repo get`. + You can see information about your Velero's Restic repositories by running `velero restic repo get`. -- `PodVolumeBackup` - represents a restic backup of a volume in a pod. The main Velero backup process creates +- `PodVolumeBackup` - represents a Restic backup of a volume in a pod. The main Velero backup process creates one or more of these when it finds an annotated pod. Each node in the cluster runs a controller for this resource (in a daemonset) that handles the `PodVolumeBackups` for pods on that node. The controller executes `restic backup` commands to backup pod volume data. -- `PodVolumeRestore` - represents a restic restore of a pod volume. The main Velero restore process creates one -or more of these when it encounters a pod that has associated restic backups. Each node in the cluster runs a +- `PodVolumeRestore` - represents a Restic restore of a pod volume. The main Velero restore process creates one +or more of these when it encounters a pod that has associated Restic backups. Each node in the cluster runs a controller for this resource (in the same daemonset as above) that handles the `PodVolumeRestores` for pods on that node. The controller executes `restic restore` commands to restore pod volume data. ### Backup -1. Based on configuration, the main Velero backup process uses the opt-in or opt-out approach to check each pod that it's backing up for the volumes to be backed up using restic. -1. When found, Velero first ensures a restic repository exists for the pod's namespace, by: +1. Based on configuration, the main Velero backup process uses the opt-in or opt-out approach to check each pod that it's backing up for the volumes to be backed up using Restic. +1. When found, Velero first ensures a Restic repository exists for the pod's namespace, by: - checking if a `ResticRepository` custom resource already exists - if not, creating a new one, and waiting for the `ResticRepository` controller to init/check it 1. Velero then creates a `PodVolumeBackup` custom resource per volume listed in the pod annotation @@ -501,14 +501,14 @@ on that node. The controller executes `restic restore` commands to restore pod v ### Restore 1. The main Velero restore process checks each existing `PodVolumeBackup` custom resource in the cluster to backup from. -1. For each `PodVolumeBackup` found, Velero first ensures a restic repository exists for the pod's namespace, by: +1. For each `PodVolumeBackup` found, Velero first ensures a Restic repository exists for the pod's namespace, by: - checking if a `ResticRepository` custom resource already exists - if not, creating a new one, and waiting for the `ResticRepository` controller to init/check it (note that in this case, the actual repository should already exist in object storage, so the Velero controller will simply check it for integrity) -1. Velero adds an init container to the pod, whose job is to wait for all restic restores for the pod to complete (more +1. Velero adds an init container to the pod, whose job is to wait for all Restic restores for the pod to complete (more on this shortly) -1. Velero creates the pod, with the added init container, by submitting it to the Kubernetes API +1. Velero creates the pod, with the added init container, by submitting it to the Kubernetes API. Then, the Kubernetes scheduler schedules this pod to a worker node, and the pod must be in a running state. If the pod fails to start for some reason (i.e. lack of cluster resources), the Restic restore will not be done. 1. Velero creates a `PodVolumeRestore` custom resource for each volume to be restored in the pod 1. The main Velero process now waits for each `PodVolumeRestore` resource to complete or fail 1. Meanwhile, each `PodVolumeRestore` is handled by the controller on the appropriate node, which: @@ -528,7 +528,7 @@ on to running other init containers/the main containers. ### Monitor backup annotation -Velero does not provide a mechanism to detect persistent volume claims that are missing the restic backup annotation. +Velero does not provide a mechanism to detect persistent volume claims that are missing the Restic backup annotation. To solve this, a controller was written by Thomann Bits&Beats: [velero-pvc-watcher][7] @@ -542,4 +542,3 @@ To solve this, a controller was written by Thomann Bits&Beats: [velero-pvc-watch [8]: https://docs.microsoft.com/en-us/azure/aks/azure-files-dynamic-pv [9]: https://github.com/restic/restic/issues/1800 [11]: customize-installation.md#default-pod-volume-backup-to-restic - diff --git a/site/content/docs/main/supported-providers.md b/site/content/docs/main/supported-providers.md index c1c20a06a..5e75dc1e2 100644 --- a/site/content/docs/main/supported-providers.md +++ b/site/content/docs/main/supported-providers.md @@ -14,8 +14,8 @@ Velero supports a variety of storage providers for different backup and snapshot | [Amazon Web Services (AWS)](https://aws.amazon.com) | AWS S3 | AWS EBS | [Velero plugin for AWS](https://github.com/vmware-tanzu/velero-plugin-for-aws) | [AWS Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-aws#setup) | | [Google Cloud Platform (GCP)](https://cloud.google.com) | Google Cloud Storage| Google Compute Engine Disks | [Velero plugin for GCP](https://github.com/vmware-tanzu/velero-plugin-for-gcp) | [GCP Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-gcp#setup) | | [Microsoft Azure](https://azure.com) | Azure Blob Storage | Azure Managed Disks | [Velero plugin for Microsoft Azure](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure) | [Azure Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#setup) | -| [VMware vSphere](https://github.com/vmware-tanzu/velero-plugin-for-vsphere) | 🚫 | vSphere Volumes | [VMware vSphere](https://github.com/vmware-tanzu/velero-plugin-for-vsphere) | [vSphere Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-vsphere#velero-plugin-for-vsphere-installation-and-configuration-details) | -| [Container Storage Interface (CSI)](https://github.com/vmware-tanzu/velero-plugin-for-csi/)| 🚫 | CSI Volumes | [Velero plugin for CSI](https://github.com/vmware-tanzu/velero-plugin-for-csi/) | [CSI Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-csi#kinds-of-plugins-included) | +| [VMware vSphere](https://www.vmware.com/ca/products/vsphere.html) | 🚫 | vSphere Volumes | [VMware vSphere](https://github.com/vmware-tanzu/velero-plugin-for-vsphere) | [vSphere Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-vsphere#velero-plugin-for-vsphere-installation-and-configuration-details) | +| [Container Storage Interface (CSI)](https://kubernetes.io/blog/2019/01/15/container-storage-interface-ga/)| 🚫 | CSI Volumes | [Velero plugin for CSI](https://github.com/vmware-tanzu/velero-plugin-for-csi/) | [CSI Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-csi#kinds-of-plugins-included) | {{< /table >}} Contact: [#Velero Slack](https://kubernetes.slack.com/messages/velero), [GitHub Issues](https://github.com/vmware-tanzu/velero/issues) diff --git a/site/content/docs/main/tilt.md b/site/content/docs/main/tilt.md index 42f68cfe4..3f2a8d90e 100644 --- a/site/content/docs/main/tilt.md +++ b/site/content/docs/main/tilt.md @@ -19,7 +19,7 @@ files in this directory are gitignored so you may configure your setup according ## Prerequisites 1. [Docker](https://docs.docker.com/install/) v19.03 or newer -1. A Kubernetes cluster v1.12 or greater (does not have to be Kind) +1. A Kubernetes cluster v1.16 or greater (does not have to be Kind) 1. [Tilt](https://docs.tilt.dev/install.html) v0.12.0 or newer 1. Clone the [Velero project](https://github.com/vmware-tanzu/velero) repository locally @@ -133,7 +133,7 @@ Here are two ways to use MinIO as the storage: In the `tilt-settings.json` file, set `"setup-minio": true`. This will configure a Kubernetes deployment containing a running instance of MinIO inside your cluster. There are [extra steps](contributions/minio/#expose-minio-outside-your-cluster-with-a-service) -necessary to expose MinIO outside the cluster. +necessary to expose MinIO outside the cluster. To access this storage, you will need to expose MinIO outside the cluster by forwarding the MinIO port to the local machine using kubectl port-forward -n svc/minio 9000. Update the BSL configuration to use that as its "public URL" by adding `publicUrl: http://localhost:9000` to the BSL config. This is necessary to do things like download a backup file. diff --git a/site/content/docs/main/troubleshooting.md b/site/content/docs/main/troubleshooting.md index 7b035006f..cdfd25305 100644 --- a/site/content/docs/main/troubleshooting.md +++ b/site/content/docs/main/troubleshooting.md @@ -17,13 +17,14 @@ These tips can help you troubleshoot known issues. If they don't help, you can [ You can use the `velero bug` command to open a [Github issue][4] by launching a browser window with some prepopulated values. Values included are OS, CPU architecture, `kubectl` client and server versions (if available) and the `velero` client version. This information isn't submitted to Github until you click the `Submit new issue` button in the Github UI, so feel free to add, remove or update whatever information you like. -Some general commands for troubleshooting that may be helpful: +You can use the `velero debug` command to generate a debug bundle, which is a tarball +that contains: +* Version information +* Logs of velero server and plugins +* Resources managed by velero server such as backup, restore, podvolumebackup, podvolumerestore, etc. +* Logs of the backup and restore, if specified in the parameters -* `velero backup describe ` - describe the details of a backup -* `velero backup logs ` - fetch the logs for this specific backup. Useful for viewing failures and warnings, including resources that could not be backed up. -* `velero restore describe ` - describe the details of a restore -* `velero restore logs ` - fetch the logs for this specific restore. Useful for viewing failures and warnings, including resources that could not be restored. -* `kubectl logs deployment/velero -n velero` - fetch the logs of the Velero server pod. This provides the output of the Velero server processes. +Please use command `velero debug --help` to see more usage details. ### Getting velero debug logs diff --git a/site/content/docs/main/upgrade-to-1.8.md b/site/content/docs/main/upgrade-to-1.8.md new file mode 100644 index 000000000..3a1e80cf5 --- /dev/null +++ b/site/content/docs/main/upgrade-to-1.8.md @@ -0,0 +1,92 @@ +--- +title: "Upgrading to Velero 1.8" +layout: docs +--- + +## Prerequisites + +- Velero [v1.7.x][7] installed. + +If you're not yet running at least Velero v1.6, see the following: + +- [Upgrading to v1.1][1] +- [Upgrading to v1.2][2] +- [Upgrading to v1.3][3] +- [Upgrading to v1.4][4] +- [Upgrading to v1.5][5] +- [Upgrading to v1.6][6] +- [Upgrading to v1.7][7] + +Before upgrading, check the [Velero compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatabilty-matrix) to make sure your version of Kubernetes is supported by the new version of Velero. + +## Instructions + +1. Install the Velero v1.8 command-line interface (CLI) by following the [instructions here][0]. + + Verify that you've properly installed it by running: + + ```bash + velero version --client-only + ``` + + You should see the following output: + + ```bash + Client: + Version: v1.8.0 + Git commit: + ``` + +1. Update the Velero custom resource definitions (CRDs) to include schema changes across all CRDs that are at the core of the new features in this release: + + ```bash + velero install --crds-only --dry-run -o yaml | kubectl apply -f - + ``` + + **NOTE:** Since velero v1.8.0 only v1 CRD will be supported during installation, therefore, the v1.8.0 will only work on kubernetes version >= v1.16 + +1. Update the container image used by the Velero deployment and, optionally, the restic daemon set: + + ```bash + kubectl set image deployment/velero \ + velero=velero/velero:v1.8.0 \ + --namespace velero + + # optional, if using the restic daemon set + kubectl set image daemonset/restic \ + restic=velero/velero:v1.8.0 \ + --namespace velero + ``` + +1. Confirm that the deployment is up and running with the correct version by running: + + ```bash + velero version + ``` + + You should see the following output: + + ```bash + Client: + Version: v1.8.0 + Git commit: + + Server: + Version: v1.8.0 + ``` + +## Notes +### Default backup storage location +We have deprecated the way to indicate the default backup storage location. Previously, that was indicated according to the backup storage location name set on the velero server-side via the flag `velero server --default-backup-storage-location`. Now we configure the default backup storage location on the velero client-side. Please refer to the [About locations][9] on how to indicate which backup storage location is the default one. + +After upgrading, if there is a previously created backup storage location with the name that matches what was defined on the server side as the default, it will be automatically set as the `default`. + +[0]: basic-install.md#install-the-cli +[1]: https://velero.io/docs/v1.1.0/upgrade-to-1.1/ +[2]: https://velero.io/docs/v1.2.0/upgrade-to-1.2/ +[3]: https://velero.io/docs/v1.3.2/upgrade-to-1.3/ +[4]: https://velero.io/docs/v1.4/upgrade-to-1.4/ +[5]: https://velero.io/docs/v1.5/upgrade-to-1.5 +[6]: https://velero.io/docs/v1.6/upgrade-to-1.6 +[7]: https://velero.io/docs/v1.7/upgrade-to-1.7 +[9]: https://velero.io/docs/v1.8/locations diff --git a/site/content/docs/v1.4/basic-install.md b/site/content/docs/v1.4/basic-install.md index 58b840a9b..a553bafa9 100644 --- a/site/content/docs/v1.4/basic-install.md +++ b/site/content/docs/v1.4/basic-install.md @@ -8,7 +8,7 @@ Refer [this document](customize-installation.md) to customize your installation. ## Prerequisites -- Access to a Kubernetes cluster, v1.10 or later, with DNS and container networking enabled. +- Access to a Kubernetes cluster, v1.10-v1.21, with DNS and container networking enabled. For more information on supported Kubernetes versions, see the Velero [compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatabilty-matrix). - `kubectl` installed locally Velero uses object storage to store backups and associated artifacts. It also optionally integrates with supported block storage systems to snapshot your persistent volumes. Before beginning the installation process, you should identify the object storage provider and optional block storage provider(s) you'll be using from the list of [compatible providers][0]. diff --git a/site/content/docs/v1.4/upgrade-to-1.4.md b/site/content/docs/v1.4/upgrade-to-1.4.md index 92ddc9193..7559b1791 100644 --- a/site/content/docs/v1.4/upgrade-to-1.4.md +++ b/site/content/docs/v1.4/upgrade-to-1.4.md @@ -15,6 +15,8 @@ If you're not yet running at least Velero v1.3, see the following: - [Upgrading to v1.2][2] - [Upgrading to v1.3][3] +Before upgrading, check the [Velero compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatabilty-matrix) to make sure your version of Kubernetes is supported by the new version of Velero. + ## Instructions 1. Install the Velero v1.4 command-line interface (CLI) by following the [instructions here][0]. diff --git a/site/content/docs/v1.5/basic-install.md b/site/content/docs/v1.5/basic-install.md index 9f3349ef3..ffc374d56 100644 --- a/site/content/docs/v1.5/basic-install.md +++ b/site/content/docs/v1.5/basic-install.md @@ -8,7 +8,7 @@ Refer [this document](customize-installation.md) to customize your installation. ## Prerequisites -- Access to a Kubernetes cluster, v1.10 or later, with DNS and container networking enabled. +- Access to a Kubernetes cluster, v1.12-v1.21, with DNS and container networking enabled. For more information on supported Kubernetes versions, see the Velero [compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatabilty-matrix). - `kubectl` installed locally Velero uses object storage to store backups and associated artifacts. It also optionally integrates with supported block storage systems to snapshot your persistent volumes. Before beginning the installation process, you should identify the object storage provider and optional block storage provider(s) you'll be using from the list of [compatible providers][0]. diff --git a/site/content/docs/v1.5/contributions/tencent-config.md b/site/content/docs/v1.5/contributions/tencent-config.md index f8ff6969d..b50b08556 100644 --- a/site/content/docs/v1.5/contributions/tencent-config.md +++ b/site/content/docs/v1.5/contributions/tencent-config.md @@ -11,7 +11,7 @@ You can deploy Velero on Tencent [TKE](https://cloud.tencent.com/document/produc - Registered [Tencent Cloud Account](https://cloud.tencent.com/register). - [Tencent Cloud COS](https://console.cloud.tencent.com/cos) service, referred to as COS, has been launched -- A Kubernetes cluster has been created, cluster version v1.10 or later, and the cluster can use DNS and Internet services normally. If you need to create a TKE cluster, refer to the Tencent [create a cluster](https://cloud.tencent.com/document/product/457/32189) documentation. +- A Kubernetes cluster has been created, cluster version v1.12-v1.21, and the cluster can use DNS and Internet services normally. If you need to create a TKE cluster, refer to the Tencent [create a cluster](https://cloud.tencent.com/document/product/457/32189) documentation. ## Create a Tencent Cloud COS bucket @@ -21,7 +21,7 @@ Set access to the bucket through the object storage console, the bucket needs to ## Get bucket access credentials -Velero uses an AWS S3-compatible API to access Tencent Cloud COS storage, which requires authentication using a pair of access key IDs and key-created signatures. +Velero uses an AWS S3-compatible API to access Tencent Cloud COS storage, which requires authentication using a pair of access key IDs and key-created signatures. In the S3 API parameter, the "access_key_id" field is the access key ID and the "secret_access_key" field is the key. @@ -116,7 +116,7 @@ After deleting the MinIO resource, use your backup to restore the deleted MinIO kubectl patch backupstoragelocation default --namespace velero \ --type merge \ --patch '{"spec":{"accessMode":"ReadOnly"}}' - + ``` Modifying access to Velero's storage location is "ReadOnly," as shown in the following image: diff --git a/site/content/docs/v1.5/restic.md b/site/content/docs/v1.5/restic.md index d4d4d0b04..375994e8e 100644 --- a/site/content/docs/v1.5/restic.md +++ b/site/content/docs/v1.5/restic.md @@ -24,7 +24,7 @@ cross-volume-type data migrations. - Understand how Velero performs [backups with the restic integration](#how-backup-and-restore-work-with-restic). - [Download][3] the latest Velero release. -- Kubernetes v1.10.0 and later. Velero's restic integration requires the Kubernetes [MountPropagation feature][6], which is enabled by default in Kubernetes v1.10.0 and later. +- Kubernetes v1.12.0-v1.21. Velero's restic integration requires the Kubernetes [MountPropagation feature][6], which is enabled by default in Kubernetes v1.10.0 and later. ### Install restic diff --git a/site/content/docs/v1.5/tilt.md b/site/content/docs/v1.5/tilt.md index 1221b36b9..6b33c18a0 100644 --- a/site/content/docs/v1.5/tilt.md +++ b/site/content/docs/v1.5/tilt.md @@ -19,7 +19,7 @@ files in this directory are gitignored so you may configure your setup according ## Prerequisites 1. [Docker](https://docs.docker.com/install/) v19.03 or newer -1. A Kubernetes cluster v1.10 or greater (does not have to be Kind) +1. A Kubernetes cluster v1.12-v1.21 (does not have to be Kind) 1. [Tilt](https://docs.tilt.dev/install.html) v0.12.0 or newer 1. Clone the [Velero project](https://github.com/vmware-tanzu/velero) repository locally @@ -133,7 +133,7 @@ Here are two ways to use MinIO as the storage: In the `tilt-settings.json` file, set `"setup-minio": true`. This will configure a Kubernetes deployment containing a running instance of MinIO inside your cluster. There are [extra steps](contributions/minio/#expose-minio-outside-your-cluster-with-a-service) -necessary to expose MinIO outside the cluster. +necessary to expose MinIO outside the cluster. To access this storage, you will need to expose MinIO outside the cluster by forwarding the MinIO port to the local machine using kubectl port-forward -n svc/minio 9000. Update the BSL configuration to use that as its "public URL" by adding `publicUrl: http://localhost:9000` to the BSL config. This is necessary to do things like download a backup file. diff --git a/site/content/docs/v1.5/upgrade-to-1.5.md b/site/content/docs/v1.5/upgrade-to-1.5.md index eb4df5d7e..58c54ac9f 100644 --- a/site/content/docs/v1.5/upgrade-to-1.5.md +++ b/site/content/docs/v1.5/upgrade-to-1.5.md @@ -14,6 +14,8 @@ If you're not yet running at least Velero v1.4, see the following: - [Upgrading to v1.3][3] - [Upgrading to v1.4][4] +Before upgrading, check the [Velero compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatabilty-matrix) to make sure your version of Kubernetes is supported by the new version of Velero. + ## Instructions 1. Install the Velero v1.5 command-line interface (CLI) by following the [instructions here][0]. diff --git a/site/content/docs/v1.6/basic-install.md b/site/content/docs/v1.6/basic-install.md index eea51782f..494ffd94e 100644 --- a/site/content/docs/v1.6/basic-install.md +++ b/site/content/docs/v1.6/basic-install.md @@ -8,7 +8,7 @@ Refer [this document](customize-installation.md) to customize your installation. ## Prerequisites -- Access to a Kubernetes cluster, v1.12 or later, with DNS and container networking enabled. +- Access to a Kubernetes cluster, v1.12 or later, with DNS and container networking enabled. Note that Velero versions 1.6.0-1.6.2, only support Kubernetes versions v1.12-1.21. If you are installing on a newer version of Kubernetes, its recommended that you install a newer version of Velero. For more information on supported Kubernetes versions, see the Velero [compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatabilty-matrix). - `kubectl` installed locally Velero uses object storage to store backups and associated artifacts. It also optionally integrates with supported block storage systems to snapshot your persistent volumes. Before beginning the installation process, you should identify the object storage provider and optional block storage provider(s) you'll be using from the list of [compatible providers][0]. @@ -23,7 +23,7 @@ If you want to perform your own testing of Velero on Windows, you must deploy Ve ## Install the CLI -### Option 1: macOS - Homebrew +### Option 1: MacOS - Homebrew On macOS, you can use [Homebrew](https://brew.sh) to install the `velero` client: diff --git a/site/content/docs/v1.6/contributions/minio.md b/site/content/docs/v1.6/contributions/minio.md index ae2076380..df605d138 100644 --- a/site/content/docs/v1.6/contributions/minio.md +++ b/site/content/docs/v1.6/contributions/minio.md @@ -22,7 +22,17 @@ If you encounter issues with installing or configuring, see [Debugging Installat * Sufficient disk space to store backups in Minio. You will need sufficient disk space available to handle any backups plus at least 1GB additional. Minio will not operate if less than 1GB of free disk space is available. -## Download Velero +## Install the CLI + +### Option 1: MacOS - Homebrew + +On macOS, you can use [Homebrew](https://brew.sh) to install the `velero` client: + +```bash +brew install velero +``` + +### Option 2: GitHub release 1. Download the [latest official release's](https://github.com/vmware-tanzu/velero/releases) tarball for your client platform. @@ -40,19 +50,11 @@ of the Velero repository is under active development and is not guaranteed to be 1. Move the `velero` binary from the Velero directory to somewhere in your PATH. -### MacOS Installation - -On Mac, you can use [HomeBrew](https://brew.sh) to install the `velero` client: - -```bash -brew install velero -``` - ## Set up server -These instructions start the Velero server and a Minio instance that is accessible from within the cluster only. See [Expose Minio outside your cluster][31] for information about configuring your cluster for outside access to Minio. Outside access is required to access logs and run `velero describe` commands. +These instructions start the Velero server and a Minio instance that is accessible from within the cluster only. See [Expose Minio outside your cluster](#expose-minio-outside-your-cluster-with-a-service) for information about configuring your cluster for outside access to Minio. Outside access is required to access logs and run `velero describe` commands. -1. Create a Velero-specific credentials file (`credentials-velero`) in your local directory: +1. Create a Velero-specific credentials file (`credentials-velero`) in your Velero directory: ``` [default] @@ -83,6 +85,7 @@ These instructions start the Velero server and a Minio instance that is accessib Additionally, you can specify `--use-restic` to enable restic support, and `--wait` to wait for the deployment to be ready. + This example also assumes you have named your Minio bucket "velero". 1. Deploy the example nginx application: diff --git a/site/content/docs/v1.6/migration-case.md b/site/content/docs/v1.6/migration-case.md index c547ebbb4..9ad7b809a 100644 --- a/site/content/docs/v1.6/migration-case.md +++ b/site/content/docs/v1.6/migration-case.md @@ -53,6 +53,7 @@ If you encounter issues, make sure that Velero is running in the same namespace ## Migrating Workloads Across Different Kubernetes Versions Migration across clusters that are not running the same version of Kubernetes might be possible, but some factors need to be considered: compatibility of API groups between clusters for each custom resource, and if a Kubernetes version upgrade breaks the compatibility of core/native API groups. For more information about API group versions, please see [EnableAPIGroupVersions](enable-api-group-versions-feature.md). +**Note:** Velero doesn't support restoring into a cluster with a lower Kubernetes version than where the backup was taken. [1]: how-velero-works.md#set-a-backup-to-expire [2]: restic.md diff --git a/site/content/docs/v1.6/restic.md b/site/content/docs/v1.6/restic.md index 878219a19..a15e08bf2 100644 --- a/site/content/docs/v1.6/restic.md +++ b/site/content/docs/v1.6/restic.md @@ -10,41 +10,41 @@ the supported cloud providers’ block storage offerings (Amazon EBS Volumes, Az It also provides a plugin model that enables anyone to implement additional object and block storage backends, outside the main Velero repository. -The restic intergation was added to give you an out-of-the-box solution for backing up and restoring almost any type of Kubernetes volume. This integration is an addition to Velero's capabilities, not a replacement for existing functionality. If you're running on AWS, and taking EBS snapshots as part of your regular Velero backups, there's no need to switch to using restic. However, if you need a volume snapshot plugin for your storage platform, or if you're using EFS, AzureFile, NFS, emptyDir, -local, or any other volume type that doesn't have a native snapshot concept, restic might be for you. +Velero's Restic integration was added to give you an out-of-the-box solution for backing up and restoring almost any type of Kubernetes volume. This integration is an addition to Velero's capabilities, not a replacement for existing functionality. If you're running on AWS, and taking EBS snapshots as part of your regular Velero backups, there's no need to switch to using Restic. However, if you need a volume snapshot plugin for your storage platform, or if you're using EFS, AzureFile, NFS, emptyDir, +local, or any other volume type that doesn't have a native snapshot concept, Restic might be for you. Restic is not tied to a specific storage platform, which means that this integration also paves the way for future work to enable cross-volume-type data migrations. **NOTE:** hostPath volumes are not supported, but the [local volume type][4] is supported. -## Setup restic +## Setup Restic ### Prerequisites -- Understand how Velero performs [backups with the restic integration](#how-backup-and-restore-work-with-restic). +- Understand how Velero performs [backups with the Restic integration](#how-backup-and-restore-work-with-restic). - [Download][3] the latest Velero release. -- Kubernetes v1.12.0 and later. Velero's restic integration requires the Kubernetes [MountPropagation feature][6], which is enabled by default in Kubernetes v1.12.0 and later. +- Kubernetes v1.12.0 and later. Velero's Restic integration requires the Kubernetes [MountPropagation feature][6], which is enabled by default in Kubernetes v1.12.0 and later. -### Install restic +### Install Restic -To install restic, use the `--use-restic` flag in the `velero install` command. See the [install overview][2] for more details on other flags for the install command. +To install Restic, use the `--use-restic` flag in the `velero install` command. See the [install overview][2] for more details on other flags for the install command. ``` velero install --use-restic ``` -When using restic on a storage provider that doesn't have Velero support for snapshots, the `--use-volume-snapshots=false` flag prevents an unused `VolumeSnapshotLocation` from being created on installation. +When using Restic on a storage provider that doesn't have Velero support for snapshots, the `--use-volume-snapshots=false` flag prevents an unused `VolumeSnapshotLocation` from being created on installation. -### Configure restic DaemonSet spec +### Configure Restic DaemonSet spec -After installation, some PaaS/CaaS platforms based on Kubernetes also require modifications the restic DaemonSet spec. The steps in this section are only needed if you are installing on RancherOS, OpenShift, VMware Tanzu Kubernetes Grid Integrated Edition (formerly VMware Enterprise PKS), or Microsoft Azure. +After installation, some PaaS/CaaS platforms based on Kubernetes also require modifications the Restic DaemonSet spec. The steps in this section are only needed if you are installing on RancherOS, OpenShift, VMware Tanzu Kubernetes Grid Integrated Edition (formerly VMware Enterprise PKS), or Microsoft Azure. **RancherOS** -Update the host path for volumes in the restic DaemonSet in the Velero namespace from `/var/lib/kubelet/pods` to `/opt/rke/var/lib/kubelet/pods`. +Update the host path for volumes in the Restic DaemonSet in the Velero namespace from `/var/lib/kubelet/pods` to `/opt/rke/var/lib/kubelet/pods`. ```yaml hostPath: @@ -62,7 +62,7 @@ hostPath: **OpenShift** -To mount the correct hostpath to pods volumes, run the restic pod in `privileged` mode. +To mount the correct hostpath to pods volumes, run the Restic pod in `privileged` mode. 1. Add the `velero` ServiceAccount to the `privileged` SCC: @@ -125,7 +125,7 @@ To mount the correct hostpath to pods volumes, run the restic pod in `privileged ``` -If restic is not running in a privileged mode, it will not be able to access pods volumes within the mounted hostpath directory because of the default enforced SELinux mode configured in the host system level. You can [create a custom SCC](https://docs.openshift.com/container-platform/3.11/admin_guide/manage_scc.html) to relax the security in your cluster so that restic pods are allowed to use the hostPath volume plug-in without granting them access to the `privileged` SCC. +If Restic is not running in a privileged mode, it will not be able to access pods volumes within the mounted hostpath directory because of the default enforced SELinux mode configured in the host system level. You can [create a custom SCC](https://docs.openshift.com/container-platform/3.11/admin_guide/manage_scc.html) to relax the security in your cluster so that Restic pods are allowed to use the hostPath volume plug-in without granting them access to the `privileged` SCC. By default a userland openshift namespace will not schedule pods on all nodes in the cluster. @@ -147,7 +147,7 @@ oc create -n -f ds.yaml **VMware Tanzu Kubernetes Grid Integrated Edition (formerly VMware Enterprise PKS)** -You need to enable the `Allow Privileged` option in your plan configuration so that restic is able to mount the hostpath. +You need to enable the `Allow Privileged` option in your plan configuration so that Restic is able to mount the hostpath. The hostPath should be changed from `/var/lib/kubelet/pods` to `/var/vcap/data/kubelet/pods` @@ -172,16 +172,16 @@ kubectl patch storageclass/ \ ## To back up -Velero supports two approaches of discovering pod volumes that need to be backed up using restic: +Velero supports two approaches of discovering pod volumes that need to be backed up using Restic: -- Opt-in approach: Where every pod containing a volume to be backed up using restic must be annotated with the volume's name. -- Opt-out approach: Where all pod volumes are backed up using restic, with the ability to opt-out any volumes that should not be backed up. +- Opt-in approach: Where every pod containing a volume to be backed up using Restic must be annotated with the volume's name. +- Opt-out approach: Where all pod volumes are backed up using Restic, with the ability to opt-out any volumes that should not be backed up. The following sections provide more details on the two approaches. ### Using the opt-out approach -In this approach, Velero will back up all pod volumes using restic with the exception of: +In this approach, Velero will back up all pod volumes using Restic with the exception of: - Volumes mounting the default service account token, kubernetes secrets, and config maps - Hostpath volumes @@ -190,12 +190,12 @@ It is possible to exclude volumes from being backed up using the `backup.velero. Instructions to back up using this approach are as follows: -1. Run the following command on each pod that contains volumes that should **not** be backed up using restic +1. Run the following command on each pod that contains volumes that should **not** be backed up using Restic ```bash kubectl -n YOUR_POD_NAMESPACE annotate pod/YOUR_POD_NAME backup.velero.io/backup-volumes-excludes=YOUR_VOLUME_NAME_1,YOUR_VOLUME_NAME_2,... ``` - where the volume names are the names of the volumes in the pod sepc. + where the volume names are the names of the volumes in the pod spec. For example, in the following pod: @@ -221,7 +221,7 @@ Instructions to back up using this approach are as follows: - name: pvc2-vm claimName: pvc2 ``` - to exclude restic backup of volume `pvc1-vm`, you would run: + to exclude Restic backup of volume `pvc1-vm`, you would run: ```bash kubectl -n sample annotate pod/app1 backup.velero.io/backup-volumes-excludes=pvc1-vm @@ -248,7 +248,7 @@ Instructions to back up using this approach are as follows: ### Using opt-in pod volume backup -Velero, by default, uses this approach to discover pod volumes that need to be backed up using restic, where every pod containing a volume to be backed up using restic must be annotated with the volume's name. +Velero, by default, uses this approach to discover pod volumes that need to be backed up using Restic, where every pod containing a volume to be backed up using Restic must be annotated with the volume's name. Instructions to back up using this approach are as follows: @@ -310,7 +310,7 @@ Instructions to back up using this approach are as follows: ## To restore -Regardless of how volumes are discovered for backup using restic, the process of restoring remains the same. +Regardless of how volumes are discovered for backup using Restic, the process of restoring remains the same. 1. Restore from your Velero backup: @@ -331,20 +331,20 @@ Regardless of how volumes are discovered for backup using restic, the process of - `hostPath` volumes are not supported. [Local persistent volumes][4] are supported. - Those of you familiar with [restic][1] may know that it encrypts all of its data. Velero uses a static, -common encryption key for all restic repositories it creates. **This means that anyone who has access to your -bucket can decrypt your restic backup data**. Make sure that you limit access to the restic bucket +common encryption key for all Restic repositories it creates. **This means that anyone who has access to your +bucket can decrypt your Restic backup data**. Make sure that you limit access to the Restic bucket appropriately. - An incremental backup chain will be maintained across pod reschedules for PVCs. However, for pod volumes that are *not* PVCs, such as `emptyDir` volumes, when a pod is deleted/recreated (for example, by a ReplicaSet/Deployment), the next backup of those volumes will be full rather than incremental, because the pod volume's lifecycle is assumed to be defined by its pod. - Restic scans each file in a single thread. This means that large files (such as ones storing a database) will take a long time to scan for data deduplication, even if the actual difference is small. -- If you plan to use the Velero restic integration to backup 100GB of data or more, you may need to [customize the resource limits](/docs/main/customize-installation/#customize-resource-requests-and-limits) to make sure backups complete successfully. -- Velero's restic integration backs up data from volumes by accessing the node's filesystem, on which the pod is running. For this reason, restic integration can only backup volumes that are mounted by a pod and not directly from the PVC. +- If you plan to use Velero's Restic integration to backup 100GB of data or more, you may need to [customize the resource limits](/docs/main/customize-installation/#customize-resource-requests-and-limits) to make sure backups complete successfully. +- Velero's Restic integration backs up data from volumes by accessing the node's filesystem, on which the pod is running. For this reason, Velero's Restic integration can only backup volumes that are mounted by a pod and not directly from the PVC. For orphan PVC/PV pairs (without running pods), some Velero users overcame this limitation running a staging pod (i.e. a busybox or alpine container with an infinite sleep) to mount these PVC/PV pairs prior taking a Velero backup. ## Customize Restore Helper Container -Velero uses a helper init container when performing a restic restore. By default, the image for this container is `velero/velero-restic-restore-helper:`, +Velero uses a helper init container when performing a Restic restore. By default, the image for this container is `velero/velero-restic-restore-helper:`, where `VERSION` matches the version/tag of the main Velero image. You can customize the image that is used for this helper by creating a ConfigMap in the Velero namespace with the alternate image. @@ -410,7 +410,7 @@ Are your Velero server and daemonset pods running? kubectl get pods -n velero ``` -Does your restic repository exist, and is it ready? +Does your Restic repository exist, and is it ready? ```bash velero restic repo get @@ -446,31 +446,31 @@ kubectl -n velero logs DAEMON_POD_NAME **NOTE**: You can increase the verbosity of the pod logs by adding `--log-level=debug` as an argument to the container command in the deployment/daemonset pod template spec. -## How backup and restore work with restic +## How backup and restore work with Restic Velero has three custom resource definitions and associated controllers: - `ResticRepository` - represents/manages the lifecycle of Velero's [restic repositories][5]. Velero creates -a restic repository per namespace when the first restic backup for a namespace is requested. The controller -for this custom resource executes restic repository lifecycle commands -- `restic init`, `restic check`, +a Restic repository per namespace when the first Restic backup for a namespace is requested. The controller +for this custom resource executes Restic repository lifecycle commands -- `restic init`, `restic check`, and `restic prune`. - You can see information about your Velero restic repositories by running `velero restic repo get`. + You can see information about your Velero's Restic repositories by running `velero restic repo get`. -- `PodVolumeBackup` - represents a restic backup of a volume in a pod. The main Velero backup process creates +- `PodVolumeBackup` - represents a Restic backup of a volume in a pod. The main Velero backup process creates one or more of these when it finds an annotated pod. Each node in the cluster runs a controller for this resource (in a daemonset) that handles the `PodVolumeBackups` for pods on that node. The controller executes `restic backup` commands to backup pod volume data. -- `PodVolumeRestore` - represents a restic restore of a pod volume. The main Velero restore process creates one -or more of these when it encounters a pod that has associated restic backups. Each node in the cluster runs a +- `PodVolumeRestore` - represents a Restic restore of a pod volume. The main Velero restore process creates one +or more of these when it encounters a pod that has associated Restic backups. Each node in the cluster runs a controller for this resource (in the same daemonset as above) that handles the `PodVolumeRestores` for pods on that node. The controller executes `restic restore` commands to restore pod volume data. ### Backup -1. Based on configuration, the main Velero backup process uses the opt-in or opt-out approach to check each pod that it's backing up for the volumes to be backed up using restic. -1. When found, Velero first ensures a restic repository exists for the pod's namespace, by: +1. Based on configuration, the main Velero backup process uses the opt-in or opt-out approach to check each pod that it's backing up for the volumes to be backed up using Restic. +1. When found, Velero first ensures a Restic repository exists for the pod's namespace, by: - checking if a `ResticRepository` custom resource already exists - if not, creating a new one, and waiting for the `ResticRepository` controller to init/check it 1. Velero then creates a `PodVolumeBackup` custom resource per volume listed in the pod annotation @@ -485,14 +485,14 @@ on that node. The controller executes `restic restore` commands to restore pod v ### Restore 1. The main Velero restore process checks each existing `PodVolumeBackup` custom resource in the cluster to backup from. -1. For each `PodVolumeBackup` found, Velero first ensures a restic repository exists for the pod's namespace, by: +1. For each `PodVolumeBackup` found, Velero first ensures a Restic repository exists for the pod's namespace, by: - checking if a `ResticRepository` custom resource already exists - if not, creating a new one, and waiting for the `ResticRepository` controller to init/check it (note that in this case, the actual repository should already exist in object storage, so the Velero controller will simply check it for integrity) -1. Velero adds an init container to the pod, whose job is to wait for all restic restores for the pod to complete (more +1. Velero adds an init container to the pod, whose job is to wait for all Restic restores for the pod to complete (more on this shortly) -1. Velero creates the pod, with the added init container, by submitting it to the Kubernetes API +1. Velero creates the pod, with the added init container, by submitting it to the Kubernetes API. Then, the Kubernetes scheduler schedules this pod to a worker node, and the pod must be in a running state. If the pod fails to start for some reason (i.e. lack of cluster resources), the Restic restore will not be done. 1. Velero creates a `PodVolumeRestore` custom resource for each volume to be restored in the pod 1. The main Velero process now waits for each `PodVolumeRestore` resource to complete or fail 1. Meanwhile, each `PodVolumeRestore` is handled by the controller on the appropriate node, which: @@ -512,7 +512,7 @@ on to running other init containers/the main containers. ### Monitor backup annotation -Velero does not provide a mechanism to detect persistent volume claims that are missing the restic backup annotation. +Velero does not provide a mechanism to detect persistent volume claims that are missing the Restic backup annotation. To solve this, a controller was written by Thomann Bits&Beats: [velero-pvc-watcher][7] diff --git a/site/content/docs/v1.6/supported-providers.md b/site/content/docs/v1.6/supported-providers.md index c1c20a06a..5e75dc1e2 100644 --- a/site/content/docs/v1.6/supported-providers.md +++ b/site/content/docs/v1.6/supported-providers.md @@ -14,8 +14,8 @@ Velero supports a variety of storage providers for different backup and snapshot | [Amazon Web Services (AWS)](https://aws.amazon.com) | AWS S3 | AWS EBS | [Velero plugin for AWS](https://github.com/vmware-tanzu/velero-plugin-for-aws) | [AWS Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-aws#setup) | | [Google Cloud Platform (GCP)](https://cloud.google.com) | Google Cloud Storage| Google Compute Engine Disks | [Velero plugin for GCP](https://github.com/vmware-tanzu/velero-plugin-for-gcp) | [GCP Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-gcp#setup) | | [Microsoft Azure](https://azure.com) | Azure Blob Storage | Azure Managed Disks | [Velero plugin for Microsoft Azure](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure) | [Azure Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#setup) | -| [VMware vSphere](https://github.com/vmware-tanzu/velero-plugin-for-vsphere) | 🚫 | vSphere Volumes | [VMware vSphere](https://github.com/vmware-tanzu/velero-plugin-for-vsphere) | [vSphere Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-vsphere#velero-plugin-for-vsphere-installation-and-configuration-details) | -| [Container Storage Interface (CSI)](https://github.com/vmware-tanzu/velero-plugin-for-csi/)| 🚫 | CSI Volumes | [Velero plugin for CSI](https://github.com/vmware-tanzu/velero-plugin-for-csi/) | [CSI Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-csi#kinds-of-plugins-included) | +| [VMware vSphere](https://www.vmware.com/ca/products/vsphere.html) | 🚫 | vSphere Volumes | [VMware vSphere](https://github.com/vmware-tanzu/velero-plugin-for-vsphere) | [vSphere Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-vsphere#velero-plugin-for-vsphere-installation-and-configuration-details) | +| [Container Storage Interface (CSI)](https://kubernetes.io/blog/2019/01/15/container-storage-interface-ga/)| 🚫 | CSI Volumes | [Velero plugin for CSI](https://github.com/vmware-tanzu/velero-plugin-for-csi/) | [CSI Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-csi#kinds-of-plugins-included) | {{< /table >}} Contact: [#Velero Slack](https://kubernetes.slack.com/messages/velero), [GitHub Issues](https://github.com/vmware-tanzu/velero/issues) diff --git a/site/content/docs/v1.6/upgrade-to-1.6.md b/site/content/docs/v1.6/upgrade-to-1.6.md index deeb97116..84da0fccc 100644 --- a/site/content/docs/v1.6/upgrade-to-1.6.md +++ b/site/content/docs/v1.6/upgrade-to-1.6.md @@ -15,6 +15,8 @@ If you're not yet running at least Velero v1.5, see the following: - [Upgrading to v1.4][4] - [Upgrading to v1.5][5] +Before upgrading, check the [Velero compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatabilty-matrix) to make sure your version of Kubernetes is supported by the new version of Velero. + ## Instructions 1. Install the Velero v1.6 command-line interface (CLI) by following the [instructions here][0]. @@ -73,7 +75,7 @@ If you're not yet running at least Velero v1.5, see the following: ## Notes ### Default backup storage location -We have deprecated the way to indicate the default backup storage location. Previously, that was indicated according to the backup storage location name set on the velero server-side via the flag `velero server --default-backup-storage-location`. Now we configure the default backup storage location on the velero client-side. Please refer to the [About locations][9] on how to indicate which backup storage location is the default one. +We have deprecated the way to indicate the default backup storage location. Previously, that was indicated according to the backup storage location name set on the velero server-side via the flag `velero server --default-backup-storage-location`. Now we configure the default backup storage location on the velero client-side. Please refer to the [About locations][9] on how to indicate which backup storage location is the default one. After upgrading, if there is a previously created backup storage location with the name that matches what was defined on the server side as the default, it will be automatically set as the `default`. diff --git a/site/content/docs/v1.7/_index.md b/site/content/docs/v1.7/_index.md new file mode 100644 index 000000000..2fdbc8f9a --- /dev/null +++ b/site/content/docs/v1.7/_index.md @@ -0,0 +1,58 @@ +--- +toc: "false" +cascade: + version: v1.7 + toc: "true" +--- +![100] + +[![Build Status][1]][2] + +## Overview + +Velero (formerly Heptio Ark) gives you tools to back up and restore your Kubernetes cluster resources and persistent volumes. You can run Velero with a cloud provider or on-premises. Velero lets you: + +* Take backups of your cluster and restore in case of loss. +* Migrate cluster resources to other clusters. +* Replicate your production cluster to development and testing clusters. + +Velero consists of: + +* A server that runs on your cluster +* A command-line client that runs locally + +## Documentation + +This site is our documentation home with installation instructions, plus information about customizing Velero for your needs, architecture, extending Velero, contributing to Velero and more. + +Please use the version selector at the top of the site to ensure you are using the appropriate documentation for your version of Velero. + +## Troubleshooting + +If you encounter issues, review the [troubleshooting docs][30], [file an issue][4], or talk to us on the [#velero channel][25] on the Kubernetes Slack server. + +## Contributing + +If you are ready to jump in and test, add code, or help with documentation, follow the instructions on our [Start contributing](https://velero.io/docs/v1.7/start-contributing/) documentation for guidance on how to setup Velero for development. + +## Changelog + +See [the list of releases][6] to find out about feature changes. + +[1]: https://github.com/vmware-tanzu/velero/workflows/Main%20CI/badge.svg +[2]: https://github.com/vmware-tanzu/velero/actions?query=workflow%3A"Main+CI" + +[4]: https://github.com/vmware-tanzu/velero/issues +[6]: https://github.com/vmware-tanzu/velero/releases + +[9]: https://kubernetes.io/docs/setup/ +[10]: https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-with-homebrew-on-macos +[11]: https://kubernetes.io/docs/tasks/tools/install-kubectl/#tabset-1 +[12]: https://github.com/kubernetes/kubernetes/blob/main/cluster/addons/dns/README.md +[14]: https://github.com/kubernetes/kubernetes +[24]: https://groups.google.com/forum/#!forum/projectvelero +[25]: https://kubernetes.slack.com/messages/velero + +[30]: troubleshooting.md + +[100]: img/velero.png diff --git a/site/content/docs/v1.7/api-types/README.md b/site/content/docs/v1.7/api-types/README.md new file mode 100644 index 000000000..54c23544d --- /dev/null +++ b/site/content/docs/v1.7/api-types/README.md @@ -0,0 +1,21 @@ +--- +title: "Table of Contents" +layout: docs +--- + +## API types + +Here we list the API types that have some functionality that you can only configure via json/yaml vs the `velero` cli +(hooks) + +* [Backup][1] +* [Restore][2] +* [Schedule][3] +* [BackupStorageLocation][4] +* [VolumeSnapshotLocation][5] + +[1]: backup.md +[2]: restore.md +[3]: schedule.md +[4]: backupstoragelocation.md +[5]: volumesnapshotlocation.md diff --git a/site/content/docs/v1.7/api-types/_index.md b/site/content/docs/v1.7/api-types/_index.md new file mode 100644 index 000000000..e60832500 --- /dev/null +++ b/site/content/docs/v1.7/api-types/_index.md @@ -0,0 +1,19 @@ +--- +layout: docs +title: API types +--- + +Here's a list the API types that have some functionality that you can only configure via json/yaml vs the `velero` cli +(hooks) + +* [Backup][1] +* [Restore][2] +* [Schedule][3] +* [BackupStorageLocation][4] +* [VolumeSnapshotLocation][5] + +[1]: backup.md +[2]: restore.md +[3]: schedule.md +[4]: backupstoragelocation.md +[5]: volumesnapshotlocation.md diff --git a/site/content/docs/v1.7/api-types/backup.md b/site/content/docs/v1.7/api-types/backup.md new file mode 100644 index 000000000..5620c9ff6 --- /dev/null +++ b/site/content/docs/v1.7/api-types/backup.md @@ -0,0 +1,148 @@ +--- +title: "Backup API Type" +layout: docs +--- + +## Use + +Use the `Backup` API type to request the Velero server to perform a backup. Once created, the +Velero Server immediately starts the backup process. + +## API GroupVersion + +Backup belongs to the API group version `velero.io/v1`. + +## Definition + +Here is a sample `Backup` object with each of the fields documented: + +```yaml +# Standard Kubernetes API Version declaration. Required. +apiVersion: velero.io/v1 +# Standard Kubernetes Kind declaration. Required. +kind: Backup +# Standard Kubernetes metadata. Required. +metadata: + # Backup name. May be any valid Kubernetes object name. Required. + name: a + # Backup namespace. Must be the namespace of the Velero server. Required. + namespace: velero +# Parameters about the backup. Required. +spec: + # Array of namespaces to include in the backup. If unspecified, all namespaces are included. + # Optional. + includedNamespaces: + - '*' + # Array of namespaces to exclude from the backup. Optional. + excludedNamespaces: + - some-namespace + # Array of resources to include in the backup. Resources may be shortcuts (for example 'po' for 'pods') + # or fully-qualified. If unspecified, all resources are included. Optional. + includedResources: + - '*' + # Array of resources to exclude from the backup. Resources may be shortcuts (for example 'po' for 'pods') + # or fully-qualified. Optional. + excludedResources: + - storageclasses.storage.k8s.io + # Whether or not to include cluster-scoped resources. Valid values are true, false, and + # null/unset. If true, all cluster-scoped resources are included (subject to included/excluded + # resources and the label selector). If false, no cluster-scoped resources are included. If unset, + # all cluster-scoped resources are included if and only if all namespaces are included and there are + # no excluded namespaces. Otherwise, if there is at least one namespace specified in either + # includedNamespaces or excludedNamespaces, then the only cluster-scoped resources that are backed + # up are those associated with namespace-scoped resources included in the backup. For example, if a + # PersistentVolumeClaim is included in the backup, its associated PersistentVolume (which is + # cluster-scoped) would also be backed up. + includeClusterResources: null + # Individual objects must match this label selector to be included in the backup. Optional. + labelSelector: + matchLabels: + app: velero + component: server + # Whether or not to snapshot volumes. This only applies to PersistentVolumes for Azure, GCE, and + # AWS. Valid values are true, false, and null/unset. If unset, Velero performs snapshots as long as + # a persistent volume provider is configured for Velero. + snapshotVolumes: null + # Where to store the tarball and logs. + storageLocation: aws-primary + # The list of locations in which to store volume snapshots created for this backup. + volumeSnapshotLocations: + - aws-primary + - gcp-primary + # The amount of time before this backup is eligible for garbage collection. If not specified, + # a default value of 30 days will be used. The default can be configured on the velero server + # by passing the flag --default-backup-ttl. + ttl: 24h0m0s + # Whether restic should be used to take a backup of all pod volumes by default. + defaultVolumesToRestic: true + # Actions to perform at different times during a backup. The only hook supported is + # executing a command in a container in a pod using the pod exec API. Optional. + hooks: + # Array of hooks that are applicable to specific resources. Optional. + resources: + - + # Name of the hook. Will be displayed in backup log. + name: my-hook + # Array of namespaces to which this hook applies. If unspecified, the hook applies to all + # namespaces. Optional. + includedNamespaces: + - '*' + # Array of namespaces to which this hook does not apply. Optional. + excludedNamespaces: + - some-namespace + # Array of resources to which this hook applies. The only resource supported at this time is + # pods. + includedResources: + - pods + # Array of resources to which this hook does not apply. Optional. + excludedResources: [] + # This hook only applies to objects matching this label selector. Optional. + labelSelector: + matchLabels: + app: velero + component: server + # An array of hooks to run before executing custom actions. Only "exec" hooks are supported. + pre: + - + # The type of hook. This must be "exec". + exec: + # The name of the container where the command will be executed. If unspecified, the + # first container in the pod will be used. Optional. + container: my-container + # The command to execute, specified as an array. Required. + command: + - /bin/uname + - -a + # How to handle an error executing the command. Valid values are Fail and Continue. + # Defaults to Fail. Optional. + onError: Fail + # How long to wait for the command to finish executing. Defaults to 30 seconds. Optional. + timeout: 10s + # An array of hooks to run after all custom actions and additional items have been + # processed. Only "exec" hooks are supported. + post: + # Same content as pre above. +# Status about the Backup. Users should not set any data here. +status: + # The version of this Backup. The only version supported is 1. + version: 1 + # The date and time when the Backup is eligible for garbage collection. + expiration: null + # The current phase. Valid values are New, FailedValidation, InProgress, Completed, PartiallyFailed, Failed. + phase: "" + # An array of any validation errors encountered. + validationErrors: null + # Date/time when the backup started being processed. + startTimestamp: 2019-04-29T15:58:43Z + # Date/time when the backup finished being processed. + completionTimestamp: 2019-04-29T15:58:56Z + # Number of volume snapshots that Velero tried to create for this backup. + volumeSnapshotsAttempted: 2 + # Number of volume snapshots that Velero successfully created for this backup. + volumeSnapshotsCompleted: 1 + # Number of warnings that were logged by the backup. + warnings: 2 + # Number of errors that were logged by the backup. + errors: 0 + +``` diff --git a/site/content/docs/v1.7/api-types/backupstoragelocation.md b/site/content/docs/v1.7/api-types/backupstoragelocation.md new file mode 100644 index 000000000..b6c58ece7 --- /dev/null +++ b/site/content/docs/v1.7/api-types/backupstoragelocation.md @@ -0,0 +1,54 @@ +--- +title: "Velero Backup Storage Locations" +layout: docs +--- + +## Backup Storage Location + +Velero can store backups in a number of locations. These are represented in the cluster via the `BackupStorageLocation` CRD. + +Velero must have at least one `BackupStorageLocation`. By default, this is expected to be named `default`, however the name can be changed by specifying `--default-backup-storage-location` on `velero server`. Backups that do not explicitly specify a storage location will be saved to this `BackupStorageLocation`. + +A sample YAML `BackupStorageLocation` looks like the following: + +```yaml +apiVersion: velero.io/v1 +kind: BackupStorageLocation +metadata: + name: default + namespace: velero +spec: + backupSyncPeriod: 2m0s + provider: aws + objectStorage: + bucket: myBucket + credential: + name: secret-name + key: key-in-secret + config: + region: us-west-2 + profile: "default" +``` + +### Parameter Reference + +The configurable parameters are as follows: + +#### Main config parameters + +{{< table caption="Main config parameters" >}} +| Key | Type | Default | Meaning | +| --- | --- | --- | --- | +| `provider` | String | Required Field | The name for whichever object storage provider will be used to store the backups. See [your object storage provider's plugin documentation](../supported-providers) for the appropriate value to use. | +| `objectStorage` | ObjectStorageLocation | Required Field | Specification of the object storage for the given provider. | +| `objectStorage/bucket` | String | Required Field | The storage bucket where backups are to be uploaded. | +| `objectStorage/prefix` | String | Optional Field | The directory inside a storage bucket where backups are to be uploaded. | +| `objectStorage/caCert` | String | Optional Field | A base64 encoded CA bundle to be used when verifying TLS connections | +| `config` | map[string]string | None (Optional) | Provider-specific configuration keys/values to be passed to the object store plugin. See [your object storage provider's plugin documentation](../supported-providers) for details. | +| `accessMode` | String | `ReadWrite` | How Velero can access the backup storage location. Valid values are `ReadWrite`, `ReadOnly`. | +| `backupSyncPeriod` | metav1.Duration | Optional Field | How frequently Velero should synchronize backups in object storage. Default is Velero's server backup sync period. Set this to `0s` to disable sync. | +| `validationFrequency` | metav1.Duration | Optional Field | How frequently Velero should validate the object storage . Default is Velero's server validation frequency. Set this to `0s` to disable validation. Default 1 minute. | +| `credential` | [corev1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#secretkeyselector-v1-core) | Optional Field | The credential information to be used with this location. | +| `credential/name` | String | Optional Field | The name of the secret within the Velero namespace which contains the credential information. | +| `credential/key` | String | Optional Field | The key to use within the secret. | +{{< /table >}} diff --git a/site/content/docs/v1.7/api-types/restore.md b/site/content/docs/v1.7/api-types/restore.md new file mode 100644 index 000000000..817e549bc --- /dev/null +++ b/site/content/docs/v1.7/api-types/restore.md @@ -0,0 +1,168 @@ +--- +title: "Restore API Type" +layout: docs +--- + +## Use + +The `Restore` API type is used as a request for the Velero server to perform a Restore. Once created, the +Velero Server immediately starts the Restore process. + +## API GroupVersion + +Restore belongs to the API group version `velero.io/v1`. + +## Definition + +Here is a sample `Restore` object with each of the fields documented: + +```yaml +# Standard Kubernetes API Version declaration. Required. +apiVersion: velero.io/v1 +# Standard Kubernetes Kind declaration. Required. +kind: Restore +# Standard Kubernetes metadata. Required. +metadata: + # Restore name. May be any valid Kubernetes object name. Required. + name: a-very-special-backup-0000111122223333 + # Restore namespace. Must be the namespace of the Velero server. Required. + namespace: velero +# Parameters about the restore. Required. +spec: + # BackupName is the unique name of the Velero backup to restore from. + backupName: a-very-special-backup + # Array of namespaces to include in the restore. If unspecified, all namespaces are included. + # Optional. + includedNamespaces: + - '*' + # Array of namespaces to exclude from the restore. Optional. + excludedNamespaces: + - some-namespace + # Array of resources to include in the restore. Resources may be shortcuts (for example 'po' for 'pods') + # or fully-qualified. If unspecified, all resources are included. Optional. + includedResources: + - '*' + # Array of resources to exclude from the restore. Resources may be shortcuts (for example 'po' for 'pods') + # or fully-qualified. Optional. + excludedResources: + - storageclasses.storage.k8s.io + # Whether or not to include cluster-scoped resources. Valid values are true, false, and + # null/unset. If true, all cluster-scoped resources are included (subject to included/excluded + # resources and the label selector). If false, no cluster-scoped resources are included. If unset, + # all cluster-scoped resources are included if and only if all namespaces are included and there are + # no excluded namespaces. Otherwise, if there is at least one namespace specified in either + # includedNamespaces or excludedNamespaces, then the only cluster-scoped resources that are backed + # up are those associated with namespace-scoped resources included in the restore. For example, if a + # PersistentVolumeClaim is included in the restore, its associated PersistentVolume (which is + # cluster-scoped) would also be backed up. + includeClusterResources: null + # Individual objects must match this label selector to be included in the restore. Optional. + labelSelector: + matchLabels: + app: velero + component: server + # NamespaceMapping is a map of source namespace names to + # target namespace names to restore into. Any source namespaces not + # included in the map will be restored into namespaces of the same name. + namespaceMapping: + namespace-backup-from: namespace-to-restore-to + # RestorePVs specifies whether to restore all included PVs + # from snapshot (via the cloudprovider). + restorePVs: true + # ScheduleName is the unique name of the Velero schedule + # to restore from. If specified, and BackupName is empty, Velero will + # restore from the most recent successful backup created from this schedule. + scheduleName: my-scheduled-backup-name + # Actions to perform during or post restore. The only hooks currently supported are + # adding an init container to a pod before it can be restored and executing a command in a + # restored pod's container. Optional. + hooks: + # Array of hooks that are applicable to specific resources. Optional. + resources: + # Name is the name of this hook. + - name: restore-hook-1 + # Array of namespaces to which this hook applies. If unspecified, the hook applies to all + # namespaces. Optional. + includedNamespaces: + - ns1 + # Array of namespaces to which this hook does not apply. Optional. + excludedNamespaces: + - ns3 + # Array of resources to which this hook applies. The only resource supported at this time is + # pods. + includedResources: + - pods + # Array of resources to which this hook does not apply. Optional. + excludedResources: [] + # This hook only applies to objects matching this label selector. Optional. + labelSelector: + matchLabels: + app: velero + component: server + # An array of hooks to run during or after restores. Currently only "init" and "exec" hooks + # are supported. + postHooks: + # The type of the hook. This must be "init" or "exec". + - init: + # An array of container specs to be added as init containers to pods to which this hook applies to. + initContainers: + - name: restore-hook-init1 + image: alpine:latest + # Mounting volumes from the podSpec to which this hooks applies to. + volumeMounts: + - mountPath: /restores/pvc1-vm + # Volume name from the podSpec + name: pvc1-vm + command: + - /bin/ash + - -c + - echo -n "FOOBARBAZ" >> /restores/pvc1-vm/foobarbaz + - name: restore-hook-init2 + image: alpine:latest + # Mounting volumes from the podSpec to which this hooks applies to. + volumeMounts: + - mountPath: /restores/pvc2-vm + # Volume name from the podSpec + name: pvc2-vm + command: + - /bin/ash + - -c + - echo -n "DEADFEED" >> /restores/pvc2-vm/deadfeed + - exec: + # The container name where the hook will be executed. Defaults to the first container. + # Optional. + container: foo + # The command that will be executed in the container. Required. + command: + - /bin/bash + - -c + - "psql < /backup/backup.sql" + # How long to wait for a container to become ready. This should be long enough for the + # container to start plus any preceding hooks in the same container to complete. The wait + # timeout begins when the container is restored and may require time for the image to pull + # and volumes to mount. If not set the restore will wait indefinitely. Optional. + waitTimeout: 5m + # How long to wait once execution begins. Defaults to 30 seconds. Optional. + execTimeout: 1m + # How to handle execution failures. Valid values are `Fail` and `Continue`. Defaults to + # `Continue`. With `Continue` mode, execution failures are logged only. With `Fail` mode, + # no more restore hooks will be executed in any container in any pod and the status of the + # Restore will be `PartiallyFailed`. Optional. + onError: Continue +# RestoreStatus captures the current status of a Velero restore. Users should not set any data here. +status: + # The current phase. Valid values are New, FailedValidation, InProgress, Completed, PartiallyFailed, Failed. + phase: "" + # An array of any validation errors encountered. + validationErrors: null + # Number of warnings that were logged by the restore. + warnings: 2 + # Errors is a count of all error messages that were generated + # during execution of the restore. The actual errors are stored in object + # storage. + errors: 0 + # FailureReason is an error that caused the entire restore + # to fail. + failureReason: + +``` diff --git a/site/content/docs/v1.7/api-types/schedule.md b/site/content/docs/v1.7/api-types/schedule.md new file mode 100644 index 000000000..3c98bef16 --- /dev/null +++ b/site/content/docs/v1.7/api-types/schedule.md @@ -0,0 +1,140 @@ +--- +title: "Schedule API Type" +layout: docs +--- + +## Use + +The `Schedule` API type is used as a repeatable request for the Velero server to perform a backup for a given cron notation. Once created, the +Velero Server will start the backup process. It will then wait for the next valid point of the given cron expression and execute the backup +process on a repeating basis. + +## API GroupVersion + +Schedule belongs to the API group version `velero.io/v1`. + +## Definition + +Here is a sample `Schedule` object with each of the fields documented: + +```yaml +# Standard Kubernetes API Version declaration. Required. +apiVersion: velero.io/v1 +# Standard Kubernetes Kind declaration. Required. +kind: Schedule +# Standard Kubernetes metadata. Required. +metadata: + # Schedule name. May be any valid Kubernetes object name. Required. + name: a + # Schedule namespace. Must be the namespace of the Velero server. Required. + namespace: velero +# Parameters about the scheduled backup. Required. +spec: + # Schedule is a Cron expression defining when to run the Backup + schedule: 0 7 * * * + # Template is the spec that should be used for each backup triggered by this schedule. + template: + # Array of namespaces to include in the scheduled backup. If unspecified, all namespaces are included. + # Optional. + includedNamespaces: + - '*' + # Array of namespaces to exclude from the scheduled backup. Optional. + excludedNamespaces: + - some-namespace + # Array of resources to include in the scheduled backup. Resources may be shortcuts (for example 'po' for 'pods') + # or fully-qualified. If unspecified, all resources are included. Optional. + includedResources: + - '*' + # Array of resources to exclude from the scheduled backup. Resources may be shortcuts (for example 'po' for 'pods') + # or fully-qualified. Optional. + excludedResources: + - storageclasses.storage.k8s.io + # Whether or not to include cluster-scoped resources. Valid values are true, false, and + # null/unset. If true, all cluster-scoped resources are included (subject to included/excluded + # resources and the label selector). If false, no cluster-scoped resources are included. If unset, + # all cluster-scoped resources are included if and only if all namespaces are included and there are + # no excluded namespaces. Otherwise, if there is at least one namespace specified in either + # includedNamespaces or excludedNamespaces, then the only cluster-scoped resources that are backed + # up are those associated with namespace-scoped resources included in the scheduled backup. For example, if a + # PersistentVolumeClaim is included in the backup, its associated PersistentVolume (which is + # cluster-scoped) would also be backed up. + includeClusterResources: null + # Individual objects must match this label selector to be included in the scheduled backup. Optional. + labelSelector: + matchLabels: + app: velero + component: server + # Whether or not to snapshot volumes. This only applies to PersistentVolumes for Azure, GCE, and + # AWS. Valid values are true, false, and null/unset. If unset, Velero performs snapshots as long as + # a persistent volume provider is configured for Velero. + snapshotVolumes: null + # Where to store the tarball and logs. + storageLocation: aws-primary + # The list of locations in which to store volume snapshots created for backups under this schedule. + volumeSnapshotLocations: + - aws-primary + - gcp-primary + # The amount of time before backups created on this schedule are eligible for garbage collection. If not specified, + # a default value of 30 days will be used. The default can be configured on the velero server + # by passing the flag --default-backup-ttl. + ttl: 24h0m0s + # The labels you want on backup objects, created from this schedule (instead of copying the labels you have on schedule object itself). + # When this field is set, the labels from the Schedule resource are not copied to the Backup resource. + metadata: + labels: + labelname: somelabelvalue + # Actions to perform at different times during a backup. The only hook supported is + # executing a command in a container in a pod using the pod exec API. Optional. + hooks: + # Array of hooks that are applicable to specific resources. Optional. + resources: + - + # Name of the hook. Will be displayed in backup log. + name: my-hook + # Array of namespaces to which this hook applies. If unspecified, the hook applies to all + # namespaces. Optional. + includedNamespaces: + - '*' + # Array of namespaces to which this hook does not apply. Optional. + excludedNamespaces: + - some-namespace + # Array of resources to which this hook applies. The only resource supported at this time is + # pods. + includedResources: + - pods + # Array of resources to which this hook does not apply. Optional. + excludedResources: [] + # This hook only applies to objects matching this label selector. Optional. + labelSelector: + matchLabels: + app: velero + component: server + # An array of hooks to run before executing custom actions. Only "exec" hooks are supported. + pre: + - + # The type of hook. This must be "exec". + exec: + # The name of the container where the command will be executed. If unspecified, the + # first container in the pod will be used. Optional. + container: my-container + # The command to execute, specified as an array. Required. + command: + - /bin/uname + - -a + # How to handle an error executing the command. Valid values are Fail and Continue. + # Defaults to Fail. Optional. + onError: Fail + # How long to wait for the command to finish executing. Defaults to 30 seconds. Optional. + timeout: 10s + # An array of hooks to run after all custom actions and additional items have been + # processed. Only "exec" hooks are supported. + post: + # Same content as pre above. +status: + # The current phase of the latest scheduled backup. Valid values are New, FailedValidation, InProgress, Completed, PartiallyFailed, Failed. + phase: "" + # Date/time of the last backup for a given schedule + lastBackup: + # An array of any validation errors encountered. + validationErrors: +``` diff --git a/site/content/docs/v1.7/api-types/volumesnapshotlocation.md b/site/content/docs/v1.7/api-types/volumesnapshotlocation.md new file mode 100644 index 000000000..28ac33222 --- /dev/null +++ b/site/content/docs/v1.7/api-types/volumesnapshotlocation.md @@ -0,0 +1,40 @@ +--- +title: "Velero Volume Snapshot Location" +layout: docs +--- + +## Volume Snapshot Location + +A volume snapshot location is the location in which to store the volume snapshots created for a backup. + +Velero can be configured to take snapshots of volumes from multiple providers. Velero also allows you to configure multiple possible `VolumeSnapshotLocation` per provider, although you can only select one location per provider at backup time. + +Each VolumeSnapshotLocation describes a provider + location. These are represented in the cluster via the `VolumeSnapshotLocation` CRD. Velero must have at least one `VolumeSnapshotLocation` per cloud provider. + +A sample YAML `VolumeSnapshotLocation` looks like the following: + +```yaml +apiVersion: velero.io/v1 +kind: VolumeSnapshotLocation +metadata: + name: aws-default + namespace: velero +spec: + provider: aws + config: + region: us-west-2 + profile: "default" +``` + +### Parameter Reference + +The configurable parameters are as follows: + +#### Main config parameters + +{{< table caption="Main config parameters" >}} +| Key | Type | Default | Meaning | +| --- | --- | --- | --- | +| `provider` | String | Required Field | The name for whichever storage provider will be used to create/store the volume snapshots. See [your volume snapshot provider's plugin documentation](../supported-providers) for the appropriate value to use. | +| `config` | map string string | None (Optional) | Provider-specific configuration keys/values to be passed to the volume snapshotter plugin. See [your volume snapshot provider's plugin documentation](../supported-providers) for details. | +{{< /table >}} diff --git a/site/content/docs/v1.7/backup-hooks.md b/site/content/docs/v1.7/backup-hooks.md new file mode 100644 index 000000000..57176fa74 --- /dev/null +++ b/site/content/docs/v1.7/backup-hooks.md @@ -0,0 +1,92 @@ +--- +title: "Backup Hooks" +layout: docs +--- + +Velero supports executing commands in containers in pods during a backup. + +## Backup Hooks + +When performing a backup, you can specify one or more commands to execute in a container in a pod +when that pod is being backed up. The commands can be configured to run *before* any custom action +processing ("pre" hooks), or after all custom actions have been completed and any additional items +specified by custom action have been backed up ("post" hooks). Note that hooks are _not_ executed within a shell +on the containers. + +There are two ways to specify hooks: annotations on the pod itself, and in the Backup spec. + +### Specifying Hooks As Pod Annotations + +You can use the following annotations on a pod to make Velero execute a hook when backing up the pod: + +#### Pre hooks + +* `pre.hook.backup.velero.io/container` + * The container where the command should be executed. Defaults to the first container in the pod. Optional. +* `pre.hook.backup.velero.io/command` + * The command to execute. If you need multiple arguments, specify the command as a JSON array, such as `["/usr/bin/uname", "-a"]` +* `pre.hook.backup.velero.io/on-error` + * What to do if the command returns a non-zero exit code. Defaults to Fail. Valid values are Fail and Continue. Optional. +* `pre.hook.backup.velero.io/timeout` + * How long to wait for the command to execute. The hook is considered in error if the command exceeds the timeout. Defaults to 30s. Optional. + + +#### Post hooks + +* `post.hook.backup.velero.io/container` + * The container where the command should be executed. Defaults to the first container in the pod. Optional. +* `post.hook.backup.velero.io/command` + * The command to execute. If you need multiple arguments, specify the command as a JSON array, such as `["/usr/bin/uname", "-a"]` +* `post.hook.backup.velero.io/on-error` + * What to do if the command returns a non-zero exit code. Defaults to Fail. Valid values are Fail and Continue. Optional. +* `post.hook.backup.velero.io/timeout` + * How long to wait for the command to execute. The hook is considered in error if the command exceeds the timeout. Defaults to 30s. Optional. + +### Specifying Hooks in the Backup Spec + +Please see the documentation on the [Backup API Type][1] for how to specify hooks in the Backup +spec. + +## Hook Example with fsfreeze + +This examples walks you through using both pre and post hooks for freezing a file system. Freezing the +file system is useful to ensure that all pending disk I/O operations have completed prior to taking a snapshot. + +This example uses [examples/nginx-app/with-pv.yaml][2]. Follow the [steps for your provider][3] to +setup this example. + +### Annotations + +The Velero [example/nginx-app/with-pv.yaml][2] serves as an example of adding the pre and post hook annotations directly +to your declarative deployment. Below is an example of what updating an object in place might look like. + +```shell +kubectl annotate pod -n nginx-example -l app=nginx \ + pre.hook.backup.velero.io/command='["/sbin/fsfreeze", "--freeze", "/var/log/nginx"]' \ + pre.hook.backup.velero.io/container=fsfreeze \ + post.hook.backup.velero.io/command='["/sbin/fsfreeze", "--unfreeze", "/var/log/nginx"]' \ + post.hook.backup.velero.io/container=fsfreeze +``` + +Now test the pre and post hooks by creating a backup. You can use the Velero logs to verify that the pre and post +hooks are running and exiting without error. + +```shell +velero backup create nginx-hook-test + +velero backup get nginx-hook-test +velero backup logs nginx-hook-test | grep hookCommand +``` + +## Using Multiple Commands + +To use multiple commands, wrap your target command in a shell and separate them with `;`, `&&`, or other shell conditional constructs. + +```shell + pre.hook.backup.velero.io/command='["/bin/bash", "-c", "echo hello > hello.txt && echo goodbye > goodbye.txt"]' +``` + + +[1]: api-types/backup.md +[2]: https://github.com/vmware-tanzu/velero/blob/v1.7.0/examples/nginx-app/with-pv.yaml +[3]: cloud-common.md diff --git a/site/content/docs/v1.7/backup-reference.md b/site/content/docs/v1.7/backup-reference.md new file mode 100644 index 000000000..60e019188 --- /dev/null +++ b/site/content/docs/v1.7/backup-reference.md @@ -0,0 +1,70 @@ +--- +title: "Backup Reference" +layout: docs +--- + +## Exclude Specific Items from Backup + +It is possible to exclude individual items from being backed up, even if they match the resource/namespace/label selectors defined in the backup spec. To do this, label the item as follows: + +```bash +kubectl label -n / velero.io/exclude-from-backup=true +``` + +## Specify Backup Orders of Resources of Specific Kind + +To backup resources of specific Kind in a specific order, use option --ordered-resources to specify a mapping Kinds to an ordered list of specific resources of that Kind. Resource names are separated by commas and their names are in format 'namespace/resourcename'. For cluster scope resource, simply use resource name. Key-value pairs in the mapping are separated by semi-colon. Kind name is in plural form. + +```bash +velero backup create backupName --include-cluster-resources=true --ordered-resources 'pods=ns1/pod1,ns1/pod2;persistentvolumes=pv4,pv8' --include-namespaces=ns1 +velero backup create backupName --ordered-resources 'statefulsets=ns1/sts1,ns1/sts0' --include-namespaces=ns1 +``` +## Schedule a Backup + +The **schedule** operation allows you to create a backup of your data at a specified time, defined by a [Cron expression](https://en.wikipedia.org/wiki/Cron). + +``` +velero schedule create NAME --schedule="* * * * *" [flags] +``` + +Cron schedules use the following format. + +``` +# ┌───────────── minute (0 - 59) +# │ ┌───────────── hour (0 - 23) +# │ │ ┌───────────── day of the month (1 - 31) +# │ │ │ ┌───────────── month (1 - 12) +# │ │ │ │ ┌───────────── day of the week (0 - 6) (Sunday to Saturday; +# │ │ │ │ │ 7 is also Sunday on some systems) +# │ │ │ │ │ +# │ │ │ │ │ +# * * * * * +``` + +For example, the command below creates a backup that runs every day at 3am. + +``` +velero schedule create example-schedule --schedule="0 3 * * *" +``` + +This command will create the backup, `example-schedule`, within Velero, but the backup will not be taken until the next scheduled time, 3am. Backups created by a schedule are saved with the name `-`, where `` is formatted as *YYYYMMDDhhmmss*. For a full list of available configuration flags use the Velero CLI help command. + +``` +velero schedule create --help +``` + +Once you create the scheduled backup, you can then trigger it manually using the `velero backup` command. + +``` +velero backup create --from-schedule example-schedule +``` + +This command will immediately trigger a new backup based on your template for `example-schedule`. This will not affect the backup schedule, and another backup will trigger at the scheduled time. + +## Kubernetes API Pagination + +By default, Velero will paginate the LIST API call for each resource type in the Kubernetes API when collecting items into a backup. The `--client-page-size` flag for the Velero server configures the size of each page. + +Depending on the cluster's scale, tuning the page size can improve backup performance. You can experiment with higher values, noting their impact on the relevant `apiserver_request_duration_seconds_*` metrics from the Kubernetes apiserver. + +Pagination can be entirely disabled by setting `--client-page-size` to `0`. This will request all items in a single unpaginated LIST call. diff --git a/site/content/docs/v1.7/basic-install.md b/site/content/docs/v1.7/basic-install.md new file mode 100644 index 000000000..efa082119 --- /dev/null +++ b/site/content/docs/v1.7/basic-install.md @@ -0,0 +1,73 @@ +--- +title: "Basic Install" +layout: docs +--- + +Use this doc to get a basic installation of Velero. +Refer [this document](customize-installation.md) to customize your installation. + +## Prerequisites + +- Access to a Kubernetes cluster, v1.12 or later, with DNS and container networking enabled. For more information on supported Kubernetes versions, see the Velero [compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatabilty-matrix). +- `kubectl` installed locally + +Velero uses object storage to store backups and associated artifacts. It also optionally integrates with supported block storage systems to snapshot your persistent volumes. Before beginning the installation process, you should identify the object storage provider and optional block storage provider(s) you'll be using from the list of [compatible providers][0]. + +Velero supports storage providers for both cloud-provider environments and on-premises environments. For more details on on-premises scenarios, see the [on-premises documentation][2]. + +### Velero on Windows + +Velero does not officially support Windows. In testing, the Velero team was able to backup stateless Windows applications only. The restic integration and backups of stateful applications or PersistentVolumes were not supported. + +If you want to perform your own testing of Velero on Windows, you must deploy Velero as a Windows container. Velero does not provide official Windows images, but its possible for you to build your own Velero Windows container image to use. Note that you must build this image on a Windows node. + +## Install the CLI + +### Option 1: MacOS - Homebrew + +On macOS, you can use [Homebrew](https://brew.sh) to install the `velero` client: + +```bash +brew install velero +``` + +### Option 2: GitHub release + +1. Download the [latest release][1]'s tarball for your client platform. +1. Extract the tarball: + + ```bash + tar -xvf .tar.gz + ``` + +1. Move the extracted `velero` binary to somewhere in your `$PATH` (`/usr/local/bin` for most users). + +### Option 3: Windows - Chocolatey + +On Windows, you can use [Chocolatey](https://chocolatey.org/install) to install the [velero](https://chocolatey.org/packages/velero) client: + +```powershell +choco install velero +``` + +## Install and configure the server components + +There are two supported methods for installing the Velero server components: + +- the `velero install` CLI command +- the [Helm chart](https://vmware-tanzu.github.io/helm-charts/) + +Velero uses storage provider plugins to integrate with a variety of storage systems to support backup and snapshot operations. The steps to install and configure the Velero server components along with the appropriate plugins are specific to your chosen storage provider. To find installation instructions for your chosen storage provider, follow the documentation link for your provider at our [supported storage providers][0] page + +_Note: if your object storage provider is different than your volume snapshot provider, follow the installation instructions for your object storage provider first, then return here and follow the instructions to [add your volume snapshot provider][4]._ + +## Command line Autocompletion + +Please refer to [this part of the documentation][5]. + +[0]: supported-providers.md +[1]: https://github.com/vmware-tanzu/velero/releases/latest +[2]: on-premises.md +[3]: overview-plugins.md +[4]: customize-installation.md#install-an-additional-volume-snapshot-provider +[5]: customize-installation.md#optional-velero-cli-configurations diff --git a/site/content/docs/v1.7/build-from-source.md b/site/content/docs/v1.7/build-from-source.md new file mode 100644 index 000000000..df9f738cf --- /dev/null +++ b/site/content/docs/v1.7/build-from-source.md @@ -0,0 +1,200 @@ +--- +title: "Build from source" +layout: docs +--- + +## Prerequisites + +* Access to a Kubernetes cluster, version 1.7 or later. +* A DNS server on the cluster +* `kubectl` installed +* [Go][5] installed (minimum version 1.8) + +## Get the source + +### Option 1) Get latest (recommended) + +```bash +mkdir $HOME/go +export GOPATH=$HOME/go +go get github.com/vmware-tanzu/velero +``` + +Where `go` is your [import path][4] for Go. + +For Go development, it is recommended to add the Go import path (`$HOME/go` in this example) to your path. + +### Option 2) Release archive + +Download the archive named `Source code` from the [release page][22] and extract it in your Go import path as `src/github.com/vmware-tanzu/velero`. + +Note that the Makefile targets assume building from a git repository. When building from an archive, you will be limited to the `go build` commands described below. + +## Build + +There are a number of different ways to build `velero` depending on your needs. This section outlines the main possibilities. + +When building by using `make`, it will place the binaries under `_output/bin/$GOOS/$GOARCH`. For example, you will find the binary for darwin here: `_output/bin/darwin/amd64/velero`, and the binary for linux here: `_output/bin/linux/amd64/velero`. `make` will also splice version and git commit information in so that `velero version` displays proper output. + +Note: `velero install` will also use the version information to determine which tagged image to deploy. If you would like to overwrite what image gets deployed, use the `image` flag (see below for instructions on how to build images). + +### Build the binary + +To build the `velero` binary on your local machine, compiled for your OS and architecture, run one of these two commands: + +```bash +go build ./cmd/velero +``` + +```bash +make local +``` + +### Cross compiling + +To build the velero binary targeting linux/amd64 within a build container on your local machine, run: + +```bash +make build +``` + +For any specific platform, run `make build--`. + +For example, to build for the Mac, run `make build-darwin-amd64`. + +Velero's `Makefile` has a convenience target, `all-build`, that builds the following platforms: + +* linux-amd64 +* linux-arm +* linux-arm64 +* linux-ppc64le +* darwin-amd64 +* windows-amd64 + +## Making images and updating Velero + +If after installing Velero you would like to change the image used by its deployment to one that contains your code changes, you may do so by updating the image: + +```bash +kubectl -n velero set image deploy/velero velero=myimagerepo/velero:$VERSION +``` + +To build a Velero container image, you need to configure `buildx` first. + +### Buildx + +Docker Buildx is a CLI plugin that extends the docker command with the full support of the features provided by Moby BuildKit builder toolkit. It provides the same user experience as docker build with many new features like creating scoped builder instances and building against multiple nodes concurrently. + +More information in the [docker docs][23] and in the [buildx github][24] repo. + +### Image building + +Set the `$REGISTRY` environment variable. For example, if you want to build the `gcr.io/my-registry/velero:main` image, set `$REGISTRY` to `gcr.io/my-registry`. If this variable is not set, the default is `velero`. + +Optionally, set the `$VERSION` environment variable to change the image tag or `$BIN` to change which binary to build a container image for. Then, run: + +```bash +make container +``` +_Note: To build build container images for both `velero` and `velero-restic-restore-helper`, run: `make all-containers`_ + +### Publishing container images to a registry + +To publish container images to a registry, the following one time setup is necessary: + +1. If you are building cross platform container images + ```bash + $ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + ``` +1. Create and bootstrap a new docker buildx builder + ```bash + $ docker buildx create --use --name builder + builder + $ docker buildx inspect --bootstrap + [+] Building 2.6s (1/1) FINISHED + => [internal] booting buildkit 2.6s + => => pulling image moby/buildkit:buildx-stable-1 1.9s + => => creating container buildx_buildkit_builder0 0.7s + Name: builder + Driver: docker-container + + Nodes: + Name: builder0 + Endpoint: unix:///var/run/docker.sock + Status: running + Platforms: linux/amd64, linux/arm64, linux/ppc64le, linux/s390x, linux/386, linux/arm/v7, linux/arm/v6 + ``` + NOTE: Without the above setup, the output of `docker buildx inspect --bootstrap` will be: + ```bash + $ docker buildx inspect --bootstrap + Name: default + Driver: docker + + Nodes: + Name: default + Endpoint: default + Status: running + Platforms: linux/amd64, linux/arm64, linux/ppc64le, linux/s390x, linux/386, linux/arm/v7, linux/arm/v6 + ``` + And the `REGISTRY=myrepo BUILDX_OUTPUT_TYPE=registry make container` will fail with the below error: + ```bash + $ REGISTRY=ashishamarnath BUILDX_PLATFORMS=linux/arm64 BUILDX_OUTPUT_TYPE=registry make container + auto-push is currently not implemented for docker driver + make: *** [container] Error 1 + ``` + +Having completed the above one time setup, now the output of `docker buildx inspect --bootstrap` should be like + +```bash +$ docker buildx inspect --bootstrap +Name: builder +Driver: docker-container + +Nodes: +Name: builder0 +Endpoint: unix:///var/run/docker.sock +Status: running +Platforms: linux/amd64, linux/arm64, linux/riscv64, linux/ppc64le, linux/s390x, linux/386, linux/arm/v7, linux/arm/v +``` + +Now build and push the container image by running the `make container` command with `$BUILDX_OUTPUT_TYPE` set to `registry` +```bash +$ REGISTRY=myrepo BUILDX_OUTPUT_TYPE=registry make container +``` + +### Cross platform building + +Docker `buildx` platforms supported: +* `linux/amd64` +* `linux/arm64` +* `linux/arm/v7` +* `linux/ppc64le` + +For any specific platform, run `BUILDX_PLATFORMS=/ make container` + +For example, to build an image for arm64, run: + +```bash +BUILDX_PLATFORMS=linux/arm64 make container +``` +_Note: By default, `$BUILDX_PLATFORMS` is set to `linux/amd64`_ + +With `buildx`, you can also build all supported platforms at the same time and push a multi-arch image to the registry. For example: + +```bash +REGISTRY=myrepo VERSION=foo BUILDX_PLATFORMS=linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le BUILDX_OUTPUT_TYPE=registry make all-containers +``` +_Note: when building for more than 1 platform at the same time, you need to set `BUILDX_OUTPUT_TYPE` to `registry` as local multi-arch images are not supported [yet][25]._ + +Note: if you want to update the image but not change its name, you will have to trigger Kubernetes to pick up the new image. One way of doing so is by deleting the Velero deployment pod: + +```bash +kubectl -n velero delete pods -l deploy=velero +``` + +[4]: https://blog.golang.org/organizing-go-code +[5]: https://golang.org/doc/install +[22]: https://github.com/vmware-tanzu/velero/releases +[23]: https://docs.docker.com/buildx/working-with-buildx/ +[24]: https://github.com/docker/buildx +[25]: https://github.com/moby/moby/pull/38738 diff --git a/site/content/docs/v1.7/code-standards.md b/site/content/docs/v1.7/code-standards.md new file mode 100644 index 000000000..80dbbe813 --- /dev/null +++ b/site/content/docs/v1.7/code-standards.md @@ -0,0 +1,151 @@ +--- +title: "Code Standards" +layout: docs +toc: "true" +--- + +## Opening PRs + +When opening a pull request, please fill out the checklist supplied the template. This will help others properly categorize and review your pull request. + +## Adding a changelog + +Authors are expected to include a changelog file with their pull requests. The changelog file +should be a new file created in the `changelogs/unreleased` folder. The file should follow the +naming convention of `pr-username` and the contents of the file should be your text for the +changelog. + + velero/changelogs/unreleased <- folder + 000-username <- file + +Add that to the PR. + +If a PR does not warrant a changelog, the CI check for a changelog can be skipped by applying a `changelog-not-required` label on the PR. + +## Copyright header + +Whenever a source code file is being modified, the copyright notice should be updated to our standard copyright notice. That is, it should read “Copyright the Velero contributors.” + +For new files, the entire copyright and license header must be added. + +Please note that doc files do not need a copyright header. + +## Code + +- Log messages are capitalized. + +- Error messages are kept lower-cased. + +- Wrap/add a stack only to errors that are being directly returned from non-velero code, such as an API call to the Kubernetes server. + + ```bash + errors.WithStack(err) + ``` + +- Prefer to use the utilities in the Kubernetes package [`sets`](https://godoc.org/github.com/kubernetes/apimachinery/pkg/util/sets). + + ```bash + k8s.io/apimachinery/pkg/util/sets + ``` + +## Imports + +For imports, we use the following convention: + +`` + +Example: + + import ( + corev1api "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + corev1listers "k8s.io/client-go/listers/core/v1" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" + ) + +## Mocks + +We use a package to generate mocks for our interfaces. + +Example: if you want to change this mock: https://github.com/vmware-tanzu/velero/blob/v1.7.0/pkg/restic/mocks/restorer.go + +Run: + +```bash +go get github.com/vektra/mockery/.../ +cd pkg/restic +mockery -name=Restorer +``` + +Might need to run `make update` to update the imports. + +## Kubernetes Labels + +When generating label values, be sure to pass them through the `label.GetValidName()` helper function. + +This will help ensure that the values are the proper length and format to be stored and queried. + +In general, UIDs are safe to persist as label values. + +This function is not relevant to annotation values, which do not have restrictions. + +## DCO Sign off + +All authors to the project retain copyright to their work. However, to ensure +that they are only submitting work that they have rights to, we are requiring +everyone to acknowledge this by signing their work. + +Any copyright notices in this repo should specify the authors as "the Velero contributors". + +To sign your work, just add a line like this at the end of your commit message: + +``` +Signed-off-by: Joe Beda +``` + +This can easily be done with the `--signoff` option to `git commit`. + +By doing this you state that you can certify the following (from https://developercertificate.org/): + +``` +Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2004, 2006 The Linux Foundation and its contributors. +1 Letterman Drive +Suite D4700 +San Francisco, CA, 94129 + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + + +Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. +``` diff --git a/site/content/docs/v1.7/contributions/ibm-config.md b/site/content/docs/v1.7/contributions/ibm-config.md new file mode 100644 index 000000000..4aaf823a2 --- /dev/null +++ b/site/content/docs/v1.7/contributions/ibm-config.md @@ -0,0 +1,101 @@ +--- +title: "Use IBM Cloud Object Storage as Velero's storage destination." +layout: docs +--- +You can deploy Velero on IBM [Public][5] or [Private][4] clouds, or even on any other Kubernetes cluster, but anyway you can use IBM Cloud Object Store as a destination for Velero's backups. + +To set up IBM Cloud Object Storage (COS) as Velero's destination, you: + +* Download an official release of Velero +* Create your COS instance +* Create an S3 bucket +* Define a service that can store data in the bucket +* Configure and start the Velero server + +## Download Velero + +1. Download the [latest official release's](https://github.com/vmware-tanzu/velero/releases) tarball for your client platform. + + _We strongly recommend that you use an [official release](https://github.com/vmware-tanzu/velero/releases) of +Velero. The tarballs for each release contain the `velero` command-line client. The code in the main branch +of the Velero repository is under active development and is not guaranteed to be stable!_ + +1. Extract the tarball: + + ```bash + tar -xvf .tar.gz -C /dir/to/extract/to + ``` + + The directory you extracted is called the "Velero directory" in subsequent steps. + +1. Move the `velero` binary from the Velero directory to somewhere in your PATH. + +## Create COS instance +If you don’t have a COS instance, you can create a new one, according to the detailed instructions in [Creating a new resource instance][1]. + +## Create an S3 bucket +Velero requires an object storage bucket to store backups in. See instructions in [Create some buckets to store your data][2]. + +## Define a service that can store data in the bucket. +The process of creating service credentials is described in [Service credentials][3]. +Several comments: + +1. The Velero service will write its backup into the bucket, so it requires the “Writer” access role. + +2. Velero uses an AWS S3 compatible API. Which means it authenticates using a signature created from a pair of access and secret keys — a set of HMAC credentials. You can create these HMAC credentials by specifying `{“HMAC”:true}` as an optional inline parameter. See [HMAC credentials][31] guide. + +3. After successfully creating a Service credential, you can view the JSON definition of the credential. Under the `cos_hmac_keys` entry there are `access_key_id` and `secret_access_key`. Use them in the next step. + +4. Create a Velero-specific credentials file (`credentials-velero`) in your local directory: + + ``` + [default] + aws_access_key_id= + aws_secret_access_key= + ``` + + Where the access key id and secret are the values that you got above. + +## Install and start Velero + +Install Velero, including all prerequisites, into the cluster and start the deployment. This will create a namespace called `velero`, and place a deployment named `velero` in it. + +```bash +velero install \ + --provider aws \ + --bucket \ + --secret-file ./credentials-velero \ + --use-volume-snapshots=false \ + --backup-location-config region=,s3ForcePathStyle="true",s3Url= +``` + +Velero does not have a volume snapshot plugin for IBM Cloud, so creating volume snapshots is disabled. + +Additionally, you can specify `--use-restic` to enable [restic support][16], and `--wait` to wait for the deployment to be ready. + +(Optional) Specify [CPU and memory resource requests and limits][15] for the Velero/restic pods. + +Once the installation is complete, remove the default `VolumeSnapshotLocation` that was created by `velero install`, since it's specific to AWS and won't work for IBM Cloud: + +```bash +kubectl -n velero delete volumesnapshotlocation.velero.io default +``` + +For more complex installation needs, use either the Helm chart, or add `--dry-run -o yaml` options for generating the YAML representation for the installation. + +## Installing the nginx example (optional) + +If you run the nginx example, in file `examples/nginx-app/with-pv.yaml`: + +Uncomment `storageClassName: ` and replace with your `StorageClass` name. + +[0]: namespace.md +[1]: https://cloud.ibm.com/docs/cloud-object-storage/getting-started.html +[2]: https://cloud.ibm.com/docs/cloud-object-storage/getting-started.html#create-buckets +[3]: https://cloud.ibm.com/docs/cloud-object-storage/iam?topic=cloud-object-storage-service-credentials +[31]: https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-uhc-hmac-credentials-main +[4]: https://www.ibm.com/docs/en/cloud-private +[5]: https://cloud.ibm.com/docs/containers/container_index.html#container_index +[14]: http://docs.aws.amazon.com/IAM/latest/UserGuide/introduction.html +[15]: customize-installation.md#customize-resource-requests-and-limits +[16]: restic.md diff --git a/site/content/docs/v1.7/contributions/img-for-tencent/15ccaacf00640a04ae29ceed4c86195b.png b/site/content/docs/v1.7/contributions/img-for-tencent/15ccaacf00640a04ae29ceed4c86195b.png new file mode 100644 index 000000000..61859ca50 Binary files /dev/null and b/site/content/docs/v1.7/contributions/img-for-tencent/15ccaacf00640a04ae29ceed4c86195b.png differ diff --git a/site/content/docs/v1.7/contributions/img-for-tencent/1d53b0115644d43657c2a5ece805c9b4.png b/site/content/docs/v1.7/contributions/img-for-tencent/1d53b0115644d43657c2a5ece805c9b4.png new file mode 100644 index 000000000..57de0cde0 Binary files /dev/null and b/site/content/docs/v1.7/contributions/img-for-tencent/1d53b0115644d43657c2a5ece805c9b4.png differ diff --git a/site/content/docs/v1.7/contributions/img-for-tencent/69194157ccd5e377d1e7d914fd8c0336.png b/site/content/docs/v1.7/contributions/img-for-tencent/69194157ccd5e377d1e7d914fd8c0336.png new file mode 100644 index 000000000..f363eb1a7 Binary files /dev/null and b/site/content/docs/v1.7/contributions/img-for-tencent/69194157ccd5e377d1e7d914fd8c0336.png differ diff --git a/site/content/docs/v1.7/contributions/img-for-tencent/9015313121ed7987558c88081b052574.png b/site/content/docs/v1.7/contributions/img-for-tencent/9015313121ed7987558c88081b052574.png new file mode 100644 index 000000000..921b54a16 Binary files /dev/null and b/site/content/docs/v1.7/contributions/img-for-tencent/9015313121ed7987558c88081b052574.png differ diff --git a/site/content/docs/v1.7/contributions/img-for-tencent/ceaca9ce6bc92bdce987c63d2fe71561.png b/site/content/docs/v1.7/contributions/img-for-tencent/ceaca9ce6bc92bdce987c63d2fe71561.png new file mode 100644 index 000000000..36805831b Binary files /dev/null and b/site/content/docs/v1.7/contributions/img-for-tencent/ceaca9ce6bc92bdce987c63d2fe71561.png differ diff --git a/site/content/docs/v1.7/contributions/img-for-tencent/e8c2ab4e5e31d1370c62fad25059a8a8.png b/site/content/docs/v1.7/contributions/img-for-tencent/e8c2ab4e5e31d1370c62fad25059a8a8.png new file mode 100644 index 000000000..c64f47a81 Binary files /dev/null and b/site/content/docs/v1.7/contributions/img-for-tencent/e8c2ab4e5e31d1370c62fad25059a8a8.png differ diff --git a/site/content/docs/v1.7/contributions/img-for-tencent/e932223585c0b19891cc085ad7f438e1.png b/site/content/docs/v1.7/contributions/img-for-tencent/e932223585c0b19891cc085ad7f438e1.png new file mode 100644 index 000000000..188c2db52 Binary files /dev/null and b/site/content/docs/v1.7/contributions/img-for-tencent/e932223585c0b19891cc085ad7f438e1.png differ diff --git a/site/content/docs/v1.7/contributions/img-for-tencent/eb2bbabae48b188748f5278bedf177f1.png b/site/content/docs/v1.7/contributions/img-for-tencent/eb2bbabae48b188748f5278bedf177f1.png new file mode 100644 index 000000000..3eaee3553 Binary files /dev/null and b/site/content/docs/v1.7/contributions/img-for-tencent/eb2bbabae48b188748f5278bedf177f1.png differ diff --git a/site/content/docs/v1.7/contributions/img-for-tencent/effe8a0a7ce3aa8e422db00bfdddc375.png b/site/content/docs/v1.7/contributions/img-for-tencent/effe8a0a7ce3aa8e422db00bfdddc375.png new file mode 100644 index 000000000..28e1bbaad Binary files /dev/null and b/site/content/docs/v1.7/contributions/img-for-tencent/effe8a0a7ce3aa8e422db00bfdddc375.png differ diff --git a/site/content/docs/v1.7/contributions/img-for-tencent/f0fff5228527edc72d6e71a50d5dc966.png b/site/content/docs/v1.7/contributions/img-for-tencent/f0fff5228527edc72d6e71a50d5dc966.png new file mode 100644 index 000000000..b732cf252 Binary files /dev/null and b/site/content/docs/v1.7/contributions/img-for-tencent/f0fff5228527edc72d6e71a50d5dc966.png differ diff --git a/site/content/docs/v1.7/contributions/minio.md b/site/content/docs/v1.7/contributions/minio.md new file mode 100644 index 000000000..3c683473b --- /dev/null +++ b/site/content/docs/v1.7/contributions/minio.md @@ -0,0 +1,296 @@ +--- +title: "Quick start evaluation install with Minio" +layout: docs +--- + +The following example sets up the Velero server and client, then backs up and restores a sample application. + +For simplicity, the example uses Minio, an S3-compatible storage service that runs locally on your cluster. +For additional functionality with this setup, see the section below on how to [expose Minio outside your cluster][1]. + +**NOTE** The example lets you explore basic Velero functionality. Configuring Minio for production is out of scope. + +See [Set up Velero on your platform][3] for how to configure Velero for a production environment. + +If you encounter issues with installing or configuring, see [Debugging Installation Issues](debugging-install.md). + +## Prerequisites + +* Access to a Kubernetes cluster, version 1.7 or later. **Note:** restic support requires Kubernetes version 1.10 or later, or an earlier version with the mount propagation feature enabled. Restic support is not required for this example, but may be of interest later. See [Restic Integration][17]. +* A DNS server on the cluster +* `kubectl` installed +* Sufficient disk space to store backups in Minio. You will need sufficient disk space available to handle any +backups plus at least 1GB additional. Minio will not operate if less than 1GB of free disk space is available. + +## Install the CLI + +### Option 1: MacOS - Homebrew + +On macOS, you can use [Homebrew](https://brew.sh) to install the `velero` client: + +```bash +brew install velero +``` + +### Option 2: GitHub release + +1. Download the [latest official release's](https://github.com/vmware-tanzu/velero/releases) tarball for your client platform. + + _We strongly recommend that you use an [official release](https://github.com/vmware-tanzu/velero/releases) of +Velero. The tarballs for each release contain the `velero` command-line client. The code in the main branch +of the Velero repository is under active development and is not guaranteed to be stable!_ + +1. Extract the tarball: + + ```bash + tar -xvf .tar.gz -C /dir/to/extract/to + ``` + + The directory you extracted is called the "Velero directory" in subsequent steps. + +1. Move the `velero` binary from the Velero directory to somewhere in your PATH. + +## Set up server + +These instructions start the Velero server and a Minio instance that is accessible from within the cluster only. See [Expose Minio outside your cluster](#expose-minio-outside-your-cluster-with-a-service) for information about configuring your cluster for outside access to Minio. Outside access is required to access logs and run `velero describe` commands. + +1. Create a Velero-specific credentials file (`credentials-velero`) in your Velero directory: + + ``` + [default] + aws_access_key_id = minio + aws_secret_access_key = minio123 + ``` + +1. Start the server and the local storage service. In the Velero directory, run: + + ``` + kubectl apply -f examples/minio/00-minio-deployment.yaml + ``` + _Note_: The example Minio yaml provided uses "empty dir". Your node needs to have enough space available to store the + data being backed up plus 1GB of free space. If the node does not have enough space, you can modify the example yaml to + use a Persistent Volume instead of "empty dir" + + ``` + velero install \ + --provider aws \ + --plugins velero/velero-plugin-for-aws:v1.2.1 \ + --bucket velero \ + --secret-file ./credentials-velero \ + --use-volume-snapshots=false \ + --backup-location-config region=minio,s3ForcePathStyle="true",s3Url=http://minio.velero.svc:9000 + ``` + + This example assumes that it is running within a local cluster without a volume provider capable of snapshots, so no `VolumeSnapshotLocation` is created (`--use-volume-snapshots=false`). You may need to update AWS plugin version to one that is [compatible](https://github.com/vmware-tanzu/velero-plugin-for-aws#compatibility) with the version of Velero you are installing. + + Additionally, you can specify `--use-restic` to enable restic support, and `--wait` to wait for the deployment to be ready. + + This example also assumes you have named your Minio bucket "velero". + + +1. Deploy the example nginx application: + + ```bash + kubectl apply -f examples/nginx-app/base.yaml + ``` + +1. Check to see that both the Velero and nginx deployments are successfully created: + + ``` + kubectl get deployments -l component=velero --namespace=velero + kubectl get deployments --namespace=nginx-example + ``` + +## Back up + +1. Create a backup for any object that matches the `app=nginx` label selector: + + ``` + velero backup create nginx-backup --selector app=nginx + ``` + + Alternatively if you want to backup all objects *except* those matching the label `backup=ignore`: + + ``` + velero backup create nginx-backup --selector 'backup notin (ignore)' + ``` + +1. (Optional) Create regularly scheduled backups based on a cron expression using the `app=nginx` label selector: + + ``` + velero schedule create nginx-daily --schedule="0 1 * * *" --selector app=nginx + ``` + + Alternatively, you can use some non-standard shorthand cron expressions: + + ``` + velero schedule create nginx-daily --schedule="@daily" --selector app=nginx + ``` + + See the [cron package's documentation][30] for more usage examples. + +1. Simulate a disaster: + + ``` + kubectl delete namespace nginx-example + ``` + +1. To check that the nginx deployment and service are gone, run: + + ``` + kubectl get deployments --namespace=nginx-example + kubectl get services --namespace=nginx-example + kubectl get namespace/nginx-example + ``` + + You should get no results. + + NOTE: You might need to wait for a few minutes for the namespace to be fully cleaned up. + +## Restore + +1. Run: + + ``` + velero restore create --from-backup nginx-backup + ``` + +1. Run: + + ``` + velero restore get + ``` + + After the restore finishes, the output looks like the following: + + ``` + NAME BACKUP STATUS WARNINGS ERRORS CREATED SELECTOR + nginx-backup-20170727200524 nginx-backup Completed 0 0 2017-07-27 20:05:24 +0000 UTC + ``` + +NOTE: The restore can take a few moments to finish. During this time, the `STATUS` column reads `InProgress`. + +After a successful restore, the `STATUS` column is `Completed`, and `WARNINGS` and `ERRORS` are 0. All objects in the `nginx-example` namespace should be just as they were before you deleted them. + +If there are errors or warnings, you can look at them in detail: + +``` +velero restore describe +``` + +For more information, see [the debugging information][18]. + +## Clean up + +If you want to delete any backups you created, including data in object storage and persistent +volume snapshots, you can run: + +``` +velero backup delete BACKUP_NAME +``` + +This asks the Velero server to delete all backup data associated with `BACKUP_NAME`. You need to do +this for each backup you want to permanently delete. A future version of Velero will allow you to +delete multiple backups by name or label selector. + +Once fully removed, the backup is no longer visible when you run: + +``` +velero backup get BACKUP_NAME +``` + +To completely uninstall Velero, minio, and the nginx example app from your Kubernetes cluster: + +``` +kubectl delete namespace/velero clusterrolebinding/velero +kubectl delete crds -l component=velero +kubectl delete -f examples/nginx-app/base.yaml +``` + +## Expose Minio outside your cluster with a Service + +When you run commands to get logs or describe a backup, the Velero server generates a pre-signed URL to download the requested items. To access these URLs from outside the cluster -- that is, from your Velero client -- you need to make Minio available outside the cluster. You can: + +- Change the Minio Service type from `ClusterIP` to `NodePort`. +- Set up Ingress for your cluster, keeping Minio Service type `ClusterIP`. + +You can also specify a `publicUrl` config field for the pre-signed URL in your backup storage location config. + +### Expose Minio with Service of type NodePort + +The Minio deployment by default specifies a Service of type `ClusterIP`. You can change this to `NodePort` to easily expose a cluster service externally if you can reach the node from your Velero client. + +You must also get the Minio URL, which you can then specify as the value of the `publicUrl` field in your backup storage location config. + +1. In `examples/minio/00-minio-deployment.yaml`, change the value of Service `spec.type` from `ClusterIP` to `NodePort`. + +1. Get the Minio URL: + + - if you're running Minikube: + + ```shell + minikube service minio --namespace=velero --url + ``` + + - in any other environment: + 1. Get the value of an external IP address or DNS name of any node in your cluster. You must be able to reach this address from the Velero client. + 1. Append the value of the NodePort to get a complete URL. You can get this value by running: + + ```shell + kubectl -n velero get svc/minio -o jsonpath='{.spec.ports[0].nodePort}' + ``` + +1. Edit your `BackupStorageLocation` YAML, adding `publicUrl: ` as a field under `spec.config`. You must include the `http://` or `https://` prefix. + +## Accessing logs with an HTTPS endpoint + +If you're using Minio with HTTPS, you may see unintelligible text in the output of `velero describe`, or `velero logs` commands. + +To fix this, you can add a public URL to the `BackupStorageLocation`. + +In a terminal, run the following: + +```shell +kubectl patch -n velero backupstoragelocation default --type merge -p '{"spec":{"config":{"publicUrl":"https://:9000"}}}' +``` + +If your certificate is self-signed, see the [documentation on self-signed certificates][32]. + +## Expose Minio outside your cluster with Kubernetes in Docker (KinD): + +Kubernetes in Docker does not have support for NodePort services (see [this issue](https://github.com/kubernetes-sigs/kind/issues/99)). In this case, you can use a port forward to access the Minio bucket. + +In a terminal, run the following: + +```shell +MINIO_POD=$(kubectl get pods -n velero -l component=minio -o jsonpath='{.items[0].metadata.name}') + +kubectl port-forward $MINIO_POD -n velero 9000:9000 +``` + +Then, in another terminal: + +```shell +kubectl edit backupstoragelocation default -n velero +``` + +Add `publicUrl: http://localhost:9000` under the `spec.config` section. + + +### Work with Ingress + +Configuring Ingress for your cluster is out of scope for the Velero documentation. If you have already set up Ingress, however, it makes sense to continue with it while you run the example Velero configuration with Minio. + +In this case: + +1. Keep the Service type as `ClusterIP`. + +1. Edit your `BackupStorageLocation` YAML, adding `publicUrl: ` as a field under `spec.config`. + +[1]: #expose-minio-with-service-of-type-nodeport +[3]: ../customize-installation.md +[17]: ../restic.md +[18]: ../debugging-restores.md +[26]: https://github.com/vmware-tanzu/velero/releases +[30]: https://godoc.org/github.com/robfig/cron +[32]: ../self-signed-certificates.md diff --git a/site/content/docs/v1.7/contributions/oracle-config.md b/site/content/docs/v1.7/contributions/oracle-config.md new file mode 100644 index 000000000..8e2bae095 --- /dev/null +++ b/site/content/docs/v1.7/contributions/oracle-config.md @@ -0,0 +1,248 @@ +--- +title: "Use Oracle Cloud as a Backup Storage Provider for Velero" +layout: docs +--- + +## Introduction + +[Velero](https://velero.io/) is a tool used to backup and migrate Kubernetes applications. Here are the steps to use [Oracle Cloud Object Storage](https://docs.cloud.oracle.com/iaas/Content/Object/Concepts/objectstorageoverview.htm) as a destination for Velero backups. + +1. [Download Velero](#download-velero) +2. [Create A Customer Secret Key](#create-a-customer-secret-key) +3. [Create An Oracle Object Storage Bucket](#create-an-oracle-object-storage-bucket) +4. [Install Velero](#install-velero) +5. [Clean Up](#clean-up) +6. [Examples](#examples) +7. [Additional Reading](#additional-reading) + +## Download Velero + +1. Download the [latest release](https://github.com/vmware-tanzu/velero/releases/) of Velero to your development environment. This includes the `velero` CLI utility and example Kubernetes manifest files. For example: + + ``` + wget https://github.com/vmware-tanzu/velero/releases/download/v1.0.0/velero-v1.0.0-linux-amd64.tar.gz + ``` + + **NOTE:** Its strongly recommend that you use an official release of Velero. The tarballs for each release contain the velero command-line client. The code in the main branch of the Velero repository is under active development and is not guaranteed to be stable! + +2. Untar the release in your `/usr/bin` directory: `tar -xzvf .tar.gz` + + You may choose to rename the directory `velero` for the sake of simplicity: `mv velero-v1.0.0-linux-amd64 velero` + +3. Add it to your PATH: `export PATH=/usr/local/bin/velero:$PATH` + +4. Run `velero` to confirm the CLI has been installed correctly. You should see an output like this: + +``` +$ velero +Velero is a tool for managing disaster recovery, specifically for Kubernetes +cluster resources. It provides a simple, configurable, and operationally robust +way to back up your application state and associated data. + +If you're familiar with kubectl, Velero supports a similar model, allowing you to +execute commands such as 'velero get backup' and 'velero create schedule'. The same +operations can also be performed as 'velero backup get' and 'velero schedule create'. + +Usage: + velero [command] +``` + + + +## Create A Customer Secret Key + +1. Oracle Object Storage provides an API to enable interoperability with Amazon S3. To use this Amazon S3 Compatibility API, you need to generate the signing key required to authenticate with Amazon S3. This special signing key is an Access Key/Secret Key pair. Follow these steps to [create a Customer Secret Key](https://docs.cloud.oracle.com/iaas/Content/Identity/Tasks/managingcredentials.htm#To4). Refer to this link for more information about [Working with Customer Secret Keys](https://docs.cloud.oracle.com/iaas/Content/Identity/Tasks/managingcredentials.htm#s3). + +2. Create a Velero credentials file with your Customer Secret Key: + + ``` + $ vi credentials-velero + + [default] + aws_access_key_id=bae031188893d1eb83719648790ac850b76c9441 + aws_secret_access_key=MmY9heKrWiNVCSZQ2Mf5XTJ6Ys93Bw2d2D6NMSTXZlk= + ``` + + + +## Create An Oracle Object Storage Bucket + +Create an Oracle Cloud Object Storage bucket called `velero` in the root compartment of your Oracle Cloud tenancy. Refer to this page for [more information about creating a bucket with Object Storage](https://docs.cloud.oracle.com/iaas/Content/Object/Tasks/managingbuckets.htm#usingconsole). + + + +## Install Velero + +You will need the following information to install Velero into your Kubernetes cluster with Oracle Object Storage as the Backup Storage provider: + +``` +velero install \ + --provider [provider name] \ + --bucket [bucket name] \ + --prefix [tenancy name] \ + --use-volume-snapshots=false \ + --secret-file [secret file location] \ + --backup-location-config region=[region],s3ForcePathStyle="true",s3Url=[storage API endpoint] +``` + +- `--provider` This example uses the S3-compatible API, so use `aws` as the provider. +- `--bucket` The name of the bucket created in Oracle Object Storage - in our case this is named `velero`. +- ` --prefix` The name of your Oracle Cloud tenancy - in our case this is named `oracle-cloudnative`. +- `--use-volume-snapshots=false` Velero does not have a volume snapshot plugin for Oracle Cloud, so creating volume snapshots is disabled. +- `--secret-file` The path to your `credentials-velero` file. +- `--backup-location-config` The path to your Oracle Object Storage bucket. This consists of your `region` which corresponds to your Oracle Cloud region name ([List of Oracle Cloud Regions](https://docs.cloud.oracle.com/iaas/Content/General/Concepts/regions.htm?Highlight=regions)) and the `s3Url`, the S3-compatible API endpoint for Oracle Object Storage based on your region: `https://oracle-cloudnative.compat.objectstorage.[region name].oraclecloud.com` + +For example: + +``` +velero install \ + --provider aws \ + --bucket velero \ + --prefix oracle-cloudnative \ + --use-volume-snapshots=false \ + --secret-file /Users/mboxell/bin/velero/credentials-velero \ + --backup-location-config region=us-phoenix-1,s3ForcePathStyle="true",s3Url=https://oracle-cloudnative.compat.objectstorage.us-phoenix-1.oraclecloud.com +``` + +This will create a `velero` namespace in your cluster along with a number of CRDs, a ClusterRoleBinding, ServiceAccount, Secret, and Deployment for Velero. If your pod fails to successfully provision, you can troubleshoot your installation by running: `kubectl logs [velero pod name]`. + + + +## Clean Up + +To remove Velero from your environment, delete the namespace, ClusterRoleBinding, ServiceAccount, Secret, and Deployment and delete the CRDs, run: + +``` +kubectl delete namespace/velero clusterrolebinding/velero +kubectl delete crds -l component=velero +``` + +This will remove all resources created by `velero install`. + + + +## Examples + +After creating the Velero server in your cluster, try this example: + +### Basic example (without PersistentVolumes) + +1. Start the sample nginx app: `kubectl apply -f examples/nginx-app/base.yaml` + + This will create an `nginx-example` namespace with a `nginx-deployment` deployment, and `my-nginx` service. + + ``` + $ kubectl apply -f examples/nginx-app/base.yaml + namespace/nginx-example created + deployment.apps/nginx-deployment created + service/my-nginx created + ``` + + You can see the created resources by running `kubectl get all` + + ``` + $ kubectl get all + NAME READY STATUS RESTARTS AGE + pod/nginx-deployment-67594d6bf6-4296p 1/1 Running 0 20s + pod/nginx-deployment-67594d6bf6-f9r5s 1/1 Running 0 20s + + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + service/my-nginx LoadBalancer 10.96.69.166 80:31859/TCP 21s + + NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE + deployment.apps/nginx-deployment 2 2 2 2 21s + + NAME DESIRED CURRENT READY AGE + replicaset.apps/nginx-deployment-67594d6bf6 2 2 2 21s + ``` + +2. Create a backup: `velero backup create nginx-backup --include-namespaces nginx-example` + + ``` + $ velero backup create nginx-backup --include-namespaces nginx-example + Backup request "nginx-backup" submitted successfully. + Run `velero backup describe nginx-backup` or `velero backup logs nginx-backup` for more details. + ``` + + At this point you can navigate to appropriate bucket, called `velero`, in the Oracle Cloud Object Storage console to see the resources backed up using Velero. + +3. Simulate a disaster by deleting the `nginx-example` namespace: `kubectl delete namespaces nginx-example` + + ``` + $ kubectl delete namespaces nginx-example + namespace "nginx-example" deleted + ``` + + Wait for the namespace to be deleted. To check that the nginx deployment, service, and namespace are gone, run: + + ``` + kubectl get deployments --namespace=nginx-example + kubectl get services --namespace=nginx-example + kubectl get namespace/nginx-example + ``` + + This should return: `No resources found.` + +4. Restore your lost resources: `velero restore create --from-backup nginx-backup` + + ``` + $ velero restore create --from-backup nginx-backup + Restore request "nginx-backup-20190604102710" submitted successfully. + Run `velero restore describe nginx-backup-20190604102710` or `velero restore logs nginx-backup-20190604102710` for more details. + ``` + + Running `kubectl get namespaces` will show that the `nginx-example` namespace has been restored along with its contents. + +5. Run: `velero restore get` to view the list of restored resources. After the restore finishes, the output looks like the following: + + ``` + $ velero restore get + NAME BACKUP STATUS WARNINGS ERRORS CREATED SELECTOR + nginx-backup-20190604104249 nginx-backup Completed 0 0 2019-06-04 10:42:39 -0700 PDT + ``` + + NOTE: The restore can take a few moments to finish. During this time, the `STATUS` column reads `InProgress`. + + After a successful restore, the `STATUS` column shows `Completed`, and `WARNINGS` and `ERRORS` will show `0`. All objects in the `nginx-example` namespace should be just as they were before you deleted them. + + If there are errors or warnings, for instance if the `STATUS` column displays `FAILED` instead of `InProgress`, you can look at them in detail with `velero restore describe ` + + +6. Clean up the environment with `kubectl delete -f examples/nginx-app/base.yaml` + + ``` + $ kubectl delete -f examples/nginx-app/base.yaml + namespace "nginx-example" deleted + deployment.apps "nginx-deployment" deleted + service "my-nginx" deleted + ``` + + If you want to delete any backups you created, including data in object storage, you can run: `velero backup delete BACKUP_NAME` + + ``` + $ velero backup delete nginx-backup + Are you sure you want to continue (Y/N)? Y + Request to delete backup "nginx-backup" submitted successfully. + The backup will be fully deleted after all associated data (disk snapshots, backup files, restores) are removed. + ``` + + This asks the Velero server to delete all backup data associated with `BACKUP_NAME`. You need to do this for each backup you want to permanently delete. A future version of Velero will allow you to delete multiple backups by name or label selector. + + Once fully removed, the backup is no longer visible when you run: `velero backup get BACKUP_NAME` or more generally `velero backup get`: + + ``` + $ velero backup get nginx-backup + An error occurred: backups.velero.io "nginx-backup" not found + ``` + + ``` + $ velero backup get + NAME STATUS CREATED EXPIRES STORAGE LOCATION SELECTOR + ``` + + + +## Additional Reading + +* [Official Velero Documentation](https://velero.io/docs/v1.7/) +* [Oracle Cloud Infrastructure Documentation](https://docs.cloud.oracle.com/) diff --git a/site/content/docs/v1.7/contributions/tencent-config.md b/site/content/docs/v1.7/contributions/tencent-config.md new file mode 100644 index 000000000..50edfce49 --- /dev/null +++ b/site/content/docs/v1.7/contributions/tencent-config.md @@ -0,0 +1,168 @@ +--- +title: "Use Tencent Cloud Object Storage as Velero's storage destination." +layout: docs +--- + + +You can deploy Velero on Tencent [TKE](https://cloud.tencent.com/document/product/457), or an other Kubernetes cluster, and use Tencent Cloud Object Store as a destination for Velero’s backups. + + +## Prerequisites + +- Registered [Tencent Cloud Account](https://cloud.tencent.com/register). +- [Tencent Cloud COS](https://console.cloud.tencent.com/cos) service, referred to as COS, has been launched +- A Kubernetes cluster has been created, cluster version v1.12 or later, and the cluster can use DNS and Internet services normally. If you need to create a TKE cluster, refer to the Tencent [create a cluster](https://cloud.tencent.com/document/product/457/32189) documentation. + +## Create a Tencent Cloud COS bucket + +Create an object bucket for Velero to store backups in the Tencent Cloud COS console. For how to create, please refer to Tencent Cloud COS [Create a bucket](https://cloud.tencent.com/document/product/436/13309) usage instructions. + +Set access to the bucket through the object storage console, the bucket needs to be **read** and **written**, so the account is granted data reading, data writing permissions. For how to configure, see the [permission access settings](https://cloud.tencent.com/document/product/436/13315.E5.8D.95.E4.B8.AA.E6.8E.88.E6.9D.83) Tencent user instructions. + +## Get bucket access credentials + +Velero uses an AWS S3-compatible API to access Tencent Cloud COS storage, which requires authentication using a pair of access key IDs and key-created signatures. + +In the S3 API parameter, the "access_key_id" field is the access key ID and the "secret_access_key" field is the key. + +In the [Tencent Cloud Access Management Console](https://console.cloud.tencent.com/cam/capi), Create and acquire Tencent Cloud Keys "SecretId" and "SecretKey" for COS authorized account. **Where the "SecretId" value corresponds to the value of S3 API parameter "access_key_id" field, the "SecretKey" value corresponds to the value of S3 API parameter "secret_access_key" field**. + +Create the credential profile "credentials-velero" required by Velero in the local directory based on the above correspondence: + +```bash +[default] +aws_access_key_id= +aws_secret_access_key= +``` + +## Install Velero Resources + +You need to install the Velero CLI first, see [Install the CLI](https://velero.io/docs/v1.5/basic-install/#install-the-cli) for how to install. + +Follow the Velero installation command below to create velero and restic workloads and other necessary resource objects. + +```bash +velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.1.0 --bucket \ +--secret-file ./credentials-velero \ +--use-restic \ +--default-volumes-to-restic \ +--backup-location-config \ +region=ap-guangzhou,s3ForcePathStyle="true",s3Url=https://cos.ap-guangzhou.myqcloud.com +``` + +Description of the parameters: + +- `--provider`: Declares the type of plug-in provided by "aws". + +- `--plugins`: Use the AWS S3 compatible API plug-in "velero-plugin-for-aws". + +- `--bucket`: The bucket name created at Tencent Cloud COS. + +- `--secret-file`: Access tencent cloud COS access credential file for the "credentials-velero" credential file created above. + +- `--use-restic`: Back up and restore persistent volume data using the open source free backup tool [restic](https://github.com/restic/restic). However, 'hostPath' volumes are not supported, see the [restic limit](https://velero.io/docs/v1.5/restic/#limitations) for details), an integration that complements Velero's backup capabilities and is recommended to be turned on. + +- `--default-volumes-to-restic`: Enable the use of Restic to back up all Pod volumes, provided that the `--use-restic`parameter needs to be turned on. + +- `--backup-location-config`: Back up the bucket access-related configuration: + + `region`: Tencent cloud COS bucket area, for example, if the created region is Guangzhou, the Region parameter value is "ap-guangzhou". + + `s3ForcePathStyle`: Use the S3 file path format. + + `s3Url`: Tencent Cloud COS-compatible S3 API access address,Note that instead of creating a COS bucket for public network access domain name, you must use a format of "https://cos.`region`.myqcloud.com" URL, for example, if the region is Guangzhou, the parameter value is "https://cos.ap-guangzhou.myqcloud.com.". + +There are other installation parameters that can be viewed using `velero install --help`, such as setting `--use-volume-snapshots-false` to close the storage volume data snapshot backup if you do not want to back up the storage volume data. + +After executing the installation commands above, the installation process looks like this: + +{{< figure src="/docs/main/contributions/img-for-tencent/9015313121ed7987558c88081b052574.png" width="100%">}} + +After the installation command is complete, wait for the velero and restic workloads to be ready to see if the configured storage location is available. + +Executing the 'velero backup-location get' command to view the storage location status and display "Available" indicates that access to Tencent Cloud COS is OK, as shown in the following image: + +{{< figure src="/docs/main/contributions/img-for-tencent/69194157ccd5e377d1e7d914fd8c0336.png" width="100%">}} + +At this point, The installation using Tencent Cloud COS as Velero storage location is complete, If you need more installation information about Velero, You can see the official website [Velero documentation](https://velero.io/docs/) . + +## Velero backup and restore example + +In the cluster, use the helm tool to create a minio test service with a persistent volume, and the minio installation method can be found in the [minio installation](https://github.com/minio/charts), in which case can bound a load balancer for the minio service to access the management page using a public address in the browser. + +{{< figure src="/docs/main/contributions/img-for-tencent/f0fff5228527edc72d6e71a50d5dc966.png" width="100%">}} + +Sign in to the minio web management page and upload some image data for the test, as shown below: + +{{< figure src="/docs/main/contributions/img-for-tencent/e932223585c0b19891cc085ad7f438e1.png" width="100%">}} + +With Velero Backup, you can back up all objects in the cluster directly, or filter objects by type, namespace, and/or label. This example uses the following command to back up all resources under the 'default' namespace. + +``` +velero backup create default-backup --include-namespaces +``` + +Use the `velero backup get` command to see if the backup task is complete, and when the backup task status is "Completed," the backup task is completed without any errors, as shown in the following below: + +{{< figure src="/docs/main/contributions/img-for-tencent/eb2bbabae48b188748f5278bedf177f1.png" width="100%">}} + +At this point delete all of MinIO's resources, including its PVC persistence volume, as shown below:: + +{{< figure src="/docs/main/contributions/img-for-tencent/15ccaacf00640a04ae29ceed4c86195b.png" width="100%">}} + +After deleting the MinIO resource, use your backup to restore the deleted MinIO resource, and temporarily update the backup storage location to read-only mode (this prevents the backup object from being created or deleted in the backup storage location during the restore process):: + +```bash +kubectl patch backupstoragelocation default --namespace velero \ + --type merge \ + --patch '{"spec":{"accessMode":"ReadOnly"}}' + +``` + +Modifying access to Velero's storage location is "ReadOnly," as shown in the following image: + +{{< figure src="/docs/main/contributions/img-for-tencent/e8c2ab4e5e31d1370c62fad25059a8a8.png" width="100%">}} + +Now use the backup "default-backup" that Velero just created to create the restore task: + +```bash +velero restore create --from-backup +``` + +You can also use `velero restore get` to see the status of the restore task, and if the restore status is "Completed," the restore task is complete, as shown in the following image: + +{{< figure src="/docs/main/contributions/img-for-tencent/effe8a0a7ce3aa8e422db00bfdddc375.png" width="100%">}} + +When the restore is complete, you can see that the previously deleted minio-related resources have been restored successfully, as shown in the following image: + +{{< figure src="/docs/main/contributions/img-for-tencent/1d53b0115644d43657c2a5ece805c9b4.png" width="100%">}} + +Log in to minio's management page on your browser and you can see that the previously uploaded picture data is still there, indicating that the persistent volume's data was successfully restored, as shown below: + +{{< figure src="/docs/main/contributions/img-for-tencent/ceaca9ce6bc92bdce987c63d2fe71561.png" width="100%">}} + +When the restore is complete, don't forget to restore the backup storage location to read and write mode so that the next backup task can be used successfully: + +```bash +kubectl patch backupstoragelocation default --namespace velero \ + --type merge \ + --patch '{"spec":{"accessMode":"ReadWrite"}}' +``` + + + +## Uninstall Velero Resources + +To uninstall velero resources in a cluster, you can do so using the following command: + +```bash +kubectl delete namespace/velero clusterrolebinding/velero +kubectl delete crds -l component=velero +``` + + + +## Additional Reading + +- [Official Velero Documentation](https://velero.io/docs/) +- [Tencent Cloud Documentation](https://cloud.tencent.com/document/product) diff --git a/site/content/docs/v1.7/csi.md b/site/content/docs/v1.7/csi.md new file mode 100644 index 000000000..49f4f77e9 --- /dev/null +++ b/site/content/docs/v1.7/csi.md @@ -0,0 +1,77 @@ +--- +title: "Container Storage Interface Snapshot Support in Velero" +layout: docs +--- + +_This feature is under development. Documentation may not be up-to-date and features may not work as expected._ + +Integrating Container Storage Interface (CSI) snapshot support into Velero enables Velero to backup and restore CSI-backed volumes using the [Kubernetes CSI Snapshot Beta APIs](https://kubernetes.io/docs/concepts/storage/volume-snapshots/). + +By supporting CSI snapshot APIs, Velero can support any volume provider that has a CSI driver, without requiring a Velero-specific plugin to be available. + +## Prerequisites + +The following are the prerequisites for using Velero to take Container Storage Interface (CSI) snapshots: + + 1. The cluster is Kubernetes version 1.17 or greater. + 1. The cluster is running a CSI driver capable of support volume snapshots at the [v1beta1 API level](https://kubernetes.io/blog/2019/12/09/kubernetes-1-17-feature-cis-volume-snapshot-beta/). + 1. When restoring CSI volumesnapshots across clusters, the name of the CSI driver in the destination cluster is the same as that on the source cluster to ensure cross cluster portability of CSI volumesnapshots + +## Installing Velero with CSI support + +Ensure that the Velero server is running with the `EnableCSI` feature flag. See [Enabling Features][1] for more information. +Also, the Velero [CSI plugin][2] ([Docker Hub][3]) is necessary to integrate with the CSI volume snapshot APIs. + +Both of these can be added with the `velero install` command. + +```bash +velero install \ +--features=EnableCSI \ +--plugins=,velero/velero-plugin-for-csi:v0.1.0 \ +... +``` + +To include the status of CSI objects associated with a Velero backup in `velero backup describe` output, run `velero client config set features=EnableCSI`. +See [Enabling Features][1] for more information about managing client-side feature flags. + +## Implementation Choices + +This section documents some of the choices made during implementation of the Velero [CSI plugin][2]: + +1. Volumesnapshots created by the plugin will be retained only for the lifetime of the backup even if the `DeletionPolicy` on the volumesnapshotclass is set to `Retain`. To accomplish this, during deletion of the backup the prior to deleting the volumesnapshot, volumesnapshotcontent object will be patched to set its `DeletionPolicy` to `Delete`. Thus deleting volumesnapshot object will result in cascade delete of the volumesnapshotcontent and the snapshot in the storage provider. +1. Volumesnapshotcontent objects created during a velero backup that are dangling, unbound to a volumesnapshot object, will also be discovered, through labels, and deleted on backup deletion. +1. The Velero CSI plugin, to backup CSI backed PVCs, will choose the VolumeSnapshotClass in the cluster that has the same driver name and also has the `velero.io/csi-volumesnapshot-class` label set on it, like +```yaml +velero.io/csi-volumesnapshot-class: "true" +``` + +## Roadmap + +Velero's support level for CSI volume snapshotting will follow upstream Kubernetes support for the feature, and will reach general availability sometime +after volume snapshotting is GA in upstream Kubernetes. Beta support is expected to launch in Velero v1.4. + +## How it Works - Overview + +Velero's CSI support does not rely on the Velero VolumeSnapshotter plugin interface. + +Instead, Velero uses a collection of BackupItemAction plugins that act first against PersistentVolumeClaims. + +When this BackupItemAction sees PersistentVolumeClaims pointing to a PersistentVolume backed by a CSI driver, it will choose the VolumeSnapshotClass with the same driver name that has the `velero.io/csi-volumesnapshot-class` label to create a CSI VolumeSnapshot object with the PersistentVolumeClaim as a source. +This VolumeSnapshot object resides in the same namespace as the PersistentVolumeClaim that was used as a source. + +From there, the CSI external-snapshotter controller will see the VolumeSnapshot and create a VolumeSnapshotContent object, a cluster-scoped resource that will point to the actual, disk-based snapshot in the storage system. +The external-snapshotter plugin will call the CSI driver's snapshot method, and the driver will call the storage system's APIs to generate the snapshot. +Once an ID is generated and the storage system marks the snapshot as usable for restore, the VolumeSnapshotContent object will be updated with a `status.snapshotHandle` and the `status.readyToUse` field will be set. + +Velero will include the generated VolumeSnapshot and VolumeSnapshotContent objects in the backup tarball, as well as upload all VolumeSnapshots and VolumeSnapshotContents objects in a JSON file to the object storage system. +When Velero synchronizes backups into a new cluster, VolumeSnapshotContent objects will be synced into the cluster as well, so that Velero can manage backup expiration appropriately. + +The `DeletionPolicy` on the VolumeSnapshotContent will be the same as the `DeletionPolicy` on the VolumeSnapshotClass that was used to create the VolumeSnapshot. Setting a `DeletionPolicy` of `Retain` on the VolumeSnapshotClass will preserve the volume snapshot in the storage system for the lifetime of the Velero backup and will prevent the deletion of the volume snapshot, in the storage system, in the event of a disaster where the namespace with the VolumeSnapshot object may be lost. + +When the Velero backup expires, the VolumeSnapshot objects will be deleted and the VolumeSnapshotContent objects will be updated to have a `DeletionPolicy` of `Delete`, to free space on the storage system. + +For more details on how each plugin works, see the [CSI plugin repo][2]'s documentation. + +[1]: customize-installation.md#enable-server-side-features +[2]: https://github.com/vmware-tanzu/velero-plugin-for-csi/ +[3]: https://hub.docker.com/repository/docker/velero/velero-plugin-for-csi diff --git a/site/content/docs/v1.7/custom-plugins.md b/site/content/docs/v1.7/custom-plugins.md new file mode 100644 index 000000000..dbb82d5a3 --- /dev/null +++ b/site/content/docs/v1.7/custom-plugins.md @@ -0,0 +1,117 @@ +--- +title: "Plugins" +layout: docs +--- + +Velero has a plugin architecture that allows users to add their own custom functionality to Velero backups & restores without having to modify/recompile the core Velero binary. To add custom functionality, users simply create their own binary containing implementations of Velero's plugin kinds (described below), plus a small amount of boilerplate code to expose the plugin implementations to Velero. This binary is added to a container image that serves as an init container for the Velero server pod and copies the binary into a shared emptyDir volume for the Velero server to access. + +Multiple plugins, of any type, can be implemented in this binary. + +A fully-functional [sample plugin repository][1] is provided to serve as a convenient starting point for plugin authors. + +## Plugin Naming + +A plugin is identified by a prefix + name. + +**Note: Please don't use `velero.io` as the prefix for a plugin not supported by the Velero team.** The prefix should help users identify the entity developing the plugin, so please use a prefix that identify yourself. + +Whenever you define a Backup Storage Location or Volume Snapshot Location, this full name will be the value for the `provider` specification. + +For example: `oracle.io/oracle`. + +``` +apiVersion: velero.io/v1 +kind: BackupStorageLocation +spec: + provider: oracle.io/oracle +``` + +``` +apiVersion: velero.io/v1 +kind: VolumeSnapshotLocation +spec: + provider: oracle.io/oracle +``` + +When naming your plugin, keep in mind that the full name needs to conform to these rules: +- have two parts, prefix + name, separated by '/' +- none of the above parts can be empty +- the prefix is a valid DNS subdomain name +- a plugin with the same prefix + name cannot not already exist + +### Some examples: + +``` +- example.io/azure +- 1.2.3.4/5678 +- example-with-dash.io/azure +``` + +You will need to give your plugin(s) the full name when registering them by calling the appropriate `RegisterX` function: + +## Plugin Kinds + +Velero supports the following kinds of plugins: + +- **Object Store** - persists and retrieves backups, backup logs and restore logs +- **Volume Snapshotter** - creates volume snapshots (during backup) and restores volumes from snapshots (during restore) +- **Backup Item Action** - executes arbitrary logic for individual items prior to storing them in a backup file +- **Restore Item Action** - executes arbitrary logic for individual items prior to restoring them into a cluster +- **Delete Item Action** - executes arbitrary logic based on individual items within a backup prior to deleting the backup + +## Plugin Logging + +Velero provides a [logger][2] that can be used by plugins to log structured information to the main Velero server log or +per-backup/restore logs. It also passes a `--log-level` flag to each plugin binary, whose value is the value of the same +flag from the main Velero process. This means that if you turn on debug logging for the Velero server via `--log-level=debug`, +plugins will also emit debug-level logs. See the [sample repository][1] for an example of how to use the logger within your plugin. + +## Plugin Configuration + +Velero uses a ConfigMap-based convention for providing configuration to plugins. If your plugin needs to be configured at runtime, +define a ConfigMap like the following: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + # any name can be used; Velero uses the labels (below) + # to identify it rather than the name + name: my-plugin-config + + # must be in the namespace where the velero deployment + # is running + namespace: velero + + labels: + # this value-less label identifies the ConfigMap as + # config for a plugin (the built-in change storageclass + # restore item action plugin) + velero.io/plugin-config: "" + + # add a label whose key corresponds to the fully-qualified + # plugin name (for example mydomain.io/my-plugin-name), and whose + # value is the plugin type (BackupItemAction, RestoreItemAction, + # ObjectStore, or VolumeSnapshotter) + : + +data: + # add your configuration data here as key-value pairs +``` + +Then, in your plugin's implementation, you can read this ConfigMap to fetch the necessary configuration. See the [restic restore action][3] +for an example of this -- in particular, the `getPluginConfig(...)` function. + +## Feature Flags + +Velero will pass any known features flags as a comma-separated list of strings to the `--features` argument. + +Once parsed into a `[]string`, the features can then be registered using the `NewFeatureFlagSet` function and queried with `features.Enabled()`. + +## Environment Variables + +Velero adds the `LD_LIBRARY_PATH` into the list of environment variables to provide the convenience for plugins that requires C libraries/extensions in the runtime. + +[1]: https://github.com/vmware-tanzu/velero-plugin-example +[2]: https://github.com/vmware-tanzu/velero/blob/v1.7.0/pkg/plugin/logger.go +[3]: https://github.com/vmware-tanzu/velero/blob/v1.7.0/pkg/restore/restic_restore_action.go diff --git a/site/content/docs/v1.7/customize-installation.md b/site/content/docs/v1.7/customize-installation.md new file mode 100644 index 000000000..57dadb38c --- /dev/null +++ b/site/content/docs/v1.7/customize-installation.md @@ -0,0 +1,368 @@ +--- +title: "Customize Velero Install" +layout: docs +--- + +## Plugins + +During install, Velero requires that at least one plugin is added (with the `--plugins` flag). Please see the documentation under [Plugins](overview-plugins.md) + +## Install in any namespace + +Velero is installed in the `velero` namespace by default. However, you can install Velero in any namespace. See [run in custom namespace][2] for details. + +## Use non-file-based identity mechanisms + +By default, `velero install` expects a credentials file for your `velero` IAM account to be provided via the `--secret-file` flag. + +If you are using an alternate identity mechanism, such as kube2iam/kiam on AWS, Workload Identity on GKE, etc., that does not require a credentials file, you can specify the `--no-secret` flag instead of `--secret-file`. + +## Enable restic integration + +By default, `velero install` does not install Velero's [restic integration][3]. To enable it, specify the `--use-restic` flag. + +If you've already run `velero install` without the `--use-restic` flag, you can run the same command again, including the `--use-restic` flag, to add the restic integration to your existing install. + +## Default Pod Volume backup to restic + +By default, `velero install` does not enable use of restic to take backups of all pod volumes. An annotation has to be applied on every pod which contains volumes to be backed up by restic. + +To backup all pod volumes using restic without having to apply annotation on the pod, run the `velero install` command with the `--default-volumes-to-restic` flag. + +Using this flag requires restic integration to be enabled with the `--use-restic` flag. Please refer to the [restic integration][3] page for more information. + +## Enable features + +New features in Velero will be released as beta features behind feature flags which are not enabled by default. A full listing of Velero feature flags can be found [here][11]. + +### Enable server side features + +Features on the Velero server can be enabled using the `--features` flag to the `velero install` command. This flag takes as value a comma separated list of feature flags to enable. As an example [CSI snapshotting of PVCs][10] can be enabled using `EnableCSI` feature flag in the `velero install` command as shown below: + +```bash +velero install --features=EnableCSI +``` + +Another example is enabling the support of multiple API group versions, as documented at [- -features=EnableAPIGroupVersions](enable-api-group-versions-feature.md). + +Feature flags, passed to `velero install` will be passed to the Velero deployment and also to the `restic` daemon set, if `--use-restic` flag is used. + +Similarly, features may be disabled by removing the corresponding feature flags from the `--features` flag. + +Enabling and disabling feature flags will require modifying the Velero deployment and also the restic daemonset. This may be done from the CLI by uninstalling and re-installing Velero, or by editing the `deploy/velero` and `daemonset/restic` resources in-cluster. + +```bash +$ kubectl -n velero edit deploy/velero +$ kubectl -n velero edit daemonset/restic +``` + +### Enable client side features + +For some features it may be necessary to use the `--features` flag to the Velero client. This may be done by passing the `--features` on every command run using the Velero CLI or the by setting the features in the velero client config file using the `velero client config set` command as shown below: + +```bash +velero client config set features=EnableCSI +``` + +This stores the config in a file at `$HOME/.config/velero/config.json`. + +All client side feature flags may be disabled using the below command + +```bash +velero client config set features= +``` + +### Colored CLI output + +Velero CLI uses colored output for some commands, such as `velero describe`. If +the environment in which Velero is run doesn't support colored output, the +colored output will be automatically disabled. However, you can manually disable +colors with config file: + +```bash +velero client config set colorized=false +``` + +Note that if you specify `--colorized=true` as a CLI option it will override +the config file setting. + + +## Customize resource requests and limits + +At installation, Velero sets default resource requests and limits for the Velero pod and the restic pod, if you using the [restic integration](/docs/main/restic/). + +{{< table caption="Velero Customize resource requests and limits defaults" >}} +|Setting|Velero pod defaults|restic pod defaults| +|--- |--- |--- | +|CPU request|500m|500m| +|Memory requests|128Mi|512Mi| +|CPU limit|1000m (1 CPU)|1000m (1 CPU)| +|Memory limit|512Mi|1024Mi| +{{< /table >}} + +### Install with custom resource requests and limits + +You can customize these resource requests and limit when you first install using the [velero install][6] CLI command. + +``` +velero install \ + --velero-pod-cpu-request \ + --velero-pod-mem-request \ + --velero-pod-cpu-limit \ + --velero-pod-mem-limit \ + [--use-restic] \ + [--default-volumes-to-restic] \ + [--restic-pod-cpu-request ] \ + [--restic-pod-mem-request ] \ + [--restic-pod-cpu-limit ] \ + [--restic-pod-mem-limit ] +``` + +### Update resource requests and limits after install + +After installation you can adjust the resource requests and limits in the Velero Deployment spec or restic DeamonSet spec, if you are using the restic integration. + +**Velero pod** + +Update the `spec.template.spec.containers.resources.limits` and `spec.template.spec.containers.resources.requests` values in the Velero deployment. + +```bash +kubectl patch deployment velero -n velero --patch \ +'{"spec":{"template":{"spec":{"containers":[{"name": "velero", "resources": {"limits":{"cpu": "1", "memory": "512Mi"}, "requests": {"cpu": "1", "memory": "128Mi"}}}]}}}}' +``` + +**restic pod** + +Update the `spec.template.spec.containers.resources.limits` and `spec.template.spec.containers.resources.requests` values in the restic DeamonSet spec. + +```bash +kubectl patch daemonset restic -n velero --patch \ +'{"spec":{"template":{"spec":{"containers":[{"name": "restic", "resources": {"limits":{"cpu": "1", "memory": "1024Mi"}, "requests": {"cpu": "1", "memory": "512Mi"}}}]}}}}' +``` + +Additionally, you may want to update the the default Velero restic pod operation timeout (default 240 minutes) to allow larger backups more time to complete. You can adjust this timeout by adding the `- --restic-timeout` argument to the Velero Deployment spec. + +**NOTE:** Changes made to this timeout value will revert back to the default value if you re-run the Velero install command. + +1. Open the Velero Deployment spec. + + ``` + kubectl edit deploy velero -n velero + ``` + +1. Add `- --restic-timeout` to `spec.template.spec.containers`. + + ```yaml + spec: + template: + spec: + containers: + - args: + - --restic-timeout=240m + ``` + +## Configure more than one storage location for backups or volume snapshots + +Velero supports any number of backup storage locations and volume snapshot locations. For more details, see [about locations](locations.md). + +However, `velero install` only supports configuring at most one backup storage location and one volume snapshot location. + +To configure additional locations after running `velero install`, use the `velero backup-location create` and/or `velero snapshot-location create` commands along with provider-specific configuration. Use the `--help` flag on each of these commands for more details. + +## Do not configure a backup storage location during install + +If you need to install Velero without a default backup storage location (without specifying `--bucket` or `--provider`), the `--no-default-backup-location` flag is required for confirmation. + +## Install an additional volume snapshot provider + +Velero supports using different providers for volume snapshots than for object storage -- for example, you can use AWS S3 for object storage, and Portworx for block volume snapshots. + +However, `velero install` only supports configuring a single matching provider for both object storage and volume snapshots. + +To use a different volume snapshot provider: + +1. Install the Velero server components by following the instructions for your **object storage** provider + +1. Add your volume snapshot provider's plugin to Velero (look in [your provider][0]'s documentation for the image name): + + ```bash + velero plugin add + ``` + +1. Add a volume snapshot location for your provider, following [your provider][0]'s documentation for configuration: + + ```bash + velero snapshot-location create \ + --provider \ + [--config ] + ``` + +## Generate YAML only + +By default, `velero install` generates and applies a customized set of Kubernetes configuration (YAML) to your cluster. + +To generate the YAML without applying it to your cluster, use the `--dry-run -o yaml` flags. + +This is useful for applying bespoke customizations, integrating with a GitOps workflow, etc. + +If you are installing Velero in Kubernetes 1.14.x or earlier, you need to use `kubectl apply`'s `--validate=false` option when applying the generated configuration to your cluster. See [issue 2077][7] and [issue 2311][8] for more context. + +## Use a storage provider secured by a self-signed certificate + +If you intend to use Velero with a storage provider that is secured by a self-signed certificate, +you may need to instruct Velero to trust that certificate. See [use Velero with a storage provider secured by a self-signed certificate][9] for details. + +## Additional options + +Run `velero install --help` or see the [Helm chart documentation](https://vmware-tanzu.github.io/helm-charts/) for the full set of installation options. + +## Optional Velero CLI configurations + +### Enabling shell autocompletion + +**Velero CLI** provides autocompletion support for `Bash` and `Zsh`, which can save you a lot of typing. + +Below are the procedures to set up autocompletion for `Bash` (including the difference between `Linux` and `macOS`) and `Zsh`. + +#### Bash on Linux + +The **Velero CLI** completion script for `Bash` can be generated with the command `velero completion bash`. Sourcing the completion script in your shell enables velero autocompletion. + +However, the completion script depends on [**bash-completion**](https://github.com/scop/bash-completion), which means that you have to install this software first (you can test if you have bash-completion already installed by running `type _init_completion`). + +##### Install bash-completion + +`bash-completion` is provided by many package managers (see [here](https://github.com/scop/bash-completion#installation)). You can install it with `apt-get install bash-completion` or `yum install bash-completion`, etc. + +The above commands create `/usr/share/bash-completion/bash_completion`, which is the main script of bash-completion. Depending on your package manager, you have to manually source this file in your `~/.bashrc` file. + +To find out, reload your shell and run `type _init_completion`. If the command succeeds, you're already set, otherwise add the following to your `~/.bashrc` file: + +```shell +source /usr/share/bash-completion/bash_completion +``` + +Reload your shell and verify that bash-completion is correctly installed by typing `type _init_completion`. + +##### Enable Velero CLI autocompletion for Bash on Linux + +You now need to ensure that the **Velero CLI** completion script gets sourced in all your shell sessions. There are two ways in which you can do this: + +- Source the completion script in your `~/.bashrc` file: + + ```shell + echo 'source <(velero completion bash)' >>~/.bashrc + ``` + +- Add the completion script to the `/etc/bash_completion.d` directory: + + ```shell + velero completion bash >/etc/bash_completion.d/velero + ``` + +- If you have an alias for velero, you can extend shell completion to work with that alias: + + ```shell + echo 'alias v=velero' >>~/.bashrc + echo 'complete -F __start_velero v' >>~/.bashrc + ``` + +> `bash-completion` sources all completion scripts in `/etc/bash_completion.d`. + +Both approaches are equivalent. After reloading your shell, velero autocompletion should be working. + +#### Bash on macOS + +The **Velero CLI** completion script for Bash can be generated with `velero completion bash`. Sourcing this script in your shell enables velero completion. + +However, the velero completion script depends on [**bash-completion**](https://github.com/scop/bash-completion) which you thus have to previously install. + + +> There are two versions of bash-completion, v1 and v2. V1 is for Bash 3.2 (which is the default on macOS), and v2 is for Bash 4.1+. The velero completion script **doesn't work** correctly with bash-completion v1 and Bash 3.2. It requires **bash-completion v2** and **Bash 4.1+**. Thus, to be able to correctly use velero completion on macOS, you have to install and use Bash 4.1+ ([*instructions*](https://itnext.io/upgrading-bash-on-macos-7138bd1066ba)). The following instructions assume that you use Bash 4.1+ (that is, any Bash version of 4.1 or newer). + + +##### Install bash-completion + +> As mentioned, these instructions assume you use Bash 4.1+, which means you will install bash-completion v2 (in contrast to Bash 3.2 and bash-completion v1, in which case kubectl completion won't work). + +You can test if you have bash-completion v2 already installed with `type _init_completion`. If not, you can install it with Homebrew: + + ```shell + brew install bash-completion@2 + ``` + +As stated in the output of this command, add the following to your `~/.bashrc` file: + + ```shell + export BASH_COMPLETION_COMPAT_DIR="/usr/local/etc/bash_completion.d" + [[ -r "/usr/local/etc/profile.d/bash_completion.sh" ]] && . "/usr/local/etc/profile.d/bash_completion.sh" + ``` + +Reload your shell and verify that bash-completion v2 is correctly installed with `type _init_completion`. + +##### Enable Velero CLI autocompletion for Bash on macOS + +You now have to ensure that the velero completion script gets sourced in all your shell sessions. There are multiple ways to achieve this: + +- Source the completion script in your `~/.bashrc` file: + + ```shell + echo 'source <(velero completion bash)' >>~/.bashrc + + ``` + +- Add the completion script to the `/usr/local/etc/bash_completion.d` directory: + + ```shell + velero completion bash >/usr/local/etc/bash_completion.d/velero + ``` + +- If you have an alias for velero, you can extend shell completion to work with that alias: + + ```shell + echo 'alias v=velero' >>~/.bashrc + echo 'complete -F __start_velero v' >>~/.bashrc + ``` + +- If you installed velero with Homebrew (as explained [above](#install-with-homebrew-on-macos)), then the velero completion script should already be in `/usr/local/etc/bash_completion.d/velero`. In that case, you don't need to do anything. + +> The Homebrew installation of bash-completion v2 sources all the files in the `BASH_COMPLETION_COMPAT_DIR` directory, that's why the latter two methods work. + +In any case, after reloading your shell, velero completion should be working. + +#### Autocompletion on Zsh + +The velero completion script for Zsh can be generated with the command `velero completion zsh`. Sourcing the completion script in your shell enables velero autocompletion. + +To do so in all your shell sessions, add the following to your `~/.zshrc` file: + + ```shell + source <(velero completion zsh) + ``` + +If you have an alias for kubectl, you can extend shell completion to work with that alias: + + ```shell + echo 'alias v=velero' >>~/.zshrc + echo 'complete -F __start_velero v' >>~/.zshrc + ``` + +After reloading your shell, kubectl autocompletion should be working. + +If you get an error like `complete:13: command not found: compdef`, then add the following to the beginning of your `~/.zshrc` file: + + ```shell + autoload -Uz compinit + compinit + ``` + +[1]: https://github.com/vmware-tanzu/velero/releases/latest +[2]: namespace.md +[3]: restic.md +[4]: on-premises.md +[6]: velero-install.md#usage +[7]: https://github.com/vmware-tanzu/velero/issues/2077 +[8]: https://github.com/vmware-tanzu/velero/issues/2311 +[9]: self-signed-certificates.md +[10]: csi.md +[11]: https://github.com/vmware-tanzu/velero/blob/v1.7.0/pkg/apis/velero/v1/constants.go diff --git a/site/content/docs/v1.7/debugging-install.md b/site/content/docs/v1.7/debugging-install.md new file mode 100644 index 000000000..e6a75e525 --- /dev/null +++ b/site/content/docs/v1.7/debugging-install.md @@ -0,0 +1,74 @@ +--- +title: "Debugging Installation Issues" +layout: docs +--- + +## General + +### `invalid configuration: no configuration has been provided` +This typically means that no `kubeconfig` file can be found for the Velero client to use. Velero looks for a kubeconfig in the +following locations: +* the path specified by the `--kubeconfig` flag, if any +* the path specified by the `$KUBECONFIG` environment variable, if any +* `~/.kube/config` + +### Backups or restores stuck in `New` phase +This means that the Velero controllers are not processing the backups/restores, which usually happens because the Velero server is not running. Check the pod description and logs for errors: +``` +kubectl -n velero describe pods +kubectl -n velero logs deployment/velero +``` + + +## AWS + +### `NoCredentialProviders: no valid providers in chain` + +#### Using credentials +This means that the secret containing the AWS IAM user credentials for Velero has not been created/mounted properly +into the Velero server pod. Ensure the following: + +* The `cloud-credentials` secret exists in the Velero server's namespace +* The `cloud-credentials` secret has a single key, `cloud`, whose value is the contents of the `credentials-velero` file +* The `credentials-velero` file is formatted properly and has the correct values: + + ``` + [default] + aws_access_key_id= + aws_secret_access_key= + ``` + +* The `cloud-credentials` secret is defined as a volume for the Velero deployment +* The `cloud-credentials` secret is being mounted into the Velero server pod at `/credentials` + +#### Using kube2iam +This means that Velero can't read the content of the S3 bucket. Ensure the following: + +* A Trust Policy document exists that allows the role used by kube2iam to assume Velero's role, as stated in the AWS config documentation. +* The new Velero role has all the permissions listed in the documentation regarding S3. + + +## Azure + +### `Failed to refresh the Token` or `adal: Refresh request failed` +This means that the secrets containing the Azure service principal credentials for Velero has not been created/mounted +properly into the Velero server pod. Ensure the following: + +* The `cloud-credentials` secret exists in the Velero server's namespace +* The `cloud-credentials` secret has all of the expected keys and each one has the correct value (see [setup instructions][0]) +* The `cloud-credentials` secret is defined as a volume for the Velero deployment +* The `cloud-credentials` secret is being mounted into the Velero server pod at `/credentials` + + +## GCE/GKE + +### `open credentials/cloud: no such file or directory` +This means that the secret containing the GCE service account credentials for Velero has not been created/mounted properly +into the Velero server pod. Ensure the following: + +* The `cloud-credentials` secret exists in the Velero server's namespace +* The `cloud-credentials` secret has a single key, `cloud`, whose value is the contents of the `credentials-velero` file +* The `cloud-credentials` secret is defined as a volume for the Velero deployment +* The `cloud-credentials` secret is being mounted into the Velero server pod at `/credentials` + +[0]: azure-config.md#create-service-principal diff --git a/site/content/docs/v1.7/debugging-restores.md b/site/content/docs/v1.7/debugging-restores.md new file mode 100644 index 000000000..3320349cd --- /dev/null +++ b/site/content/docs/v1.7/debugging-restores.md @@ -0,0 +1,105 @@ +--- +title: "Debugging Restores" +layout: docs +--- + +## Example + +When Velero finishes a Restore, its status changes to "Completed" regardless of whether or not there are issues during the process. The number of warnings and errors are indicated in the output columns from `velero restore get`: + +``` +NAME BACKUP STATUS WARNINGS ERRORS CREATED SELECTOR +backup-test-20170726180512 backup-test Completed 155 76 2017-07-26 11:41:14 -0400 EDT +backup-test-20170726180513 backup-test Completed 121 14 2017-07-26 11:48:24 -0400 EDT +backup-test-2-20170726180514 backup-test-2 Completed 0 0 2017-07-26 13:31:21 -0400 EDT +backup-test-2-20170726180515 backup-test-2 Completed 0 1 2017-07-26 13:32:59 -0400 EDT +``` + +To delve into the warnings and errors into more detail, you can use `velero restore describe`: + +```bash +velero restore describe backup-test-20170726180512 +``` + +The output looks like this: + +``` +Name: backup-test-20170726180512 +Namespace: velero +Labels: +Annotations: + +Backup: backup-test + +Namespaces: + Included: * + Excluded: + +Resources: + Included: serviceaccounts + Excluded: nodes, events, events.events.k8s.io + Cluster-scoped: auto + +Namespace mappings: + +Label selector: + +Restore PVs: auto + +Preserve Service NodePorts: auto + +Phase: Completed + +Validation errors: + +Warnings: + Velero: + Cluster: + Namespaces: + velero: serviceaccounts "velero" already exists + serviceaccounts "default" already exists + kube-public: serviceaccounts "default" already exists + kube-system: serviceaccounts "attachdetach-controller" already exists + serviceaccounts "certificate-controller" already exists + serviceaccounts "cronjob-controller" already exists + serviceaccounts "daemon-set-controller" already exists + serviceaccounts "default" already exists + serviceaccounts "deployment-controller" already exists + serviceaccounts "disruption-controller" already exists + serviceaccounts "endpoint-controller" already exists + serviceaccounts "generic-garbage-collector" already exists + serviceaccounts "horizontal-pod-autoscaler" already exists + serviceaccounts "job-controller" already exists + serviceaccounts "kube-dns" already exists + serviceaccounts "namespace-controller" already exists + serviceaccounts "node-controller" already exists + serviceaccounts "persistent-volume-binder" already exists + serviceaccounts "pod-garbage-collector" already exists + serviceaccounts "replicaset-controller" already exists + serviceaccounts "replication-controller" already exists + serviceaccounts "resourcequota-controller" already exists + serviceaccounts "service-account-controller" already exists + serviceaccounts "service-controller" already exists + serviceaccounts "statefulset-controller" already exists + serviceaccounts "ttl-controller" already exists + default: serviceaccounts "default" already exists + +Errors: + Velero: + Cluster: + Namespaces: +``` + +## Structure + +Errors appear for incomplete or partial restores. Warnings appear for non-blocking issues, for example, the +restore looks "normal" and all resources referenced in the backup exist in some form, although some +of them may have been pre-existing. + +Both errors and warnings are structured in the same way: + +* `Velero`: A list of system-related issues encountered by the Velero server. For example, Velero couldn't read a directory. + +* `Cluster`: A list of issues related to the restore of cluster-scoped resources. + +* `Namespaces`: A map of namespaces to the list of issues related to the restore of their respective resources. diff --git a/site/content/docs/v1.7/development.md b/site/content/docs/v1.7/development.md new file mode 100644 index 000000000..82ace894d --- /dev/null +++ b/site/content/docs/v1.7/development.md @@ -0,0 +1,56 @@ +--- +title: "Development " +layout: docs +--- + +## Update generated files + +Run `make update` to regenerate files if you make the following changes: + +* Add/edit/remove command line flags and/or their help text +* Add/edit/remove commands or subcommands +* Add new API types +* Add/edit/remove plugin protobuf message or service definitions + +The following files are automatically generated from the source code: + +* The clientset +* Listers +* Shared informers +* Documentation +* Protobuf/gRPC types + +You can run `make verify` to ensure that all generated files (clientset, listers, shared informers, docs) are up to date. + +## Linting + +You can run `make lint` which executes golangci-lint inside the build image, or `make local-lint` which executes outside of the build image. +Both `make lint` and `make local-lint` will only run the linter against changes. + +Use `lint-all` to run the linter against the entire code base. + +The default linters are defined in the `Makefile` via the `LINTERS` variable. + +You can also override the default list of linters by running the command + +`$ make lint LINTERS=gosec` + +## Test + +To run unit tests, use `make test`. + +## Vendor dependencies + +If you need to add or update the vendored dependencies, see [Vendoring dependencies][11]. + +## Using the main branch + +If you are developing or using the main branch, note that you may need to update the Velero CRDs to get new changes as other development work is completed. + +```bash +velero install --crds-only --dry-run -o yaml | kubectl apply -f - +``` + +**NOTE:** You could change the default CRD API version (v1beta1 _or_ v1) if Velero CLI can't discover the Kubernetes preferred CRD API version. The Kubernetes version < 1.16 preferred CRD API version is v1beta1; the Kubernetes version >= 1.16 preferred CRD API version is v1. + +[11]: vendoring-dependencies.md diff --git a/site/content/docs/v1.7/disaster-case.md b/site/content/docs/v1.7/disaster-case.md new file mode 100644 index 000000000..6b73e6716 --- /dev/null +++ b/site/content/docs/v1.7/disaster-case.md @@ -0,0 +1,44 @@ +--- +title: "Disaster recovery" +layout: docs +--- + +*Using Schedules and Read-Only Backup Storage Locations* + +If you periodically back up your cluster's resources, you are able to return to a previous state in case of some unexpected mishap, such as a service outage. Doing so with Velero looks like the following: + +1. After you first run the Velero server on your cluster, set up a daily backup (replacing `` in the command as desired): + + ``` + velero schedule create --schedule "0 7 * * *" + ``` + + This creates a Backup object with the name `-`. The default backup retention period, expressed as TTL (time to live), is 30 days (720 hours); you can use the `--ttl ` flag to change this as necessary. See [how velero works][1] for more information about backup expiry. + +1. A disaster happens and you need to recreate your resources. + +1. Update your backup storage location to read-only mode (this prevents backup objects from being created or deleted in the backup storage location during the restore process): + + ```bash + kubectl patch backupstoragelocation \ + --namespace velero \ + --type merge \ + --patch '{"spec":{"accessMode":"ReadOnly"}}' + ``` + +1. Create a restore with your most recent Velero Backup: + + ``` + velero restore create --from-backup - + ``` + +1. When ready, revert your backup storage location to read-write mode: + + ```bash + kubectl patch backupstoragelocation \ + --namespace velero \ + --type merge \ + --patch '{"spec":{"accessMode":"ReadWrite"}}' + ``` + +[1]: how-velero-works.md#set-a-backup-to-expire diff --git a/site/content/docs/v1.7/enable-api-group-versions-feature.md b/site/content/docs/v1.7/enable-api-group-versions-feature.md new file mode 100644 index 000000000..5c40b3e71 --- /dev/null +++ b/site/content/docs/v1.7/enable-api-group-versions-feature.md @@ -0,0 +1,115 @@ +--- +title: "Enable API Group Versions Feature" +layout: docs +--- + +## Background + +Velero serves to both restore and migrate Kubernetes applications. Typically, backup and restore does not involve upgrading Kubernetes API group versions. However, when migrating from a source cluster to a destination cluster, it is not unusual to see the API group versions differing between clusters. + +**NOTE:** Kubernetes applications are made up of various resources. Common resources are pods, jobs, and deployments. Custom resources are created via custom resource definitions (CRDs). Every resource, whether custom or not, is part of a group, and each group has a version called the API group version. + +Kubernetes by default allows changing API group versions between clusters as long as the upgrade is a single version, for example, v1 -> v2beta1. Jumping multiple versions, for example, v1 -> v3, is not supported out of the box. This is where the Velero Enable API Group Version feature can help you during an upgrade. + +Currently, the Enable API Group Version feature is in beta and can be enabled by installing Velero with a [feature flag](customize-installation.md/#enable-server-side-features), `--features=EnableAPIGroupVersions`. + +For the most up-to-date information on Kubernetes API version compatibility, you should always review the [Kubernetes release notes](https://github.com/kubernetes/kubernetes/tree/master/CHANGELOG) for the source and destination cluster version to before starting an upgrade, migration, or restore. If there is a difference between Kubernetes API versions, use the Enable API Group Version feature to help mitigate compatibility issues. + +## How the Enable API Group Versions Feature Works + +When the Enable API Group Versions feature is enabled on the source cluster, Velero will not only back up Kubernetes preferred API group versions, but it will also back up all supported versions on the cluster. As an example, consider the resource `horizontalpodautoscalers` which falls under the `autoscaling` group. Without the feature flag enabled, only the preferred API group version for autoscaling, `v1` will be backed up. With the feature enabled, the remaining supported versions, `v2beta1` and `v2beta2` will also be backed up. Once the versions are stored in the backup tarball file, they will be available to be restored on the destination cluster. + +When the Enable API Group Versions feature is enabled on the destination cluster, Velero restore will choose the version to restore based on an API group version priority order. + +The version priorities are listed from highest to lowest priority below: + +- Priority 1: destination cluster preferred version +- Priority 2: source cluster preferred version +- Priority 3: non-preferred common supported version with the highest [Kubernetes version priority](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definition-versioning/#version-priority) + +The highest priority (Priority 1) will be the destination cluster's preferred API group version. If the destination preferred version is found in the backup tarball, it will be the API group version chosen for restoration for that resource. However, if the destination preferred version is not found in the backup tarball, the next version in the list will be selected: the source cluster preferred version (Priority 2). + +If the source cluster preferred version is found to be supported by the destination cluster, it will be chosen as the API group version to restore. However, if the source preferred version is not supported by the destination cluster, then the next version in the list will be considered: a non-preferred common supported version (Priority 3). + +In the case that there are more than one non-preferred common supported version, which version will be chosen? The answer requires understanding the [Kubernetes version priority order](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definition-versioning/#version-priority). Kubernetes prioritizes group versions by making the latest, most stable version the highest priority. The highest priority version is the Kubernetes preferred version. Here is a sorted version list example from the Kubernetes.io documentation: + +- v10 +- v2 +- v1 +- v11beta2 +- v10beta3 +- v3beta1 +- v12alpha1 +- v11alpha2 +- foo1 +- foo10 + +Of the non-preferred common versions, the version that has the highest Kubernetes version priority will be chosen. See the example for Priority 3 below. + +To better understand which API group version will be chosen, the following provides some concrete examples. The examples use the term "target cluster" which is synonymous to "destination cluster". + +![Priority 1 Case A example](/docs/main/img/gv_priority1-caseA.png) + +![Priority 1 Case B example](/docs/main/img/gv_priority1-caseB.png) + +![Priority 2 Case C example](/docs/main/img/gv_priority2-caseC.png) + +![Priority 3 Case D example](/docs/main/img/gv_priority3-caseD.png) + +## Procedure for Using the Enable API Group Versions Feature + +1. [Install Velero](basic-install.md) on source cluster with the [feature flag enabled](customize-installation.md/#enable-server-side-features). The flag is `--features=EnableAPIGroupVersions`. For the enable API group versions feature to work, the feature flag needs to be used for Velero installations on both the source and destination clusters. +2. Back up and restore following the [migration case instructions](migration-case.md). Note that "Cluster 1" in the instructions refers to the source cluster, and "Cluster 2" refers to the destination cluster. + +## Advanced Procedure for Customizing the Version Prioritization + +Optionally, users can create a config map to override the default API group prioritization for some or all of the resources being migrated. For each resource that is specified by the user, Velero will search for the version in both the backup tarball and the destination cluster. If there is a match, the user-specified API group version will be restored. If the backup tarball and the destination cluster does not have or support any of the user-specified versions, then the default version prioritization will be used. + +Here are the steps for creating a config map that allows users to override the default version prioritization. These steps must happen on the destination cluster before a Velero restore is initiated. + +1. Create a file called `restoreResourcesVersionPriority`. The file name will become a key in the `data` field of the config map. + - In the file, write a line for each resource group you'd like to override. Make sure each line follows the format `.=,` + - Note that the resource group and versions are separated by a single equal (=) sign. Each version is listed in order of user's priority separated by commas. + - Here is an example of the contents of a config map file: + + ```cm + rockbands.music.example.io=v2beta1,v2beta2 + orchestras.music.example.io=v2,v3alpha1 + subscriptions.operators.coreos.com=v2,v1 + ``` + +2. Apply config map with + + ```bash + kubectl create configmap enableapigroupversions --from-file=/restoreResourcesVersionPriority -n velero + ``` + +3. See the config map with + + ```bash + kubectl describe configmap enableapigroupversions -n velero + ``` + + The config map should look something like + + ```bash + Name: enableapigroupversions + Namespace: velero + Labels: + Annotations: + + Data + ==== + restoreResourcesVersionPriority: + ---- + rockbands.music.example.io=v2beta1,v2beta2 + orchestras.music.example.io=v2,v3alpha1 + subscriptions.operators.coreos.com=v2,v1 + Events: + ``` + +## Troubleshooting + +1. Refer to the [troubleshooting section](troubleshooting.md) of the docs as the techniques generally apply here as well. +2. The [debug logs](troubleshooting.md/#getting-velero-debug-logs) will contain information on which version was chosen to restore. +3. If no API group version could be found that both exists in the backup tarball file and is supported by the destination cluster, then the following error will be recorded (no need to activate debug level logging): `"error restoring rockbands.music.example.io/rockstars/beatles: the server could not find the requested resource"`. diff --git a/site/content/docs/v1.7/examples.md b/site/content/docs/v1.7/examples.md new file mode 100644 index 000000000..58a35fd5d --- /dev/null +++ b/site/content/docs/v1.7/examples.md @@ -0,0 +1,70 @@ +--- +title: "Examples" +layout: docs +--- + +After you set up the Velero server, you can clone the examples used in the following sections by running the following: +``` +git clone https://github.com/vmware-tanzu/velero.git +cd velero +``` + +## Basic example (without PersistentVolumes) + +1. Start the sample nginx app: + + ```bash + kubectl apply -f examples/nginx-app/base.yaml + ``` + +1. Create a backup: + + ```bash + velero backup create nginx-backup --include-namespaces nginx-example + ``` + +1. Simulate a disaster: + + ```bash + kubectl delete namespaces nginx-example + ``` + + Wait for the namespace to be deleted. + +1. Restore your lost resources: + + ```bash + velero restore create --from-backup nginx-backup + ``` + +## Snapshot example (with PersistentVolumes) + +> NOTE: For Azure, you must run Kubernetes version 1.7.2 or later to support PV snapshotting of managed disks. + +1. Start the sample nginx app: + + ```bash + kubectl apply -f examples/nginx-app/with-pv.yaml + ``` + +1. Create a backup with PV snapshotting: + + ```bash + velero backup create nginx-backup --include-namespaces nginx-example + ``` + +1. Simulate a disaster: + + ```bash + kubectl delete namespaces nginx-example + ``` + + Because the default [reclaim policy][1] for dynamically-provisioned PVs is "Delete", these commands should trigger your cloud provider to delete the disk that backs the PV. Deletion is asynchronous, so this may take some time. **Before continuing to the next step, check your cloud provider to confirm that the disk no longer exists.** + +1. Restore your lost resources: + + ```bash + velero restore create --from-backup nginx-backup + ``` + +[1]: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming diff --git a/site/content/docs/v1.7/how-velero-works.md b/site/content/docs/v1.7/how-velero-works.md new file mode 100644 index 000000000..250ab2dab --- /dev/null +++ b/site/content/docs/v1.7/how-velero-works.md @@ -0,0 +1,87 @@ +--- +title: "How Velero Works" +layout: docs +--- + +Each Velero operation -- on-demand backup, scheduled backup, restore -- is a custom resource, defined with a Kubernetes [Custom Resource Definition (CRD)][20] and stored in [etcd][22]. Velero also includes controllers that process the custom resources to perform backups, restores, and all related operations. + +You can back up or restore all objects in your cluster, or you can filter objects by type, namespace, and/or label. + +Velero is ideal for the disaster recovery use case, as well as for snapshotting your application state, prior to performing system operations on your cluster, like upgrades. + +## On-demand backups + +The **backup** operation: + +1. Uploads a tarball of copied Kubernetes objects into cloud object storage. + +1. Calls the cloud provider API to make disk snapshots of persistent volumes, if specified. + +You can optionally specify backup hooks to be executed during the backup. For example, you might +need to tell a database to flush its in-memory buffers to disk before taking a snapshot. [More about backup hooks][10]. + +Note that cluster backups are not strictly atomic. If Kubernetes objects are being created or edited at the time of backup, they might not be included in the backup. The odds of capturing inconsistent information are low, but it is possible. + +## Scheduled backups + +The **schedule** operation allows you to back up your data at recurring intervals. You can create a scheduled backup at any time, and the first backup is then performed at the schedule's specified interval. These intervals are specified by a Cron expression. + +Velero saves backups created from a schedule with the name `-`, where `` is formatted as *YYYYMMDDhhmmss*. For more information see the [Backup Reference documentation](backup-reference.md). + +## Restores + +The **restore** operation allows you to restore all of the objects and persistent volumes from a previously created backup. You can also restore only a filtered subset of objects and persistent volumes. Velero supports multiple namespace remapping--for example, in a single restore, objects in namespace "abc" can be recreated under namespace "def", and the objects in namespace "123" under "456". + +The default name of a restore is `-`, where `` is formatted as *YYYYMMDDhhmmss*. You can also specify a custom name. A restored object also includes a label with key `velero.io/restore-name` and value ``. + +By default, backup storage locations are created in read-write mode. However, during a restore, you can configure a backup storage location to be in read-only mode, which disables backup creation and deletion for the storage location. This is useful to ensure that no backups are inadvertently created or deleted during a restore scenario. + +You can optionally specify restore hooks to be executed during a restore or after resources are restored. For example, you might need to perform a custom database restore operation before the database application containers start. [More about restore hooks][11]. + +## Backup workflow + +When you run `velero backup create test-backup`: + +1. The Velero client makes a call to the Kubernetes API server to create a `Backup` object. + +1. The `BackupController` notices the new `Backup` object and performs validation. + +1. The `BackupController` begins the backup process. It collects the data to back up by querying the API server for resources. + +1. The `BackupController` makes a call to the object storage service -- for example, AWS S3 -- to upload the backup file. + +By default, `velero backup create` makes disk snapshots of any persistent volumes. You can adjust the snapshots by specifying additional flags. Run `velero backup create --help` to see available flags. Snapshots can be disabled with the option `--snapshot-volumes=false`. + +![19] + +## Backed-up API versions + +Velero backs up resources using the Kubernetes API server's *preferred version* for each group/resource. When restoring a resource, this same API group/version must exist in the target cluster in order for the restore to be successful. + +For example, if the cluster being backed up has a `gizmos` resource in the `things` API group, with group/versions `things/v1alpha1`, `things/v1beta1`, and `things/v1`, and the server's preferred group/version is `things/v1`, then all `gizmos` will be backed up from the `things/v1` API endpoint. When backups from this cluster are restored, the target cluster **must** have the `things/v1` endpoint in order for `gizmos` to be restored. Note that `things/v1` **does not** need to be the preferred version in the target cluster; it just needs to exist. + +## Set a backup to expire + +When you create a backup, you can specify a TTL (time to live) by adding the flag `--ttl `. If Velero sees that an existing backup resource is expired, it removes: + +* The backup resource +* The backup file from cloud object storage +* All PersistentVolume snapshots +* All associated Restores + +The TTL flag allows the user to specify the backup retention period with the value specified in hours, minutes and seconds in the form `--ttl 24h0m0s`. If not specified, a default TTL value of 30 days will be applied. + +## Object storage sync + +Velero treats object storage as the source of truth. It continuously checks to see that the correct backup resources are always present. If there is a properly formatted backup file in the storage bucket, but no corresponding backup resource in the Kubernetes API, Velero synchronizes the information from object storage to Kubernetes. + +This allows restore functionality to work in a cluster migration scenario, where the original backup objects do not exist in the new cluster. + +Likewise, if a backup object exists in Kubernetes but not in object storage, it will be deleted from Kubernetes since the backup tarball no longer exists. + +[10]: backup-hooks.md +[11]: restore-hooks.md +[19]: /docs/main/img/backup-process.png +[20]: https://kubernetes.io/docs/concepts/api-extension/custom-resources/#customresourcedefinitions +[21]: https://kubernetes.io/docs/concepts/api-extension/custom-resources/#custom-controllers +[22]: https://github.com/coreos/etcd diff --git a/site/content/docs/v1.7/image-tagging.md b/site/content/docs/v1.7/image-tagging.md new file mode 100644 index 000000000..64e9310b5 --- /dev/null +++ b/site/content/docs/v1.7/image-tagging.md @@ -0,0 +1,24 @@ +--- +title: "Image tagging policy" +layout: docs +--- + +This document describes Velero's image tagging policy. + +## Released versions + +`velero/velero:` + +Velero follows the [Semantic Versioning](http://semver.org/) standard for releases. Each tag in the `github.com/vmware-tanzu/velero` repository has a matching image, `velero/velero:v1.0.0`. + +### Latest + +`velero/velero:latest` + +The `latest` tag follows the most recently released version of Velero. + +## Development + +`velero/velero:main` + +The `main` tag follows the latest commit to land on the `main` branch. diff --git a/site/content/docs/v1.7/img/README.md b/site/content/docs/v1.7/img/README.md new file mode 100644 index 000000000..85c071c63 --- /dev/null +++ b/site/content/docs/v1.7/img/README.md @@ -0,0 +1 @@ +Some of these diagrams (for instance backup-process.png), have been created on [draw.io](https://www.draw.io), using the "Include a copy of my diagram" option. If you want to make changes to these diagrams, try importing them into draw.io, and you should have access to the original shapes/text that went into the originals. diff --git a/site/content/docs/v1.7/img/backup-process.png b/site/content/docs/v1.7/img/backup-process.png new file mode 100644 index 000000000..7d4f10d56 Binary files /dev/null and b/site/content/docs/v1.7/img/backup-process.png differ diff --git a/site/content/docs/v1.7/img/gv_priority1-caseA.png b/site/content/docs/v1.7/img/gv_priority1-caseA.png new file mode 100644 index 000000000..a1940453f Binary files /dev/null and b/site/content/docs/v1.7/img/gv_priority1-caseA.png differ diff --git a/site/content/docs/v1.7/img/gv_priority1-caseB.png b/site/content/docs/v1.7/img/gv_priority1-caseB.png new file mode 100644 index 000000000..6623f7f81 Binary files /dev/null and b/site/content/docs/v1.7/img/gv_priority1-caseB.png differ diff --git a/site/content/docs/v1.7/img/gv_priority2-caseC.png b/site/content/docs/v1.7/img/gv_priority2-caseC.png new file mode 100644 index 000000000..980c0d5f2 Binary files /dev/null and b/site/content/docs/v1.7/img/gv_priority2-caseC.png differ diff --git a/site/content/docs/v1.7/img/gv_priority3-caseD.png b/site/content/docs/v1.7/img/gv_priority3-caseD.png new file mode 100644 index 000000000..11c9a11ce Binary files /dev/null and b/site/content/docs/v1.7/img/gv_priority3-caseD.png differ diff --git a/site/content/docs/v1.7/img/velero.png b/site/content/docs/v1.7/img/velero.png new file mode 100644 index 000000000..a1937a1dc Binary files /dev/null and b/site/content/docs/v1.7/img/velero.png differ diff --git a/site/content/docs/v1.7/locations.md b/site/content/docs/v1.7/locations.md new file mode 100644 index 000000000..2f54077dd --- /dev/null +++ b/site/content/docs/v1.7/locations.md @@ -0,0 +1,260 @@ +--- +title: "Backup Storage Locations and Volume Snapshot Locations" +layout: docs +--- + +## Overview + +Velero has two custom resources, `BackupStorageLocation` and `VolumeSnapshotLocation`, that are used to configure where Velero backups and their associated persistent volume snapshots are stored. + +A `BackupStorageLocation` is defined as a bucket or a prefix within a bucket under which all Velero data is stored and a set of additional provider-specific fields (AWS region, Azure storage account, etc.). Velero assumes it has control over the location you provide so you should use a dedicated bucket or prefix. If you provide a prefix, then the rest of the bucket is safe to use for multiple purposes. The [API documentation][1] captures the configurable parameters for each in-tree provider. + +A `VolumeSnapshotLocation` is defined entirely by provider-specific fields (AWS region, Azure resource group, Portworx snapshot type, etc.) The [API documentation][2] captures the configurable parameters for each in-tree provider. + +The user can pre-configure one or more possible `BackupStorageLocations` and one or more `VolumeSnapshotLocations`, and can select *at backup creation time* the location in which the backup and associated snapshots should be stored. + +This configuration design enables a number of different use cases, including: + +- Take snapshots of more than one kind of persistent volume in a single Velero backup. For example, in a cluster with both EBS volumes and Portworx volumes +- Have some Velero backups go to a bucket in an eastern USA region, and others go to a bucket in a western USA region, or to a different storage provider +- For volume providers that support it, like Portworx, you can have some snapshots stored locally on the cluster and have others stored in the cloud + +## Limitations / Caveats + +- Velero supports multiple credentials for `BackupStorageLocations`, allowing you to specify the credentials to use with any `BackupStorageLocation`. + However, use of this feature requires support within the plugin for the object storage provider you wish to use. + All [plugins maintained by the Velero team][5] support this feature. + If you are using a plugin from another provider, please check their documentation to determine if this feature is supported. + +- Velero only supports a single set of credentials for `VolumeSnapshotLocations`. + Velero will always use the credentials provided at install time (stored in the `cloud-credentials` secret) for volume snapshots. + +- Volume snapshots are still limited by where your provider allows you to create snapshots. For example, AWS and Azure do not allow you to create a volume snapshot in a different region than where the volume is. If you try to take a Velero backup using a volume snapshot location with a different region than where your cluster's volumes are, the backup will fail. + +- Each Velero backup has one `BackupStorageLocation`, and one `VolumeSnapshotLocation` per volume provider. It is not possible (yet) to send a single Velero backup to multiple backup storage locations simultaneously, or a single volume snapshot to multiple locations simultaneously. However, you can always set up multiple scheduled backups that differ only in the storage locations used if redundancy of backups across locations is important. + +- Cross-provider snapshots are not supported. If you have a cluster with more than one type of volume, like EBS and Portworx, but you only have a `VolumeSnapshotLocation` configured for EBS, then Velero will **only** snapshot the EBS volumes. + +- Restic data is stored under a prefix/subdirectory of the main Velero bucket, and will go into the bucket corresponding to the `BackupStorageLocation` selected by the user at backup creation time. + +- Velero's backups are split into 2 pieces - the metadata stored in object storage, and snapshots/backups of the persistent volume data. Right now, Velero *itself* does not encrypt either of them, instead it relies on the native mechanisms in the object and snapshot systems. A special case is restic, which backs up the persistent volume data at the filesystem level and send it to Velero's object storage. + +- Velero's compression for object metadata is limited, using Golang's tar implementation. In most instances, Kubernetes objects are limited to 1.5MB in size, but many don't approach that, meaning that compression may not be necessary. Note that restic has not yet implemented compression, but does have de-deduplication capabilities. + +## Examples + +Let's look at some examples of how you can use this configuration mechanism to address some common use cases: + +### Take snapshots of more than one kind of persistent volume in a single Velero backup + +During server configuration: + +```shell +velero snapshot-location create ebs-us-east-1 \ + --provider aws \ + --config region=us-east-1 + +velero snapshot-location create portworx-cloud \ + --provider portworx \ + --config type=cloud +``` + +During backup creation: + +```shell +velero backup create full-cluster-backup \ + --volume-snapshot-locations ebs-us-east-1,portworx-cloud +``` + +Alternately, since in this example there's only one possible volume snapshot location configured for each of our two providers (`ebs-us-east-1` for `aws`, and `portworx-cloud` for `portworx`), Velero doesn't require them to be explicitly specified when creating the backup: + +```shell +velero backup create full-cluster-backup +``` + +### Have some Velero backups go to a bucket in an eastern USA region (default), and others go to a bucket in a western USA region + +In this example, two `BackupStorageLocations` will be created within the same account but in different regions. +They will both use the credentials provided at install time and stored in the `cloud-credentials` secret. +If you need to configure unique credentials for each `BackupStorageLocation`, please refer to the [later example][8]. + +During server configuration: + +```shell +velero backup-location create backups-primary \ + --provider aws \ + --bucket velero-backups \ + --config region=us-east-1 \ + --default + +velero backup-location create backups-secondary \ + --provider aws \ + --bucket velero-backups \ + --config region=us-west-1 +``` + +A "default" backup storage location (BSL) is where backups get saved to when no BSL is specified at backup creation time. + +You can change the default backup storage location at any time by setting the `--default` flag using the +`velero backup-location set` command and configure a different location to be the default. + +Examples: + +```shell +velero backup-location set backups-secondary --default +``` + + + +During backup creation: + +```shell +velero backup create full-cluster-backup +``` + +Or: + +```shell +velero backup create full-cluster-alternate-location-backup \ + --storage-location backups-secondary +``` + +### For volume providers that support it (like Portworx), have some snapshots be stored locally on the cluster and have others be stored in the cloud + +During server configuration: + +```shell +velero snapshot-location create portworx-local \ + --provider portworx \ + --config type=local + +velero snapshot-location create portworx-cloud \ + --provider portworx \ + --config type=cloud +``` + +During backup creation: + +```shell +# Note that since in this example you have two possible volume snapshot locations for the Portworx +# provider, you need to explicitly specify which one to use when creating a backup. Alternately, +# you can set the --default-volume-snapshot-locations flag on the `velero server` command (run by +# the Velero deployment) to specify which location should be used for each provider by default, in +# which case you don't need to specify it when creating a backup. +velero backup create local-snapshot-backup \ + --volume-snapshot-locations portworx-local +``` + +Or: + +```shell +velero backup create cloud-snapshot-backup \ + --volume-snapshot-locations portworx-cloud +``` + +### Use a single location + +If you don't have a use case for more than one location, it's still easy to use Velero. Let's assume you're running on AWS, in the `us-west-1` region: + +During server configuration: + +```shell +velero backup-location create backups-primary \ + --provider aws \ + --bucket velero-backups \ + --config region=us-west-1 \ + --default + +velero snapshot-location create ebs-us-west-1 \ + --provider aws \ + --config region=us-west-1 +``` + +During backup creation: + +```shell +# Velero will automatically use your configured backup storage location and volume snapshot location. +# Nothing needs to be specified when creating a backup. +velero backup create full-cluster-backup +``` + +### Create a storage location that uses unique credentials + +It is possible to create additional `BackupStorageLocations` that use their own credentials. +This enables you to save backups to another storage provider or to another account with the storage provider you are already using. + +If you create additional `BackupStorageLocations` without specifying the credentials to use, Velero will use the credentials provided at install time and stored in the `cloud-credentials` secret. +Please see the [earlier example][9] for details on how to create multiple `BackupStorageLocations` that use the same credentials. + +#### Prerequisites +- This feature requires support from the [object storage provider plugin][5] you wish to use. + All plugins maintained by the Velero team support this feature. + If you are using a plugin from another provider, please check their documentation to determine if this is supported. +- The [plugin for the object storage provider][5] you wish to use must be [installed][6]. +- You must create a file with the object storage credentials. Follow the instructions provided by your object storage provider plugin to create this file. + +Once you have installed the necessary plugin and created the credentials file, create a [Kubernetes Secret][7] in the Velero namespace that contains these credentials: + +```shell +kubectl create secret generic -n velero credentials --from-file=bsl= +``` + +This will create a secret named `credentials` with a single key (`bsl`) which contains the contents of your credentials file. +Next, create a `BackupStorageLocation` that uses this Secret by passing the Secret name and key in the `--credential` flag. +When interacting with this `BackupStroageLocation` in the future, Velero will fetch the data from the key within the Secret you provide. + +For example, a new `BackupStorageLocation` with a Secret would be configured as follows: + +```bash +velero backup-location create \ + --provider \ + --bucket \ + --config region= \ + --credential== +``` + +The `BackupStorageLocation` is ready to use when it has the phase `Available`. +You can check the status with the following command: + +```bash +velero backup-location get +``` + +To use this new `BackupStorageLocation` when performing a backup, use the flag `--storage-location ` when running `velero backup create`. +You may also set this new `BackupStorageLocation` as the default with the command `velero backup-location set --default `. + +### Modify the credentials used by an existing storage location + +By default, `BackupStorageLocations` will use the credentials provided at install time and stored in the `cloud-credentials` secret in the Velero namespace. +You can modify these existing credentials by [editing the `cloud-credentials` secret][10], however, these changes will apply to all locations using this secret. +This may be the desired outcome, for example, in the case where you wish to rotate the credentials used for a particular account. + +You can also opt to modify an existing `BackupStorageLocation` such that it uses its own credentials by using the `backup-location set` command. + +If you have a credentials file that you wish to use for a `BackupStorageLocation`, follow the instructions above to create the Secret with that file in the Velero namespace. + +Once you have created the Secret, or have an existing Secret which contains the credentials you wish to use for your `BackupStorageLocation`, set the credential to use as follows: + +```bash +velero backup-location set \ + --credential== +``` + +## Additional Use Cases + +1. If you're using Azure's AKS, you may want to store your volume snapshots outside of the "infrastructure" resource group that is automatically created when you create your AKS cluster. This is possible using a `VolumeSnapshotLocation`, by specifying a `resourceGroup` under the `config` section of the snapshot location. See the [Azure volume snapshot location documentation][3] for details. + +1. If you're using Azure, you may want to store your Velero backups across multiple storage accounts and/or resource groups/subscriptions. This is possible using a `BackupStorageLocation`, by specifying a `storageAccount`, `resourceGroup` and/or `subscriptionId`, respectively, under the `config` section of the backup location. See the [Azure backup storage location documentation][4] for details. + + + +[1]: api-types/backupstoragelocation.md +[2]: api-types/volumesnapshotlocation.md +[3]: https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure/blob/main/volumesnapshotlocation.md +[4]: https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure/blob/main/backupstoragelocation.md +[5]: /plugins +[6]: overview-plugins.md +[7]: https://kubernetes.io/docs/concepts/configuration/secret/ +[8]: #create-a-storage-location-that-uses-unique-credentials +[9]: #have-some-velero-backups-go-to-a-bucket-in-an-eastern-usa-region-default-and-others-go-to-a-bucket-in-a-western-usa-region +[10]: https://kubernetes.io/docs/concepts/configuration/secret/#editing-a-secret diff --git a/site/content/docs/v1.7/maintainers.md b/site/content/docs/v1.7/maintainers.md new file mode 100644 index 000000000..5c7c1f3d6 --- /dev/null +++ b/site/content/docs/v1.7/maintainers.md @@ -0,0 +1,37 @@ +--- +title: "Instructions for Maintainers" +layout: docs +toc: "true" +--- + +There are some guidelines maintainers need to follow. We list them here for quick reference, especially for new maintainers. These guidelines apply to all projects in the Velero org, including the main project, the Velero Helm chart, and all other [related repositories](https://github.com/vmware-tanzu/velero/blob/v1.7.0/GOVERNANCE.md#code-repositories). + +Please be sure to also go through the guidance under the entire [Contribute](start-contributing/) section. + +## Reviewing PRs +- PRs require 2 approvals before it is mergeable. +- The second reviewer usually merges the PR (if you notice a PR open for a while and with 2 approvals, go ahead and merge it!) +- As you review a PR that is not yet ready to merge, please check if the "request review" needs to be refreshed for any reviewer (this is better than @mention at them) +- Refrain from @mention other maintainers to review the PR unless it is an immediate need. All maintainers already get notified through the automated add to the "request review". If it is an urgent need, please add a helpful message as to why it is so people can properly prioritize work. +- There is no need to manually request reviewers: after the PR is created, all maintainers will be automatically added to the list (note: feel free to remove people if they are on PTO, etc). +- Be familiar with the [lazy consensus](https://github.com/vmware-tanzu/velero/blob/v1.7.0/GOVERNANCE.md#lazy-consensus) policy for the project. + +Some tips for doing reviews: +- There are some [code standards and general guidelines](https://velero.io/docs/v1.7/code-standards) we aim for +- We have [guidelines for writing and reviewing documentation](https://velero.io/docs/v1.7/style-guide/) +- When reviewing a design document, ensure it follows [our format and guidelines]( https://github.com/vmware-tanzu/velero/blob/v1.7.0/design/_template.md). Also, when reviewing a PR that implements a previously accepted design, ensure the associated design doc is moved to the [design/implemented](https://github.com/vmware-tanzu/velero/tree/main/design/implemented) folder. + + +## Creating a release +Maintainers are expected to create releases for the project. We have parts of the process automated, and full [instructions](release-instructions). +We are working towards automating more the Velero testing, but there is still a need for manual testing as part of the release process. +The manual test cases for release testing are documented [here](./manual-testing). + +## Community support +Maintainers are expected to participate in the community support rotation. We have guidelines for how we handle the [support](support-process). + +## Community engagement +Maintainers for the Velero project are highly involved with the open source community. All the online community meetings for the project are listed in our [community](community) page. + +## How do I become a maintainer? +The Velero project welcomes contributors of all kinds. We are also always on the look out for a high level of engagement from contributors and opportunities to bring in new maintainers. If this is of interest, take a look at how [adding a maintainer](https://github.com/vmware-tanzu/velero/blob/v1.7.0/GOVERNANCE.md#maintainers) is decided. diff --git a/site/content/docs/v1.7/manual-testing.md b/site/content/docs/v1.7/manual-testing.md new file mode 100644 index 000000000..d7d896883 --- /dev/null +++ b/site/content/docs/v1.7/manual-testing.md @@ -0,0 +1,92 @@ +--- +title: "Manual Testing Requirements for Velero" +layout: docs +--- + +Although we have automated unit and end-to-end tests, there is still a need for Velero to undergo manual tests during a release. +This document outlines the manual test operations that Velero needs to correctly perform in order to be considered ready for release. + +## Current test cases + +The following are test cases that are currently performed as part of a Velero release. + +### Install + +- Verify that Velero CRDs are compatible with the earliest and latest versions of Kubernetes that we support: + - Kubernetes v1.12 + - Kubernetes v1.20 + +### Upgrade + +- Verify that Velero upgrade instructions work + +### Basic functionality + +The "Backup and Restore" test cases below describe general backup and restore functionality that needs to run successfully on all the following providers that we maintain plugins for: +- AWS +- GCP +- Microsoft Azure +- VMware vSphere + +#### Backup and Restore + +- Verify that a backup and restore using Volume Snapshots can be performed +- Verify that a backup and restore using Restic can be performed +- Verify that a backup of a cluster workload can be restored in a new cluster +- Verify that an installation using the latest version can be used to restore from backups created with the last 3 versions. + - e.g. Install Velero 1.6 and use it to restore backups from Velero v1.3, v1.4, v1.5. + +### Working with Multiple Providers + +The following are test cases that exercise Velero behaviour when interacting with multiple providers: + +- Verify that a backup and restore to multiple BackupStorageLocations using the same provider with unique credentials can be performed +- Verify that a backup and restore to multiple BackupStorageLocations using different providers with unique credentials can be performed +- Verify that a backup and restore that includes volume snapshots using different providers for the snapshots and object storage can be performed + - e.g. perform a backup and restore using AWS for the VolumeSnapshotLocation and Azure Blob Storage as the BackupStorageLocation + +## Future test cases + +The following are test cases that are not currently performed as part of a Velero release but cases that we will want to cover with future releases. + +### Schedules + +- Verify that schedules create a backup upon creation and create Backup resources at the correct frequency + +### Resource management + +- Verify that deleted backups are successfully removed from object storage +- Verify that backups that have been removed from object storage can still be deleted with `velero delete backup` +- Verify that Volume Snapshots associated with a deleted backup are removed +- Verify that backups that exceed their TTL are deleted +- Verify that existing backups in object storage are synced to Velero + +### Restic repository test cases + +- Verify that restic repository maintenance is performed as the specified interval + +### Backup Hooks + +- Verify that a pre backup hook provided via pod annotation is performed during backup +- Verify that a pre backup hook provided via Backup spec is performed during backup +- Verify that a post backup hook provided via pod annotation is performed during backup +- Verify that a post backup hook provided via Backup spec is performed during backup + +### Restore Hooks + +- Verify that an InitContainer restore hook provided via pod annotation is performed during restore +- Verify that an InitContainer restore hook provided via Restore spec is performed during restore +- Verify that an InitContainer restore hook provided via Restore spec is performed during restore that includes restoring restic volumes +- Verify that an Exec restore hook provided via pod annotation is performed during restore +- Verify that an Exec restore hook provided via Restore spec is performed during restore + + +#### Resource filtering + +- Verify that backups and restores correctly apply the following resource filters: + - `--include-namespaces` + - `--include-resources` + - `--include-cluster-resources` + - `--exclude-namespaces` + - `--exclude-resources` + - `velero.io/exclude-from-backup=true` label diff --git a/site/content/docs/v1.7/migration-case.md b/site/content/docs/v1.7/migration-case.md new file mode 100644 index 000000000..9ad7b809a --- /dev/null +++ b/site/content/docs/v1.7/migration-case.md @@ -0,0 +1,59 @@ +--- +title: "Cluster migration" +layout: docs +--- + +## Using Backups and Restores + +Velero can help you port your resources from one cluster to another, as long as you point each Velero instance to the same cloud object storage location. This scenario assumes that your clusters are hosted by the same cloud provider. **Note that Velero does not natively support the migration of persistent volumes snapshots across cloud providers.** If you would like to migrate volume data between cloud platforms, please enable [restic][2], which will backup volume contents at the filesystem level. + +1. *(Cluster 1)* Assuming you haven't already been checkpointing your data with the Velero `schedule` operation, you need to first back up your entire cluster (replacing `` as desired): + + ``` + velero backup create + ``` + + The default backup retention period, expressed as TTL (time to live), is 30 days (720 hours); you can use the `--ttl ` flag to change this as necessary. See [how velero works][1] for more information about backup expiry. + +1. *(Cluster 2)* Configure `BackupStorageLocations` and `VolumeSnapshotLocations`, pointing to the locations used by *Cluster 1*, using `velero backup-location create` and `velero snapshot-location create`. Make sure to configure the `BackupStorageLocations` as read-only + by using the `--access-mode=ReadOnly` flag for `velero backup-location create`. + +1. *(Cluster 2)* Make sure that the Velero Backup object is created. Velero resources are synchronized with the backup files in cloud storage. + + ``` + velero backup describe + ``` + + **Note:** The default sync interval is 1 minute, so make sure to wait before checking. You can configure this interval with the `--backup-sync-period` flag to the Velero server. + +1. *(Cluster 2)* Once you have confirmed that the right Backup (``) is now present, you can restore everything with: + + ``` + velero restore create --from-backup + ``` + +## Verify Both Clusters + +Check that the second cluster is behaving as expected: + +1. *(Cluster 2)* Run: + + ``` + velero restore get + ``` + +1. Then run: + + ``` + velero restore describe + ``` + +If you encounter issues, make sure that Velero is running in the same namespace in both clusters. + +## Migrating Workloads Across Different Kubernetes Versions + +Migration across clusters that are not running the same version of Kubernetes might be possible, but some factors need to be considered: compatibility of API groups between clusters for each custom resource, and if a Kubernetes version upgrade breaks the compatibility of core/native API groups. For more information about API group versions, please see [EnableAPIGroupVersions](enable-api-group-versions-feature.md). +**Note:** Velero doesn't support restoring into a cluster with a lower Kubernetes version than where the backup was taken. + +[1]: how-velero-works.md#set-a-backup-to-expire +[2]: restic.md diff --git a/site/content/docs/v1.7/namespace.md b/site/content/docs/v1.7/namespace.md new file mode 100644 index 000000000..68561e720 --- /dev/null +++ b/site/content/docs/v1.7/namespace.md @@ -0,0 +1,22 @@ +--- +title: "Run in a non-default namespace" +layout: docs +--- + +The Velero installation and backups by default are run in the `velero` namespace. However, it is possible to use a different namespace. + +## Customize the namespace during install + +Use the `--namespace` flag, in conjunction with the other flags in the `velero install` command (as shown in the [the Velero install instructions][0]). This will inform Velero where to install. + +## Customize the namespace for operational commands + +To have namespace consistency, specify the namespace for all Velero operational commands to be the same as the namespace used to install Velero: + +```bash +velero client config set namespace= +``` + +Alternatively, you may use the global `--namespace` flag with any operational command to tell Velero where to run. + +[0]: basic-install.md#install-the-cli diff --git a/site/content/docs/v1.7/on-premises.md b/site/content/docs/v1.7/on-premises.md new file mode 100644 index 000000000..30ddf98b3 --- /dev/null +++ b/site/content/docs/v1.7/on-premises.md @@ -0,0 +1,95 @@ +--- +title: "On-Premises Environments" +layout: docs +--- + +You can run Velero in an on-premises cluster in different ways depending on your requirements. + +### Selecting an object storage provider + +You must select an object storage backend that Velero can use to store backup data. [Supported providers][0] contains information on various +options that are supported or have been reported to work by users. + +If you do not already have an object storage system, [MinIO][2] is an open-source S3-compatible object storage system that can be installed on-premises and is compatible with Velero. The details of configuring it for production usage are out of scope for Velero's documentation, but an [evaluation install guide][3] using MinIO is provided for convenience. + +### (Optional) Selecting volume snapshot providers + +If you need to back up persistent volume data, you must select a volume backup solution. [Supported providers][0] contains information on the supported options. + +For example, if you use [Portworx][4] for persistent storage, you can install their Velero plugin to get native Portworx snapshots as part of your Velero backups. + +If there is no native snapshot plugin available for your storage platform, you can use Velero's [restic integration][1], which provides a platform-agnostic file-level backup solution for volume data. + +### Air-gapped deployments + +In an air-gapped deployment, there is no access to the public internet, and therefore no access to public container registries. + +In these scenarios, you will need to make sure that you have an internal registry, such as [Harbor][5], installed and the Velero core and plugin images loaded into your internal registry. + +Below you will find instructions to downloading the Velero images to your local machine, tagging them, then uploading them to your custom registry. + +#### Preparing the Velero image + +First, download the Velero image, tag it for the your private registry, then upload it into the registry so that it can be pulled by your cluster. + +```bash +PRIVATE_REG= +VELERO_VERSION= + +docker pull velero/velero:$VELERO_VERSION +docker tag velero/velero:$VELERO_VERSION $PRIVATE_REG/velero:$VELERO_VERSION +docker push $PRIVATE_REG/velero:$VELERO_VERSION +``` + +#### Preparing plugin images + +Next, repeat these steps for any plugins you may need. This example will use the AWS plugin, but the plugin name should be replaced with the plugins you will need. + +```bash +PRIVATE_REG= +PLUGIN_VERSION= + +docker pull velero/velero-plugin-for-aws:$PLUGIN_VERSION +docker tag velero/velero-plugin-for-aws:$PLUGIN_VERSION $PRIVATE_REG/velero-plugin-for-aws:$PLUGIN_VERSION +docker push $PRIVATE_REG/velero-plugin-for-aws:$PLUGIN_VERSION +``` + +#### Preparing the restic helper image (optional) + +If you are using restic, you will also need to upload the restic helper image. + +```bash +PRIVATE_REG= +VELERO_VERSION= + +docker pull velero/velero-restic-restore-helper:$VELERO_VERSION +docker tag velero/velero-restic-restore-helper:$VELERO_VERSION $PRIVATE_REG/velero-restic-restore-helper:$VELERO_VERSION +docker push $PRIVATE_REG/velero-restic-restore-helper:$VELERO_VERSION +``` + +#### Pulling specific architecture images (optional) + +Velero uses Docker manifests for its images, allowing Docker to pull the image needed based on your client machine's architecture. + +If you need to pull a specific image, you should replace the `velero/velero` image with the specific architecture image, such as `velero/velero-arm`. + +To see an up-to-date list of architectures, be sure to enable Docker experimental features and use `docker manifest inspect velero/velero` (or whichever image you're interested in), and join the architecture string to the end of the image name with `-`. + +#### Installing Velero + +By default, `velero install` will use the public `velero/velero` image. When using an air-gapped deployment, use your private registry's image for Velero and your private registry's images for any plugins. + +```bash +velero install \ + --image=$PRIVATE_REG/velero:$VELERO_VERSION \ + --plugins=$PRIVATE_REG/velero-plugin-for-aws:$PLUGIN_VERSION \ +<....> +``` + + +[0]: supported-providers.md +[1]: restic.md +[2]: https://min.io +[3]: contributions/minio.md +[4]: https://portworx.com +[5]: https://goharbor.io/ diff --git a/site/content/docs/v1.7/output-file-format.md b/site/content/docs/v1.7/output-file-format.md new file mode 100644 index 000000000..8d4823505 --- /dev/null +++ b/site/content/docs/v1.7/output-file-format.md @@ -0,0 +1,224 @@ +--- +title: "Output file format" +layout: docs +--- + +A backup is a gzip-compressed tar file whose name matches the Backup API resource's `metadata.name` (what is specified during `velero backup create `). + +In cloud object storage, each backup file is stored in its own subdirectory in the bucket specified in the Velero server configuration. This subdirectory includes an additional file called `velero-backup.json`. The JSON file lists all information about your associated Backup resource, including any default values. This gives you a complete historical record of the backup configuration. The JSON file also specifies `status.version`, which corresponds to the output file format. + +The directory structure in your cloud storage looks something like: + +``` +rootBucket/ + backup1234/ + velero-backup.json + backup1234.tar.gz +``` + +## Example backup JSON file + +```json +{ + "kind": "Backup", + "apiVersion": "velero.io/v1", + "metadata": { + "name": "test-backup", + "namespace": "velero", + "selfLink": "/apis/velero.io/v1/namespaces/velero/backups/test-backup", + "uid": "a12345cb-75f5-11e7-b4c2-abcdef123456", + "resourceVersion": "337075", + "creationTimestamp": "2017-07-31T13:39:15Z" + }, + "spec": { + "includedNamespaces": [ + "*" + ], + "excludedNamespaces": null, + "includedResources": [ + "*" + ], + "excludedResources": null, + "labelSelector": null, + "snapshotVolumes": true, + "ttl": "24h0m0s" + }, + "status": { + "version": 1, + "formatVersion": "1.1.0", + "expiration": "2017-08-01T13:39:15Z", + "phase": "Completed", + "volumeBackups": { + "pvc-e1e2d345-7583-11e7-b4c2-abcdef123456": { + "snapshotID": "snap-04b1a8e11dfb33ab0", + "type": "gp2", + "iops": 100 + } + }, + "validationErrors": null + } +} +``` +Note that this file includes detailed info about your volume snapshots in the `status.volumeBackups` field, which can be helpful if you want to manually check them in your cloud provider GUI. + +## Output File Format Versioning + +The Velero output file format is intended to be relatively stable, but may change over time to support new features. + +To accommodate this, Velero follows [Semantic Versioning](http://semver.org/) for the file format version. + +Minor and patch versions will indicate backwards-compatible changes that previous versions of Velero can restore, including new directories or files. + +A major version would indicate that a version of Velero older than the version that created the backup could not restore it, usually because of moved or renamed directories or files. + +Major versions of the file format will be incremented with major version releases of Velero. +However, a major version release of Velero does not necessarily mean that the backup format version changed - Velero 3.0 could still use backup file format 2.0, as an example. + +## Versions + +### File Format Version: 1.1 (Current) + +Version 1.1 added support of API groups versions as part of the backup. Previously, only the preferred version of each API groups was backed up. Each resource has one or more sub-directories: one sub-directory for each supported version of the API group. The preferred version API Group of each resource has the suffix "-preferredversion" as part of the sub-directory name. For backward compatibility, we kept the classic directory structure without the API group version, which sits on the same level as the API group sub-directory versions. + +By default, only the preferred API group of each resource is backed up. To take a backup of all API group versions, you need to run the Velero server with the `--features=EnableAPIGroupVersions` feature flag. This is an experimental flag and the restore logic to handle multiple API group versions is documented at [EnableAPIGroupVersions](enable-api-group-versions-feature.md). + +When unzipped, a typical backup directory (`backup1234.tar.gz`) taken with this file format version looks like the following (with the feature flag): + +``` +resources/ + persistentvolumes/ + cluster/ + pv01.json + ... + v1-preferredversion/ + cluster/ + pv01.json + ... + configmaps/ + namespaces/ + namespace1/ + myconfigmap.json + ... + namespace2/ + ... + v1-preferredversion/ + namespaces/ + namespace1/ + myconfigmap.json + ... + namespace2/ + ... + pods/ + namespaces/ + namespace1/ + mypod.json + ... + namespace2/ + ... + v1-preferredversion/ + namespaces/ + namespace1/ + mypod.json + ... + namespace2/ + ... + jobs.batch/ + namespaces/ + namespace1/ + awesome-job.json + ... + namespace2/ + ... + v1-preferredversion/ + namespaces/ + namespace1/ + awesome-job.json + ... + namespace2/ + ... + deployments/ + namespaces/ + namespace1/ + cool-deployment.json + ... + namespace2/ + ... + v1-preferredversion/ + namespaces/ + namespace1/ + cool-deployment.json + ... + namespace2/ + ... + horizontalpodautoscalers.autoscaling/ + namespaces/ + namespace1/ + hpa-to-the-rescue.json + ... + namespace2/ + ... + v1-preferredversion/ + namespaces/ + namespace1/ + hpa-to-the-rescue.json + ... + namespace2/ + ... + v2beta1/ + namespaces/ + namespace1/ + hpa-to-the-rescue.json + ... + namespace2/ + ... + v2beta2/ + namespaces/ + namespace1/ + hpa-to-the-rescue.json + ... + namespace2/ + ... + + ... +``` + +### File Format Version: 1 + +When unzipped, a typical backup directory (`backup1234.tar.gz`) looks like the following: + +``` +resources/ + persistentvolumes/ + cluster/ + pv01.json + ... + configmaps/ + namespaces/ + namespace1/ + myconfigmap.json + ... + namespace2/ + ... + pods/ + namespaces/ + namespace1/ + mypod.json + ... + namespace2/ + ... + jobs/ + namespaces/ + namespace1/ + awesome-job.json + ... + namespace2/ + ... + deployments/ + namespaces/ + namespace1/ + cool-deployment.json + ... + namespace2/ + ... + ... +``` diff --git a/site/content/docs/v1.7/overview-plugins.md b/site/content/docs/v1.7/overview-plugins.md new file mode 100644 index 000000000..037601d51 --- /dev/null +++ b/site/content/docs/v1.7/overview-plugins.md @@ -0,0 +1,29 @@ +--- +title: "Velero plugin system" +layout: docs +--- + +Velero uses storage provider plugins to integrate with a variety of storage systems to support backup and snapshot operations. + +For server installation, Velero requires that at least one plugin is added (with the `--plugins` flag). The plugin will be either of the type object store or volume snapshotter, or a plugin that contains both. An exception to this is that when the user is not configuring a backup storage location or a snapshot storage location at the time of install, this flag is optional. + +Any plugin can be added after Velero has been installed by using the command `velero plugin add `. + +Example with a dockerhub image: `velero plugin add velero/velero-plugin-for-aws:v1.0.0`. + +In the same way, any plugin can be removed by using the command `velero plugin remove `. + +## Creating a new plugin + +Anyone can add integrations for any platform to provide additional backup and volume storage without modifying the Velero codebase. To write a plugin for a new backup or volume storage platform, take a look at our [example repo][1] and at our documentation for [Custom plugins][2]. + +## Adding a new plugin + +After you publish your plugin on your own repository, open a PR that adds a link to it under the appropriate list of [supported providers][3] page in our documentation. + +You can also add the [`velero-plugin` GitHub Topic][4] to your repo, and it will be shown under the aggregated list of repositories automatically. + +[1]: https://github.com/vmware-tanzu/velero-plugin-example/ +[2]: custom-plugins.md +[3]: supported-providers.md +[4]: https://github.com/topics/velero-plugin diff --git a/site/content/docs/v1.7/plugin-release-instructions.md b/site/content/docs/v1.7/plugin-release-instructions.md new file mode 100644 index 000000000..4323d8c9a --- /dev/null +++ b/site/content/docs/v1.7/plugin-release-instructions.md @@ -0,0 +1,30 @@ +--- +title: Releasing Velero plugins +layout: docs +toc: "true" +--- + +Velero plugins maintained by the core maintainers do not have any shipped binaries, only container images, so there is no need to invoke a GoReleaser script. +Container images are built via a CI job on git push. + +Plugins the Velero core team is responsible include all those listed in [the Velero-supported providers list](supported-providers.md) _except_ the vSphere plugin. + + +## Steps +### Open a PR to prepare the repo +1. Update the README.md file to update the compatibility matrix and `velero install` instructions with the expected version number and open a PR. +1. Determining the version number is based on semantic versioning and whether the plugin uses any newly introduced, changed, or removed methods or variables from Velero. +2. Roll all unreleased changelogs into a new `CHANGELOG-v.md` file and delete the content of the `unreleased` folder. Edit the new changelog file as needed. +### Tag +1. Once the PR is merged, checkout the upstream `main` branch. Your local upstream might be named `upstream` or `origin`, so use this command: `git checkout /main`. +1. Tag the git version - `git tag v`. +1. Push the git tag - `git push --tags ` to trigger the image build. +2. Wait for the container images to build. You may check the progress of the GH action that triggers the image build at `https://github.com/vmware-tanzu//actions` +3. Verify that an image with the new tag is available at `https://hub.docker.com/repository/docker/velero//`. +4. Run the Velero [e2e tests][2] using the new image. Until it is made configurable, you will have to edit the [plugin version][1] in the test. +### Release +1. If all e2e tests pass, go to the GitHub release page of the plugin (`https://github.com/vmware-tanzu//releases`) and manually create a release for the new tag. +1. Copy and paste the content of the new changelog file into the release description field. + +[1]: https://github.com/vmware-tanzu/velero/blob/c8dfd648bbe85db0184ea53296de4220895497e6/test/e2e/velero_utils.go#L27 +[2]: https://github.com/vmware-tanzu/velero/tree/main/test/e2e diff --git a/site/content/docs/v1.7/rbac.md b/site/content/docs/v1.7/rbac.md new file mode 100644 index 000000000..a9c22b9f1 --- /dev/null +++ b/site/content/docs/v1.7/rbac.md @@ -0,0 +1,50 @@ +--- +title: "Run Velero more securely with restrictive RBAC settings" +layout: docs +--- + +By default Velero runs with an RBAC policy of ClusterRole `cluster-admin`. This is to make sure that Velero can back up or restore anything in your cluster. But `cluster-admin` access is wide open -- it gives Velero components access to everything in your cluster. Depending on your environment and your security needs, you should consider whether to configure additional RBAC policies with more restrictive access. + +**Note:** Roles and RoleBindings are associated with a single namespaces, not with an entire cluster. PersistentVolume backups are associated only with an entire cluster. This means that any backups or restores that use a restrictive Role and RoleBinding pair can manage only the resources that belong to the namespace. You do not need a wide open RBAC policy to manage PersistentVolumes, however. You can configure a ClusterRole and ClusterRoleBinding that allow backups and restores only of PersistentVolumes, not of all objects in the cluster. + +For more information about RBAC and access control generally in Kubernetes, see the Kubernetes documentation about [access control][1], [managing service accounts][2], and [RBAC authorization][3]. + +## Set up Roles and RoleBindings + +Here's a sample Role and RoleBinding pair. + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: YOUR_NAMESPACE_HERE + name: ROLE_NAME_HERE + labels: + component: velero +rules: + - apiGroups: + - velero.io + verbs: + - "*" + resources: + - "*" +``` + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ROLEBINDING_NAME_HERE +subjects: + - kind: ServiceAccount + name: YOUR_SERVICEACCOUNT_HERE +roleRef: + kind: Role + name: ROLE_NAME_HERE + apiGroup: rbac.authorization.k8s.io +``` + +[1]: https://kubernetes.io/docs/reference/access-authn-authz/controlling-access/ +[2]: https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/ +[3]: https://kubernetes.io/docs/reference/access-authn-authz/rbac/ +[4]: namespace.md diff --git a/site/content/docs/v1.7/release-instructions.md b/site/content/docs/v1.7/release-instructions.md new file mode 100644 index 000000000..de601c585 --- /dev/null +++ b/site/content/docs/v1.7/release-instructions.md @@ -0,0 +1,161 @@ +--- +title: "Release Instructions" +layout: docs +toc: "true" +--- +This page covers the steps to perform when releasing a new version of Velero. + +## General notes +- Please read the documented variables in each script to understand what they are for and how to properly format their values. +- You will need to have an upstream remote configured to use to the [vmware-tanzu/velero](https://github.com/vmware-tanzu/velero) repository. + You can check this using `git remote -v`. + The release script ([`tag-release.sh`](https://github.com/vmware-tanzu/velero/blob/v1.7.0/hack/release-tools/tag-release.sh)) will use `upstream` as the default remote name if it is not specified using the environment variable `REMOTE`. +- GA release: major and minor releases only. Example: 1.0 (major), 1.5 (minor). +- Pre-releases: Any release leading up to a GA. Example: 1.4.0-beta.1, 1.5.0-rc.1 +- RC releases: Release Candidate, contains everything that is supposed to ship with the GA release. This is still a pre-release. + +## Velero Release Requirements + +Velero is on a "train leaves the station" model for releases. We will generate a release candidate (RC) +at the scheduled time. Multiple release candidates may be generated, depending on if bugs are found during testing. +When testing has passed a release build will be generated. + +### Release Candidate criteria +The release candidate commit must meet the following criteria: + +* No major bugs outstanding +* Unit tests pass +* E2E tests against latest Kubernetes on AWS, vSphere and kind pass + +Once the release has moved to RC, a code freeze is in effect. Only changes needed to release are allowable. + +### Release criteria +In order for a release candidate to be released, it must meet the following criteria: + +* Unit tests pass +* E2E tests against latest K8S and earliest supported K8S on Azure, vSphere, Kind, AWS, GCP +* Manual tests pass (manual tests will be converted to automated testing) + +When bugs are identified by any of these methods, we will determine whether the bug is a release blocker or not and +a fix generated if it is. When release blocker bugs identifies in an release candidate are fixed, another RC will +be generated and the test cycle will restart. + +## Preparing + +### Create release blog post (GA only) +For each major or minor release, create and publish a blog post to let folks know what's new. Please follow these [instructions](how-to-write-and-release-a-blog-post). + +### Changelog and Docs PR +#### Troubleshooting +- If you encounter the error `You don't have enough free space in /var/cache/apt/archives/` when running `make serve-docs`: run `docker system prune`. + +#### Steps +1. If it doesn't already exist: in a branch, create the file `changelogs/CHANGELOG-..md` by copying the most recent one. +1. Update the file `changelogs/CHANGELOG-..md` + - Run `make changelog` to generate a list of all unreleased changes. + - Copy/paste the output into `CHANGELOG-..md`, under the "All Changes" section for the release. + - You *may* choose to tweak formatting on the list of changes by adding code blocks, etc. + - Update links at the top of the file to point to the new release version +1. Update the main `CHANGELOG.md` file to properly reference the release-specific changelog file + - Under "Current release": + - Should contain only the current GA release. + - Under "Development release": + - Should contain only the latest pre-release + - Move any prior pre-release into "Older releases" +1. GA Only: Remove all changelog files from `changelogs/unreleased`. +1. Generate new docs + - Run `make gen-docs`, passing the appropriate variables. Examples: + a) `VELERO_VERSION=v1.5.0-rc.1 NEW_DOCS_VERSION=v1.5.0-rc.1 make gen-docs`. + b) `VELERO_VERSION=v1.5.0 NEW_DOCS_VERSION=v1.5 make gen-docs`). + - Note: `PREVIOUS_DOCS_VERSION=` is optional; when not set, it will default to the latest doc version. +1. Clean up when there is an existing set of pre-release versioned docs for the version you are releasing + - Example: `site/content/docs/v1.5.0-beta.1` exists, and you're releasing `v1.5.0-rc.1` or `v1.5` + - Remove the directory containing the pre-release docs, i.e. `site/content/docs/`. + - Delete the pre-release docs table of contents file, i.e. `site/data/docs/-toc.yml`. + - Remove the pre-release docs table of contents mapping entry from `site/data/toc-mapping.yml`. + - Remove all references to the pre-release docs from `site/config.yml`. +1. Create the "Upgrade to $major.minor" page if it does not already exist ([example](https://velero.io/docs/v1.5/upgrade-to-1.5/)). + If it already exists, update any usage of the previous version string within this file to use the new version string instead ([example](https://github.com/vmware-tanzu/velero/pull/2941/files#diff-d594f8fd0901fed79c39aab4b348193d)). + This needs to be done in both the versioned and the `main` folders. +1. Review and submit PR + - Follow the additional instructions at `site/README-HUGO.md` to complete the docs generation process. + - Do a review of the diffs, and/or run `make serve-docs` and review the site. + - Submit a PR containing the changelog and the version-tagged docs. + +## Velero release +### Notes +- Pre-requisite: PR with the changelog and docs is merged, so that it's included in the release tag. +- This process is the same for both pre-release and GA. +- Refer to the [General notes](general-notes) above for instructions. + +#### Troubleshooting +- If the dry-run fails with random errors, try running it again. + +#### Steps +1. Create a tagged release in dry-run mode + - This won't push anything to GitHub. + - Run `VELERO_VERSION=v1.0.0-rc.1 REMOTE= GITHUB_TOKEN=REDACTED ./hack/release-tools/tag-release.sh`. + - Fix any issue. +1. Create a tagged release and push it to GitHub + - Run `VELERO_VERSION=v1.0.0-rc.1 REMOTE= GITHUB_TOKEN=REDACTED ./hack/release-tools/tag-release.sh publish`. +1. Publish the release + - Navigate to the draft GitHub release at https://github.com/vmware-tanzu/velero/releases and edit the release. + - If this is a patch release (e.g. `v1.4.1`), note that the full `CHANGELOG-1.4.md` contents will be included in the body of the GitHub release. You need to delete the previous releases' content (e.g. `v1.2.0`'s changelog) so that only the latest patch release's changelog shows. + - Do a quick review for formatting. + - **Note:** the `goreleaser` process should have detected if it's a pre-release version and, if so, checked the box at the bottom of the GitHub release page appropriately, but it's always worth double-checking. + - Verify that GitHub has built and pushed all the images (it takes a while): https://github.com/vmware-tanzu/velero/actions + - Verify that the images are on Docker Hub: https://hub.docker.com/r/velero/velero/tags + - Verify that the assets were published to the GitHub release + - Publish the release. +1. Test the release + - By now, the Docker images should have been published. + - Perform a smoke-test - for example: + - Download the CLI from the GitHub release + - Use it to install Velero into a cluster (or manually update an existing deployment to use the new images) + - Verify that `velero version` shows the expected output + - Run a backup/restore and ensure it works + +## Homebrew release (GA only) +These are the steps to update the Velero Homebrew version. + +### Steps +- If you don't already have one, create a [GitHub access token for Homebrew](https://github.com/settings/tokens/new?scopes=gist,public_repo&description=Homebrew) +- Run `export HOMEBREW_GITHUB_API_TOKEN=your_token_here` on your command line to make sure that `brew` can work on GitHub on your behalf. +- Run `hack/release-tools/brew-update.sh`. This script will download the necessary files, do the checks, and invoke the brew helper to submit the PR, which will open in your browser. +- Update Windows Chocolatey version. From a Windows computer, follow the step-by-step instructions to [create the Windows Chocolatey package for Velero CLI](https://github.com/adamrushuk/velero-choco/blob/main/README.md) +- +## Plugins + +To release plugins maintained by the Velero team, follow the [plugin release instructions](plugin-release-instructions.md). + +After the plugin images are built, be sure to update any [e2e tests][3] that use these plugins. + +## How to write and release a blog post +What to include in a release blog: +* Thank all contributors for their involvement in the release. + * Where possible shoutout folks by name or consider spotlighting new maintainers. +* Highlight the themes, or areas of focus, for the release. Some examples of themes are security, bug fixes, feature improvements. See past Velero [release blog posts][1] for more examples. +* Include summaries of new features or workflows introduced in a release. + * This can also include new project initiatives, like a code-of-conduct update. + * Consider creating additional blog posts that go through new features in more detail. Plan to publish additional blogs after the release blog (all blogs don’t have to be publish all at once). + +Release blog post PR: +* Prepare a PR containing the release blog post. Read the [website guidelines][2] for more information on creating a blog post. It's usually easiest to make a copy of the most recent existing post, then replace the content as appropriate. +* You also need to update `site/index.html` to have "Latest Release Information" contain a link to the new post. +* Plan to publish the blog post the same day as the release. + +## Announce a release +Once you are finished doing the release, let the rest of the world know it's available by posting messages in the following places. +1. GA Only: Merge the blog post PR. +1. Velero's Twitter account. Maintainers are encouraged to help spread the word by posting or reposting on social media. +1. Community Slack channel. +1. Google group message. + +What to include: +* Thank all contributors +* A brief list of highlights in the release +* Link to the release blog post, release notes, and/or github release page + +[1]: https://velero.io/blog +[2]: website-guidelines.md +[3]: https://github.com/vmware-tanzu/velero/tree/main/test/e2e diff --git a/site/content/docs/v1.7/release-schedule.md b/site/content/docs/v1.7/release-schedule.md new file mode 100644 index 000000000..ca88383d2 --- /dev/null +++ b/site/content/docs/v1.7/release-schedule.md @@ -0,0 +1,15 @@ +--- +title: "Release Schedule" +layout: docs +toc: "true" +--- + +Definitions borrowed from [the Kubernetes release process document](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-release/release.md#definitions) + +General phases for a Velero release +- Enhancement/Design freeze +- Implementation phase +- Feature freeze & pruning +- Code freeze & prerelease +- Release + diff --git a/site/content/docs/v1.7/resource-filtering.md b/site/content/docs/v1.7/resource-filtering.md new file mode 100644 index 000000000..6c1470e04 --- /dev/null +++ b/site/content/docs/v1.7/resource-filtering.md @@ -0,0 +1,132 @@ +--- +title: "Resource filtering" +layout: docs +--- + +*Filter objects by namespace, type, or labels.* + +Velero includes all objects in a backup or restore when no filtering options are used. + +## Includes + +Only specific resources are included, excluding all others. + +Wildcard takes precedence when both a wildcard and specific resource are included. + +### --include-namespaces + +* Backup a namespace and it's objects. + + ```bash + velero backup create --include-namespaces + ``` + +* Restore two namespaces and their objects. + + ```bash + velero restore create --include-namespaces , + ``` + +### --include-resources + +* Backup all deployments in the cluster. + + ```bash + velero backup create --include-resources deployments + ``` + +* Restore all deployments and configmaps in the cluster. + + ```bash + velero restore create --include-resources deployments,configmaps + ``` + +* Backup the deployments in a namespace. + + ```bash + velero backup create --include-resources deployments --include-namespaces + ``` + +### --include-cluster-resources + + This option can have three possible values: + +* `true`: all cluster-scoped resources are included. + +* `false`: no cluster-scoped resources are included. + +* `nil` ("auto" or not supplied): + + - Cluster-scoped resources are included when backing up or restoring all namespaces. Default: `true`. + + - Cluster-scoped resources are not included when namespace filtering is used. Default: `false`. + + * Some related cluster-scoped resources may still be backed/restored up if triggered by a custom action (for example, PVC->PV) unless `--include-cluster-resources=false`. + +* Backup entire cluster including cluster-scoped resources. + + ```bash + velero backup create + ``` + +* Restore only namespaced resources in the cluster. + + ```bash + velero restore create --include-cluster-resources=false + ``` + +* Backup a namespace and include cluster-scoped resources. + + ```bash + velero backup create --include-namespaces --include-cluster-resources=true + ``` + +### --selector + +* Include resources matching the label selector. + + ```bash + velero backup create --selector = + ``` + + +## Excludes + +Exclude specific resources from the backup. + +Wildcard excludes are ignored. + +### --exclude-namespaces + +* Exclude kube-system from the cluster backup. + + ```bash + velero backup create --exclude-namespaces kube-system + ``` + +* Exclude two namespaces during a restore. + + ```bash + velero restore create --exclude-namespaces , + ``` + +### --exclude-resources + +* Exclude secrets from the backup. + + ```bash + velero backup create --exclude-resources secrets + ``` + +* Exclude secrets and rolebindings. + + ```bash + velero backup create --exclude-resources secrets,rolebindings + ``` + +### velero.io/exclude-from-backup=true + +* Resources with the label `velero.io/exclude-from-backup=true` are not included in backup, even if it contains a matching selector label. + + + diff --git a/site/content/docs/v1.7/restic.md b/site/content/docs/v1.7/restic.md new file mode 100644 index 000000000..8873c1ff5 --- /dev/null +++ b/site/content/docs/v1.7/restic.md @@ -0,0 +1,529 @@ +--- +title: "Restic Integration" +layout: docs +--- + +Velero supports backing up and restoring Kubernetes volumes using a free open-source backup tool called [restic][1]. This support is considered beta quality. Please see the list of [limitations](#limitations) to understand if it fits your use case. + +Velero allows you to take snapshots of persistent volumes as part of your backups if you’re using one of +the supported cloud providers’ block storage offerings (Amazon EBS Volumes, Azure Managed Disks, Google Persistent Disks). +It also provides a plugin model that enables anyone to implement additional object and block storage backends, outside the +main Velero repository. + +The restic integration was added to give you an out-of-the-box solution for backing up and restoring almost any type of Kubernetes volume. This integration is an addition to Velero's capabilities, not a replacement for existing functionality. If you're running on AWS, and taking EBS snapshots as part of your regular Velero backups, there's no need to switch to using restic. However, if you need a volume snapshot plugin for your storage platform, or if you're using EFS, AzureFile, NFS, emptyDir, +local, or any other volume type that doesn't have a native snapshot concept, restic might be for you. + +Restic is not tied to a specific storage platform, which means that this integration also paves the way for future work to enable +cross-volume-type data migrations. + +**NOTE:** hostPath volumes are not supported, but the [local volume type][4] is supported. + +## Setup restic + +### Prerequisites + +- Understand how Velero performs [backups with the restic integration](#how-backup-and-restore-work-with-restic). +- [Download][3] the latest Velero release. +- Kubernetes v1.12.0 and later. Velero's restic integration requires the Kubernetes [MountPropagation feature][6], which is enabled by default in Kubernetes v1.12.0 and later. + +### Install restic + +To install restic, use the `--use-restic` flag in the `velero install` command. See the [install overview][2] for more details on other flags for the install command. + +``` +velero install --use-restic +``` + +When using restic on a storage provider that doesn't have Velero support for snapshots, the `--use-volume-snapshots=false` flag prevents an unused `VolumeSnapshotLocation` from being created on installation. + +### Configure restic DaemonSet spec + +After installation, some PaaS/CaaS platforms based on Kubernetes also require modifications the restic DaemonSet spec. The steps in this section are only needed if you are installing on RancherOS, OpenShift, VMware Tanzu Kubernetes Grid Integrated Edition (formerly VMware Enterprise PKS), or Microsoft Azure. + + +**RancherOS** + + +Update the host path for volumes in the restic DaemonSet in the Velero namespace from `/var/lib/kubelet/pods` to `/opt/rke/var/lib/kubelet/pods`. + +```yaml +hostPath: + path: /var/lib/kubelet/pods +``` + +to + +```yaml +hostPath: + path: /opt/rke/var/lib/kubelet/pods +``` + + +**OpenShift** + + +To mount the correct hostpath to pods volumes, run the restic pod in `privileged` mode. + +1. Add the `velero` ServiceAccount to the `privileged` SCC: + + ``` + $ oc adm policy add-scc-to-user privileged -z velero -n velero + ``` + +2. For OpenShift version >= `4.1`, modify the DaemonSet yaml to request a privileged mode: + + ```diff + @@ -67,3 +67,5 @@ spec: + value: /credentials/cloud + - name: VELERO_SCRATCH_DIR + value: /scratch + + securityContext: + + privileged: true + ``` + + or + + ```shell + oc patch ds/restic \ + --namespace velero \ + --type json \ + -p '[{"op":"add","path":"/spec/template/spec/containers/0/securityContext","value": { "privileged": true}}]' + ``` + +3. For OpenShift version < `4.1`, modify the DaemonSet yaml to request a privileged mode and mount the correct hostpath to pods volumes. + + ```diff + @@ -35,7 +35,7 @@ spec: + secretName: cloud-credentials + - name: host-pods + hostPath: + - path: /var/lib/kubelet/pods + + path: /var/lib/origin/openshift.local.volumes/pods + - name: scratch + emptyDir: {} + containers: + @@ -67,3 +67,5 @@ spec: + value: /credentials/cloud + - name: VELERO_SCRATCH_DIR + value: /scratch + + securityContext: + + privileged: true + ``` + + or + + ```shell + oc patch ds/restic \ + --namespace velero \ + --type json \ + -p '[{"op":"add","path":"/spec/template/spec/containers/0/securityContext","value": { "privileged": true}}]' + + oc patch ds/restic \ + --namespace velero \ + --type json \ + -p '[{"op":"replace","path":"/spec/template/spec/volumes/0/hostPath","value": { "path": "/var/lib/origin/openshift.local.volumes/pods"}}]' + ``` + + +If restic is not running in a privileged mode, it will not be able to access pods volumes within the mounted hostpath directory because of the default enforced SELinux mode configured in the host system level. You can [create a custom SCC](https://docs.openshift.com/container-platform/3.11/admin_guide/manage_scc.html) to relax the security in your cluster so that restic pods are allowed to use the hostPath volume plug-in without granting them access to the `privileged` SCC. + +By default a userland openshift namespace will not schedule pods on all nodes in the cluster. + +To schedule on all nodes the namespace needs an annotation: + +``` +oc annotate namespace openshift.io/node-selector="" +``` + +This should be done before velero installation. + +Or the ds needs to be deleted and recreated: + +``` +oc get ds restic -o yaml -n > ds.yaml +oc annotate namespace openshift.io/node-selector="" +oc create -n -f ds.yaml +``` + +**VMware Tanzu Kubernetes Grid Integrated Edition (formerly VMware Enterprise PKS)** + +You need to enable the `Allow Privileged` option in your plan configuration so that restic is able to mount the hostpath. + +The hostPath should be changed from `/var/lib/kubelet/pods` to `/var/vcap/data/kubelet/pods` + +```yaml +hostPath: + path: /var/vcap/data/kubelet/pods +``` + + +**Microsoft Azure** + +If you are using [Azure Files][8], you need to add `nouser_xattr` to your storage class's `mountOptions`. See [this restic issue][9] for more details. + +You can use the following command to patch the storage class: + +```bash +kubectl patch storageclass/ \ + --type json \ + --patch '[{"op":"add","path":"/mountOptions/-","value":"nouser_xattr"}]' +``` + + +## To back up + +Velero supports two approaches of discovering pod volumes that need to be backed up using restic: + +- Opt-in approach: Where every pod containing a volume to be backed up using restic must be annotated with the volume's name. +- Opt-out approach: Where all pod volumes are backed up using restic, with the ability to opt-out any volumes that should not be backed up. + +The following sections provide more details on the two approaches. + +### Using the opt-out approach + +In this approach, Velero will back up all pod volumes using restic with the exception of: + +- Volumes mounting the default service account token, kubernetes secrets, and config maps +- Hostpath volumes + +It is possible to exclude volumes from being backed up using the `backup.velero.io/backup-volumes-excludes` annotation on the pod. + +Instructions to back up using this approach are as follows: + +1. Run the following command on each pod that contains volumes that should **not** be backed up using restic + + ```bash + kubectl -n YOUR_POD_NAMESPACE annotate pod/YOUR_POD_NAME backup.velero.io/backup-volumes-excludes=YOUR_VOLUME_NAME_1,YOUR_VOLUME_NAME_2,... + ``` + where the volume names are the names of the volumes in the pod spec. + + For example, in the following pod: + + ```yaml + apiVersion: v1 + kind: Pod + metadata: + name: app1 + namespace: sample + spec: + containers: + - image: k8s.gcr.io/test-webserver + name: test-webserver + volumeMounts: + - name: pvc1-vm + mountPath: /volume-1 + - name: pvc2-vm + mountPath: /volume-2 + volumes: + - name: pvc1-vm + persistentVolumeClaim: + claimName: pvc1 + - name: pvc2-vm + claimName: pvc2 + ``` + to exclude restic backup of volume `pvc1-vm`, you would run: + + ```bash + kubectl -n sample annotate pod/app1 backup.velero.io/backup-volumes-excludes=pvc1-vm + ``` + +2. Take a Velero backup: + + ```bash + velero backup create BACKUP_NAME --default-volumes-to-restic OTHER_OPTIONS + ``` + + The above steps uses the opt-out approach on a per backup basis. + + Alternatively, this behavior may be enabled on all velero backups running the `velero install` command with the `--default-volumes-to-restic` flag. Refer [install overview][11] for details. + +3. When the backup completes, view information about the backups: + + ```bash + velero backup describe YOUR_BACKUP_NAME + ``` + ```bash + kubectl -n velero get podvolumebackups -l velero.io/backup-name=YOUR_BACKUP_NAME -o yaml + ``` + +### Using opt-in pod volume backup + +Velero, by default, uses this approach to discover pod volumes that need to be backed up using restic, where every pod containing a volume to be backed up using restic must be annotated with the volume's name. + +Instructions to back up using this approach are as follows: + +1. Run the following for each pod that contains a volume to back up: + + ```bash + kubectl -n YOUR_POD_NAMESPACE annotate pod/YOUR_POD_NAME backup.velero.io/backup-volumes=YOUR_VOLUME_NAME_1,YOUR_VOLUME_NAME_2,... + ``` + + where the volume names are the names of the volumes in the pod spec. + + For example, for the following pod: + + ```yaml + apiVersion: v1 + kind: Pod + metadata: + name: sample + namespace: foo + spec: + containers: + - image: k8s.gcr.io/test-webserver + name: test-webserver + volumeMounts: + - name: pvc-volume + mountPath: /volume-1 + - name: emptydir-volume + mountPath: /volume-2 + volumes: + - name: pvc-volume + persistentVolumeClaim: + claimName: test-volume-claim + - name: emptydir-volume + emptyDir: {} + ``` + + You'd run: + + ```bash + kubectl -n foo annotate pod/sample backup.velero.io/backup-volumes=pvc-volume,emptydir-volume + ``` + + This annotation can also be provided in a pod template spec if you use a controller to manage your pods. + +1. Take a Velero backup: + + ```bash + velero backup create NAME OPTIONS... + ``` + +1. When the backup completes, view information about the backups: + + ```bash + velero backup describe YOUR_BACKUP_NAME + ``` + ```bash + kubectl -n velero get podvolumebackups -l velero.io/backup-name=YOUR_BACKUP_NAME -o yaml + ``` + +## To restore + +Regardless of how volumes are discovered for backup using restic, the process of restoring remains the same. + +1. Restore from your Velero backup: + + ```bash + velero restore create --from-backup BACKUP_NAME OPTIONS... + ``` + +1. When the restore completes, view information about your pod volume restores: + + ```bash + velero restore describe YOUR_RESTORE_NAME + ``` + ```bash + kubectl -n velero get podvolumerestores -l velero.io/restore-name=YOUR_RESTORE_NAME -o yaml + ``` + +## Limitations + +- `hostPath` volumes are not supported. [Local persistent volumes][4] are supported. +- Those of you familiar with [restic][1] may know that it encrypts all of its data. Velero uses a static, +common encryption key for all restic repositories it creates. **This means that anyone who has access to your +bucket can decrypt your restic backup data**. Make sure that you limit access to the restic bucket +appropriately. +- An incremental backup chain will be maintained across pod reschedules for PVCs. However, for pod volumes that are *not* +PVCs, such as `emptyDir` volumes, when a pod is deleted/recreated (for example, by a ReplicaSet/Deployment), the next backup of those +volumes will be full rather than incremental, because the pod volume's lifecycle is assumed to be defined by its pod. +- Restic scans each file in a single thread. This means that large files (such as ones storing a database) will take a long time to scan for data deduplication, even if the actual +difference is small. +- If you plan to use the Velero restic integration to backup 100GB of data or more, you may need to [customize the resource limits](/docs/main/customize-installation/#customize-resource-requests-and-limits) to make sure backups complete successfully. +- Velero's restic integration backs up data from volumes by accessing the node's filesystem, on which the pod is running. For this reason, restic integration can only backup volumes that are mounted by a pod and not directly from the PVC. + +## Customize Restore Helper Container + +Velero uses a helper init container when performing a restic restore. By default, the image for this container is `velero/velero-restic-restore-helper:`, +where `VERSION` matches the version/tag of the main Velero image. You can customize the image that is used for this helper by creating a ConfigMap in the Velero namespace with +the alternate image. + +In addition, you can customize the resource requirements for the init container, should you need. + +The ConfigMap must look like the following: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + # any name can be used; Velero uses the labels (below) + # to identify it rather than the name + name: restic-restore-action-config + # must be in the velero namespace + namespace: velero + # the below labels should be used verbatim in your + # ConfigMap. + labels: + # this value-less label identifies the ConfigMap as + # config for a plugin (i.e. the built-in restic restore + # item action plugin) + velero.io/plugin-config: "" + # this label identifies the name and kind of plugin + # that this ConfigMap is for. + velero.io/restic: RestoreItemAction +data: + # The value for "image" can either include a tag or not; + # if the tag is *not* included, the tag from the main Velero + # image will automatically be used. + image: myregistry.io/my-custom-helper-image[:OPTIONAL_TAG] + + # "cpuRequest" sets the request.cpu value on the restic init containers during restore. + # If not set, it will default to "100m". A value of "0" is treated as unbounded. + cpuRequest: 200m + + # "memRequest" sets the request.memory value on the restic init containers during restore. + # If not set, it will default to "128Mi". A value of "0" is treated as unbounded. + memRequest: 128Mi + + # "cpuLimit" sets the request.cpu value on the restic init containers during restore. + # If not set, it will default to "100m". A value of "0" is treated as unbounded. + cpuLimit: 200m + + # "memLimit" sets the request.memory value on the restic init containers during restore. + # If not set, it will default to "128Mi". A value of "0" is treated as unbounded. + memLimit: 128Mi + + # "secCtxRunAsUser sets the securityContext.runAsUser value on the restic init containers during restore." + secCtxRunAsUser: 1001 + + # "secCtxRunAsGroup sets the securityContext.runAsGroup value on the restic init containers during restore." + secCtxRunAsGroup: 999 +``` + +## Troubleshooting + +Run the following checks: + +Are your Velero server and daemonset pods running? + +```bash +kubectl get pods -n velero +``` + +Does your restic repository exist, and is it ready? + +```bash +velero restic repo get + +velero restic repo get REPO_NAME -o yaml +``` + +Are there any errors in your Velero backup/restore? + +```bash +velero backup describe BACKUP_NAME +velero backup logs BACKUP_NAME + +velero restore describe RESTORE_NAME +velero restore logs RESTORE_NAME +``` + +What is the status of your pod volume backups/restores? + +```bash +kubectl -n velero get podvolumebackups -l velero.io/backup-name=BACKUP_NAME -o yaml + +kubectl -n velero get podvolumerestores -l velero.io/restore-name=RESTORE_NAME -o yaml +``` + +Is there any useful information in the Velero server or daemon pod logs? + +```bash +kubectl -n velero logs deploy/velero +kubectl -n velero logs DAEMON_POD_NAME +``` + +**NOTE**: You can increase the verbosity of the pod logs by adding `--log-level=debug` as an argument +to the container command in the deployment/daemonset pod template spec. + +## How backup and restore work with restic + +Velero has three custom resource definitions and associated controllers: + +- `ResticRepository` - represents/manages the lifecycle of Velero's [restic repositories][5]. Velero creates +a restic repository per namespace when the first restic backup for a namespace is requested. The controller +for this custom resource executes restic repository lifecycle commands -- `restic init`, `restic check`, +and `restic prune`. + + You can see information about your Velero restic repositories by running `velero restic repo get`. + +- `PodVolumeBackup` - represents a restic backup of a volume in a pod. The main Velero backup process creates +one or more of these when it finds an annotated pod. Each node in the cluster runs a controller for this +resource (in a daemonset) that handles the `PodVolumeBackups` for pods on that node. The controller executes +`restic backup` commands to backup pod volume data. + +- `PodVolumeRestore` - represents a restic restore of a pod volume. The main Velero restore process creates one +or more of these when it encounters a pod that has associated restic backups. Each node in the cluster runs a +controller for this resource (in the same daemonset as above) that handles the `PodVolumeRestores` for pods +on that node. The controller executes `restic restore` commands to restore pod volume data. + +### Backup + +1. Based on configuration, the main Velero backup process uses the opt-in or opt-out approach to check each pod that it's backing up for the volumes to be backed up using restic. +1. When found, Velero first ensures a restic repository exists for the pod's namespace, by: + - checking if a `ResticRepository` custom resource already exists + - if not, creating a new one, and waiting for the `ResticRepository` controller to init/check it +1. Velero then creates a `PodVolumeBackup` custom resource per volume listed in the pod annotation +1. The main Velero process now waits for the `PodVolumeBackup` resources to complete or fail +1. Meanwhile, each `PodVolumeBackup` is handled by the controller on the appropriate node, which: + - has a hostPath volume mount of `/var/lib/kubelet/pods` to access the pod volume data + - finds the pod volume's subdirectory within the above volume + - runs `restic backup` + - updates the status of the custom resource to `Completed` or `Failed` +1. As each `PodVolumeBackup` finishes, the main Velero process adds it to the Velero backup in a file named `-podvolumebackups.json.gz`. This file gets uploaded to object storage alongside the backup tarball. It will be used for restores, as seen in the next section. + +### Restore + +1. The main Velero restore process checks each existing `PodVolumeBackup` custom resource in the cluster to backup from. +1. For each `PodVolumeBackup` found, Velero first ensures a restic repository exists for the pod's namespace, by: + - checking if a `ResticRepository` custom resource already exists + - if not, creating a new one, and waiting for the `ResticRepository` controller to init/check it (note that + in this case, the actual repository should already exist in object storage, so the Velero controller will simply + check it for integrity) +1. Velero adds an init container to the pod, whose job is to wait for all restic restores for the pod to complete (more +on this shortly) +1. Velero creates the pod, with the added init container, by submitting it to the Kubernetes API +1. Velero creates a `PodVolumeRestore` custom resource for each volume to be restored in the pod +1. The main Velero process now waits for each `PodVolumeRestore` resource to complete or fail +1. Meanwhile, each `PodVolumeRestore` is handled by the controller on the appropriate node, which: + - has a hostPath volume mount of `/var/lib/kubelet/pods` to access the pod volume data + - waits for the pod to be running the init container + - finds the pod volume's subdirectory within the above volume + - runs `restic restore` + - on success, writes a file into the pod volume, in a `.velero` subdirectory, whose name is the UID of the Velero restore + that this pod volume restore is for + - updates the status of the custom resource to `Completed` or `Failed` +1. The init container that was added to the pod is running a process that waits until it finds a file +within each restored volume, under `.velero`, whose name is the UID of the Velero restore being run +1. Once all such files are found, the init container's process terminates successfully and the pod moves +on to running other init containers/the main containers. + +## 3rd party controllers + +### Monitor backup annotation + +Velero does not provide a mechanism to detect persistent volume claims that are missing the restic backup annotation. + +To solve this, a controller was written by Thomann Bits&Beats: [velero-pvc-watcher][7] + +[1]: https://github.com/restic/restic +[2]: customize-installation.md#enable-restic-integration +[3]: https://github.com/vmware-tanzu/velero/releases/ +[4]: https://kubernetes.io/docs/concepts/storage/volumes/#local +[5]: http://restic.readthedocs.io/en/latest/100_references.html#terminology +[6]: https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation +[7]: https://github.com/bitsbeats/velero-pvc-watcher +[8]: https://docs.microsoft.com/en-us/azure/aks/azure-files-dynamic-pv +[9]: https://github.com/restic/restic/issues/1800 +[11]: customize-installation.md#default-pod-volume-backup-to-restic + diff --git a/site/content/docs/v1.7/restore-hooks.md b/site/content/docs/v1.7/restore-hooks.md new file mode 100644 index 000000000..0ec36c5d7 --- /dev/null +++ b/site/content/docs/v1.7/restore-hooks.md @@ -0,0 +1,261 @@ +--- +title: "Restore Hooks" +layout: docs +--- + +Velero supports Restore Hooks, custom actions that can be executed during or after the restore process. There are two kinds of Restore Hooks: + +1. InitContainer Restore Hooks: These will add init containers into restored pods to perform any necessary setup before the application containers of the restored pod can start. +1. Exec Restore Hooks: These can be used to execute custom commands or scripts in containers of a restored Kubernetes pod. + +## InitContainer Restore Hooks + +Use an `InitContainer` hook to add init containers into a pod before it's restored. You can use these init containers to run any setup needed for the pod to resume running from its backed-up state. +The InitContainer added by the restore hook will be the first init container in the `podSpec` of the restored pod. +In the case where the pod had volumes backed up using restic, then, the restore hook InitContainer will be added after the `restic-wait` InitContainer. + +NOTE: This ordering can be altered by any mutating webhooks that may be installed in the cluster. + +There are two ways to specify `InitContainer` restore hooks: +1. Specifying restore hooks in annotations +1. Specifying restore hooks in the restore spec + +### Specifying Restore Hooks As Pod Annotations + +Below are the annotations that can be added to a pod to specify restore hooks: +* `init.hook.restore.velero.io/container-image` + * The container image for the init container to be added. +* `init.hook.restore.velero.io/container-name` + * The name for the init container that is being added. +* `init.hook.restore.velero.io/command` + * This is the `ENTRYPOINT` for the init container being added. This command is not executed within a shell and the container image's `ENTRYPOINT` is used if this is not provided. + +#### Example + +Use the below commands to add annotations to the pods before taking a backup. + +```bash +$ kubectl annotate pod -n \ + init.hook.restore.velero.io/container-name=restore-hook \ + init.hook.restore.velero.io/container-image=alpine:latest \ + init.hook.restore.velero.io/command='["/bin/ash", "-c", "date"]' +``` + +With the annotation above, Velero will add the following init container to the pod when it's restored. + +```json +{ + "command": [ + "/bin/ash", + "-c", + "date" + ], + "image": "alpine:latest", + "imagePullPolicy": "Always", + "name": "restore-hook" + ... +} +``` + +### Specifying Restore Hooks In Restore Spec + +Init container restore hooks can also be specified using the `RestoreSpec`. +Please refer to the documentation on the [Restore API Type][1] for how to specify hooks in the Restore spec. + +#### Example + +Below is an example of specifying restore hooks in `RestoreSpec` + +```yaml +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: r2 + namespace: velero +spec: + backupName: b2 + excludedResources: + ... + includedNamespaces: + - '*' + hooks: + resources: + - name: restore-hook-1 + includedNamespaces: + - app + postHooks: + - init: + initContainers: + - name: restore-hook-init1 + image: alpine:latest + volumeMounts: + - mountPath: /restores/pvc1-vm + name: pvc1-vm + command: + - /bin/ash + - -c + - echo -n "FOOBARBAZ" >> /restores/pvc1-vm/foobarbaz + - name: restore-hook-init2 + image: alpine:latest + volumeMounts: + - mountPath: /restores/pvc2-vm + name: pvc2-vm + command: + - /bin/ash + - -c + - echo -n "DEADFEED" >> /restores/pvc2-vm/deadfeed +``` + +The `hooks` in the above `RestoreSpec`, when restored, will add two init containers to every pod in the `app` namespace + +```json +{ + "command": [ + "/bin/ash", + "-c", + "echo -n \"FOOBARBAZ\" >> /restores/pvc1-vm/foobarbaz" + ], + "image": "alpine:latest", + "imagePullPolicy": "Always", + "name": "restore-hook-init1", + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/restores/pvc1-vm", + "name": "pvc1-vm" + } + ] + ... +} +``` + +and + +```json +{ + "command": [ + "/bin/ash", + "-c", + "echo -n \"DEADFEED\" >> /restores/pvc2-vm/deadfeed" + ], + "image": "alpine:latest", + "imagePullPolicy": "Always", + "name": "restore-hook-init2", + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/restores/pvc2-vm", + "name": "pvc2-vm" + } + ] + ... +} +``` + +## Exec Restore Hooks + +Use an Exec Restore hook to execute commands in a restored pod's containers after they start. + +There are two ways to specify `Exec` restore hooks: +1. Specifying exec restore hooks in annotations +1. Specifying exec restore hooks in the restore spec + +If a pod has the annotation `post.hook.restore.velero.io/command` then that is the only hook that will be executed in the pod. +No hooks from the restore spec will be executed in that pod. + +### Specifying Exec Restore Hooks As Pod Annotations + +Below are the annotations that can be added to a pod to specify exec restore hooks: +* `post.hook.restore.velero.io/container` + * The container name where the hook will be executed. Defaults to the first container. Optional. +* `post.hook.restore.velero.io/command` + * The command that will be executed in the container. Required. +* `post.hook.restore.velero.io/on-error` + * How to handle execution failures. Valid values are `Fail` and `Continue`. Defaults to `Continue`. With `Continue` mode, execution failures are logged only. With `Fail` mode, no more restore hooks will be executed in any container in any pod and the status of the Restore will be `PartiallyFailed`. Optional. +* `post.hook.restore.velero.io/exec-timeout` + * How long to wait once execution begins. Defaults to 30 seconds. Optional. +* `post.hook.restore.velero.io/wait-timeout` + * How long to wait for a container to become ready. This should be long enough for the container to start plus any preceding hooks in the same container to complete. The wait timeout begins when the container is restored and may require time for the image to pull and volumes to mount. If not set the restore will wait indefinitely. Optional. + +#### Example + +Use the below commands to add annotations to the pods before taking a backup. + +```bash +$ kubectl annotate pod -n \ + post.hook.restore.velero.io/container=postgres \ + post.hook.restore.velero.io/command='["/bin/bash", "-c", "psql < /backup/backup.sql"]' \ + post.hook.restore.velero.io/wait-timeout=5m \ + post.hook.restore.velero.io/exec-timeout=45s \ + post.hook.restore.velero.io/on-error=Continue +``` + +### Specifying Exec Restore Hooks in Restore Spec + +Exec restore hooks can also be specified using the `RestoreSpec`. +Please refer to the documentation on the [Restore API Type][1] for how to specify hooks in the Restore spec. + +#### Multiple Exec Restore Hooks Example + +Below is an example of specifying restore hooks in a `RestoreSpec`. +When using the restore spec it is possible to specify multiple hooks for a single pod, as this example demonstrates. + +All hooks applicable to a single container will be executed sequentially in that container once it starts. +The ordering of hooks executed in a single container follows the order of the restore spec. +In this example, the `pg_isready` hook is guaranteed to run before the `psql` hook because they both apply to the same container and the `pg_isready` hook is defined first. + +If a pod has multiple containers with applicable hooks, all hooks for a single container will be executed before executing hooks in another container. +In this example, if the postgres container starts before the sidecar container, both postgres hooks will run before the hook in the sidecar. +This means the sidecar container may be running for several minutes before its hook is executed. + +Velero guarantees that no two hooks for a single pod are executed in parallel, but hooks executing in different pods may run in parallel. + + +```yaml +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: r2 + namespace: velero +spec: + backupName: b2 + excludedResources: + ... + includedNamespaces: + - '*' + hooks: + resources: + - name: restore-hook-1 + includedNamespaces: + - app + postHooks: + - exec: + execTimeout: 1m + waitTimeout: 5m + onError: Fail + container: postgres + command: + - /bin/bash + - '-c' + - 'while ! pg_isready; do sleep 1; done' + - exec: + container: postgres + waitTimeout: 6m + execTimeout: 1m + command: + - /bin/bash + - '-c' + - 'psql < /backup/backup.sql' + - exec: + container: sidecar + command: + - /bin/bash + - '-c' + - 'date > /start' +``` + +[1]: api-types/restore.md diff --git a/site/content/docs/v1.7/restore-reference.md b/site/content/docs/v1.7/restore-reference.md new file mode 100644 index 000000000..bb14977bc --- /dev/null +++ b/site/content/docs/v1.7/restore-reference.md @@ -0,0 +1,144 @@ +--- +title: "Restore Reference" +layout: docs +--- + +## Restoring Into a Different Namespace + +Velero can restore resources into a different namespace than the one they were backed up from. To do this, use the `--namespace-mappings` flag: + +```bash +velero restore create RESTORE_NAME \ + --from-backup BACKUP_NAME \ + --namespace-mappings old-ns-1:new-ns-1,old-ns-2:new-ns-2 +``` +## What happens when user removes restore objects +A **restore** object represents the restore operation. There are two types of deletion for restore objects: +1. Deleting with **`velero restore delete`**. +This command will delete the custom resource representing it, along with its individual log and results files. But, it will not delete any objects that were created by it from your cluster. +2. Deleting with **`kubectl -n velero delete restore`**. +This command will delete the custom resource representing the restore, but will not delete log/results files from object storage, or any objects that were created during the restore in your cluster. + +## Restore command-line options +To see all commands for restores, run : `velero restore --help` +To see all options associated with a specific command, provide the --help flag to that command. For example, **`velero restore create --help`** shows all options associated with the **create** command. + +To list all options of restore, use **`velero restore --help`** + +```Usage: + velero restore [command] + +Available Commands: + create Create a restore + delete Delete restores + describe Describe restores + get Get restores + logs Get restore logs +``` + +## What happens to NodePorts when restoring Services + +**Auto assigned** NodePorts **deleted** by default and Services get new **auto assigned** nodePorts after restore. + +**Explicitly specified** NodePorts auto detected using **`last-applied-config`** annotation and **preserved** after restore. NodePorts can be explicitly specified as .spec.ports[*].nodePort field on Service definition. + +#### Always Preserve NodePorts + +It is not always possible to set nodePorts explicitly on some big clusters because of operation complexity. Official Kubernetes documents states that preventing port collisions is responsibility of the user when explicitly specifying nodePorts: + +``` +If you want a specific port number, you can specify a value in the `nodePort` field. The control plane will either allocate you that port or report that the API transaction failed. This means that you need to take care of possible port collisions yourself. You also have to use a valid port number, one that's inside the range configured for NodePort use. + +https://kubernetes.io/docs/concepts/services-networking/service/#nodeport +``` + +The clusters which are not explicitly specifying nodePorts still may need to restore original NodePorts in case of disaster. Auto assigned nodePorts most probably defined on Load Balancers which located front side of cluster. Changing all these nodePorts on Load Balancers is another operation complexity after disaster if nodePorts are changed. + +Velero has a flag to let user deciding the preservation of nodePorts. **`velero restore create`** sub command has **`--preserve-nodeports`** flag to **preserve** Service nodePorts **always** regardless of nodePorts **explicitly specified** or **not**. This flag used for preserving the original nodePorts from backup and can be used as **`--preserve-nodeports`** or **`--preserve-nodeports=true`** + +If this flag given and/or set to true, Velero does not remove the nodePorts when restoring Service and tries to use the nodePorts which written on backup. + +Trying to preserve nodePorts may cause **port conflicts** when restoring on situations below: + +- If the nodePort from the backup already allocated on the target cluster then Velero prints error log as shown below and continue to restore operation. + + ``` + time="2020-11-23T12:58:31+03:00" level=info msg="Executing item action for services" logSource="pkg/restore/restore.go:1002" restore=velero/test-with-3-svc-20201123125825 + + time="2020-11-23T12:58:31+03:00" level=info msg="Restoring Services with original NodePort(s)" cmd=_output/bin/linux/amd64/velero logSource="pkg/restore/service_action.go:61" pluginName=velero restore=velero/test-with-3-svc-20201123125825 + + time="2020-11-23T12:58:31+03:00" level=info msg="Attempting to restore Service: hello-service" logSource="pkg/restore/restore.go:1107" restore=velero/test-with-3-svc-20201123125825 + + time="2020-11-23T12:58:31+03:00" level=error msg="error restoring hello-service: Service \"hello-service\" is invalid: spec.ports[0].nodePort: Invalid value: 31536: provided port is already allocated" logSource="pkg/restore/restore.go:1170" restore=velero/test-with-3-svc-20201123125825 + ``` + + + +- If the nodePort from the backup is not in the nodePort range of target cluster then Velero prints error log as below and continue to restore operation. Kubernetes default nodePort range is 30000-32767 but on the example cluster nodePort range is 20000-22767 and tried to restore Service with nodePort 31536 + + ``` + time="2020-11-23T13:09:17+03:00" level=info msg="Executing item action for services" logSource="pkg/restore/restore.go:1002" restore=velero/test-with-3-svc-20201123130915 + + time="2020-11-23T13:09:17+03:00" level=info msg="Restoring Services with original NodePort(s)" cmd=_output/bin/linux/amd64/velero logSource="pkg/restore/service_action.go:61" pluginName=velero restore=velero/test-with-3-svc-20201123130915 + + time="2020-11-23T13:09:17+03:00" level=info msg="Attempting to restore Service: hello-service" logSource="pkg/restore/restore.go:1107" restore=velero/test-with-3-svc-20201123130915 + + time="2020-11-23T13:09:17+03:00" level=error msg="error restoring hello-service: Service \"hello-service\" is invalid: spec.ports[0].nodePort: Invalid value: 31536: provided port is not in the valid range. The range of valid ports is 20000-22767" logSource="pkg/restore/restore.go:1170" restore=velero/test-with-3-svc-20201123130915 + ``` + +## Changing PV/PVC Storage Classes + +Velero can change the storage class of persistent volumes and persistent volume claims during restores. To configure a storage class mapping, create a config map in the Velero namespace like the following: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + # any name can be used; Velero uses the labels (below) + # to identify it rather than the name + name: change-storage-class-config + # must be in the velero namespace + namespace: velero + # the below labels should be used verbatim in your + # ConfigMap. + labels: + # this value-less label identifies the ConfigMap as + # config for a plugin (i.e. the built-in restore item action plugin) + velero.io/plugin-config: "" + # this label identifies the name and kind of plugin + # that this ConfigMap is for. + velero.io/change-storage-class: RestoreItemAction +data: + # add 1+ key-value pairs here, where the key is the old + # storage class name and the value is the new storage + # class name. + : +``` + +## Changing PVC selected-node + +Velero can update the selected-node annotation of persistent volume claim during restores, if selected-node doesn't exist in the cluster then it will remove the selected-node annotation from PersistentVolumeClaim. To configure a node mapping, create a config map in the Velero namespace like the following: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + # any name can be used; Velero uses the labels (below) + # to identify it rather than the name + name: change-pvc-node-selector-config + # must be in the velero namespace + namespace: velero + # the below labels should be used verbatim in your + # ConfigMap. + labels: + # this value-less label identifies the ConfigMap as + # config for a plugin (i.e. the built-in restore item action plugin) + velero.io/plugin-config: "" + # this label identifies the name and kind of plugin + # that this ConfigMap is for. + velero.io/change-pvc-node-selector: RestoreItemAction +data: + # add 1+ key-value pairs here, where the key is the old + # node name and the value is the new node name. + : +``` diff --git a/site/content/docs/v1.7/run-locally.md b/site/content/docs/v1.7/run-locally.md new file mode 100644 index 000000000..897ee1893 --- /dev/null +++ b/site/content/docs/v1.7/run-locally.md @@ -0,0 +1,53 @@ +--- +title: "Run Velero locally in development" +layout: docs +--- + +Running the Velero server locally can speed up iterative development. This eliminates the need to rebuild the Velero server +image and redeploy it to the cluster with each change. + +## Run Velero locally with a remote cluster + +Velero runs against the Kubernetes API server as the endpoint (as per the `kubeconfig` configuration), so both the Velero server and client use the same `client-go` to communicate with Kubernetes. This means the Velero server can be run locally just as functionally as if it was running in the remote cluster. + +### Prerequisites + +When running Velero, you will need to ensure that you set up all of the following: + +* Appropriate RBAC permissions in the cluster + * Read access for all data from the source cluster and namespaces + * Write access to the target cluster and namespaces +* Cloud provider credentials + * Read/write access to volumes + * Read/write access to object storage for backup data +* A [BackupStorageLocation][20] object definition for the Velero server +* (Optional) A [VolumeSnapshotLocation][21] object definition for the Velero server, to take PV snapshots + +### 1. Install Velero + +See documentation on how to install Velero in some specific providers: [Install overview][22] + +### 2. Scale deployment down to zero + +After you use the `velero install` command to install Velero into your cluster, you scale the Velero deployment down to 0 so it is not simultaneously being run on the remote cluster and potentially causing things to get out of sync: + +`kubectl scale --replicas=0 deployment velero -n velero` + +#### 3. Start the Velero server locally + +* To run the server locally, use the full path according to the binary you need. Example, if you are on a Mac, and using `AWS` as a provider, this is how to run the binary you built from source using the full path: `AWS_SHARED_CREDENTIALS_FILE= ./_output/bin/darwin/amd64/velero`. Alternatively, you may add the `velero` binary to your `PATH`. + +* Start the server: `velero server [CLI flags]`. The following CLI flags may be useful to customize, but see `velero server --help` for full details: + * `--log-level`: set the Velero server's log level (default `info`, use `debug` for the most logging) + * `--kubeconfig`: set the path to the kubeconfig file the Velero server uses to talk to the Kubernetes apiserver (default `$KUBECONFIG`) + * `--namespace`: the set namespace where the Velero server should look for backups, schedules, restores (default `velero`) + * `--plugin-dir`: set the directory where the Velero server looks for plugins (default `/plugins`) + * The `--plugin-dir` flag requires the plugin binary to be present locally, and should be set to the directory containing this built binary. + * `--metrics-address`: set the bind address and port where Prometheus metrics are exposed (default `:8085`) + +[15]: https://docs.aws.amazon.com/cli/latest/topic/config-vars.html#the-shared-credentials-file +[16]: https://cloud.google.com/docs/authentication/getting-started#setting_the_environment_variable +[18]: https://eksctl.io/ +[20]: api-types/backupstoragelocation.md +[21]: api-types/volumesnapshotlocation.md +[22]: basic-install.md diff --git a/site/content/docs/v1.7/self-signed-certificates.md b/site/content/docs/v1.7/self-signed-certificates.md new file mode 100644 index 000000000..0bb55be8c --- /dev/null +++ b/site/content/docs/v1.7/self-signed-certificates.md @@ -0,0 +1,48 @@ +--- +title: "Use Velero with a storage provider secured by a self-signed certificate" +layout: docs +--- + +If you are using an S3-Compatible storage provider that is secured with a self-signed certificate, connections to the object store may fail with a `certificate signed by unknown authority` message. +To proceed, provide a certificate bundle when adding the storage provider. + +## Trusting a self-signed certificate during installation + +When using the `velero install` command, you can use the `--cacert` flag to provide a path +to a PEM-encoded certificate bundle to trust. + +```bash +velero install \ + --plugins + --provider \ + --bucket \ + --secret-file \ + --cacert +``` + +Velero will then automatically use the provided CA bundle to verify TLS connections to +that storage provider when backing up and restoring. + +## Trusting a self-signed certificate with the Velero client + +To use the describe, download, or logs commands to access a backup or restore contained +in storage secured by a self-signed certificate as in the above example, you must use +the `--cacert` flag to provide a path to the certificate to be trusted. + +```bash +velero backup describe my-backup --cacert +``` + +## Error with client certificate with custom S3 server + +In case you are using a custom S3-compatible server, you may encounter that the backup fails with an error similar to one below. + +``` +rpc error: code = Unknown desc = RequestError: send request failed caused by: +Get https://minio.com:3000/k8s-backup-bucket?delimiter=%2F&list-type=2&prefix=: remote error: tls: alert(116) +``` + +Error 116 represents certificate required as seen here in [error codes](https://datatracker.ietf.org/doc/html/rfc8446#appendix-B.2). +Velero as a client does not include its certificate while performing SSL handshake with the server. +From [TLS 1.3 spec](https://tools.ietf.org/html/rfc8446), verifying client certificate is optional on the server. +You will need to change this setting on the server to make it work. diff --git a/site/content/docs/v1.7/start-contributing.md b/site/content/docs/v1.7/start-contributing.md new file mode 100644 index 000000000..a9f15f0fc --- /dev/null +++ b/site/content/docs/v1.7/start-contributing.md @@ -0,0 +1,34 @@ +--- +title: "Start contributing" +layout: docs +--- + +## Before you start + +* Please familiarize yourself with the [Code of Conduct][1] before contributing. +* Also, see [CONTRIBUTING.md][2] for instructions on the developer certificate of origin that we require. + +## Creating a design doc + +Having a high level design document with the proposed change and the impacts helps the maintainers evaluate if a major change should be incorporated. + +To make a design pull request, you can copy the template found in the `design/_template.md` file into a new Markdown file. + +## Finding your way around + +You may join the Velero community and contribute in many different ways, including helping us design or test new features. For any significant feature we consider adding, we start with a design document. You may find a list of in progress new designs here: https://github.com/vmware-tanzu/velero/pulls?q=is%3Aopen+is%3Apr+label%3ADesign. Feel free to review and help us with your input. + +You can also vote on issues using :+1: and :-1:, as explained in our [Feature enhancement request][3] and [Bug issue][4] templates. This will help us quantify importance and prioritize issues. + +For information on how to connect with our maintainers and community, join our online meetings, or find good first issues, start on our [Velero community](https://velero.io/community/) page. + +Please browse our list of resources, including a playlist of past online community meetings, blog posts, and other resources to help you get familiar with our project: [Velero resources](https://velero.io/resources/). + +## Contributing + +If you are ready to jump in and test, add code, or help with documentation, please use the navigation on the left under `Contribute`. + +[1]: https://github.com/vmware-tanzu/velero/blob/v1.7.0/CODE_OF_CONDUCT.md +[2]: https://github.com/vmware-tanzu/velero/blob/v1.7.0/CONTRIBUTING.md +[3]: https://github.com/vmware-tanzu/velero/blob/v1.7.0/.github/ISSUE_TEMPLATE/feature-enhancement-request.md +[4]: https://github.com/vmware-tanzu/velero/blob/v1.7.0/.github/ISSUE_TEMPLATE/bug_report.md diff --git a/site/content/docs/v1.7/style-guide.md b/site/content/docs/v1.7/style-guide.md new file mode 100644 index 000000000..1b98e59dd --- /dev/null +++ b/site/content/docs/v1.7/style-guide.md @@ -0,0 +1,344 @@ +--- +title: "Documentation Style Guide" +layout: docs +--- + +_This style guide is adapted from the [Kubernetes style guide](https://kubernetes.io/docs/contribute/style/style-guide/)._ + +This page outlines writing style guidelines for the Velero documentation and you should use this page as a reference you write or edit content. Note that these are guidelines, not rules. Use your best judgment as you write documentation, and feel free to propose changes to these guidelines. Changes to the style guide are made by the Velero maintainers as a group. To propose a change or addition create an issue/PR, or add a suggestion to the [community meeting agenda](https://hackmd.io/Jq6F5zqZR7S80CeDWUklkA) and attend the meeting to participate in the discussion. + +The Velero documentation uses the [kramdown](https://kramdown.gettalong.org/) Markdown renderer. + +## Content best practices +### Use present tense + +{{< table caption="Do and Don't - Use present tense" >}} +|Do|Don't| +|--- |--- | +|This `command` starts a proxy.|This command will start a proxy.| +{{< /table >}} + +Exception: Use future or past tense if it is required to convey the correct meaning. + +### Use active voice + +{{< table caption="Do and Don't - Use active voice" >}} +|Do|Don't| +|--- |--- | +|You can explore the API using a browser.|The API can be explored using a browser.| +|The YAML file specifies the replica count.|The replica count is specified in the YAML file.| +{{< /table >}} + +Exception: Use passive voice if active voice leads to an awkward sentence construction. + +### Use simple and direct language + +Use simple and direct language. Avoid using unnecessary phrases, such as saying "please." + +{{< table caption="Do and Don't - Use simple and direct language" >}} +|Do|Don't| +|--- |--- | +|To create a ReplicaSet, ...|In order to create a ReplicaSet, ...| +|See the configuration file.|Please see the configuration file.| +|View the Pods.|With this next command, we'll view the Pods.| +{{< /table >}} + +### Address the reader as "you" + +{{< table caption="Do and Don't - Addressing the reader" >}} +|Do|Don't| +|--- |--- | +|You can create a Deployment by ...|We'll create a Deployment by ...| +|In the preceding output, you can see...|In the preceding output, we can see ...| +{{< /table >}} + +### Avoid Latin phrases + +Prefer English terms over Latin abbreviations. + +{{< table caption="Do and Don't - Avoid Latin phrases" >}} +|Do|Don't| +|--- |--- | +|For example, ...|e.g., ...| +|That is, ...|i.e., ...| +{{< /table >}} + +Exception: Use "etc." for et cetera. + +## Patterns to avoid + + +### Avoid using "we" + +Using "we" in a sentence can be confusing, because the reader might not know +whether they're part of the "we" you're describing. + +{{< table caption="Do and Don't - Avoid using we" >}} +|Do|Don't| +|--- |--- | +|Version 1.4 includes ...|In version 1.4, we have added ...| +|Kubernetes provides a new feature for ...|We provide a new feature ...| +|This page teaches you how to use Pods.|In this page, we are going to learn about Pods.| +{{< /table >}} + +### Avoid jargon and idioms + +Many readers speak English as a second language. Avoid jargon and idioms to help them understand better. + +{{< table caption="Do and Don't - Avoid jargon and idioms" >}} +|Do|Don't| +|--- |--- | +|Internally, ...|Under the hood, ...| +|Create a new cluster.|Turn up a new cluster.| +{{< /table >}} + +### Avoid statements about the future or that will soon be out of date + +Avoid making promises or giving hints about the future. If you need to talk about +a beta feature, put the text under a heading that identifies it as beta +information. + +Also avoid words like “recently”, "currently" and "new." A feature that is new today might not be +considered new in a few months. + +{{< table caption="Do and Don't - Avoid statements that will soon be out of date" >}} +|Do|Don't| +|--- |--- | +|In version 1.4, ...|In the current version, ...| +|The Federation feature provides ...|The new Federation feature provides ...| +{{< /table >}} + +### Language + +This documentation uses U.S. English spelling and grammar. + +## Documentation formatting standards + +### Use camel case for API objects + +When you refer to an API object, use the same uppercase and lowercase letters +that are used in the actual object name. Typically, the names of API +objects use +[camel case](https://en.wikipedia.org/wiki/Camel_case). + +Don't split the API object name into separate words. For example, use +PodTemplateList, not Pod Template List. + +Refer to API objects without saying "object," unless omitting "object" +leads to an awkward sentence construction. + +{{< table caption="Do and Don't - Do and Don't - API objects" >}} +|Do|Don't| +|--- |--- | +|The Pod has two containers.|The pod has two containers.| +|The Deployment is responsible for ...|The Deployment object is responsible for ...| +|A PodList is a list of Pods.|A Pod List is a list of pods.| +|The two ContainerPorts ...|The two ContainerPort objects ...| +|The two ContainerStateTerminated objects ...|The two ContainerStateTerminateds ...| +{{< /table >}} + +### Use angle brackets for placeholders + +Use angle brackets for placeholders. Tell the reader what a placeholder represents. + +1. Display information about a Pod: + + kubectl describe pod -n + + If the pod is in the default namespace, you can omit the '-n' parameter. + +### Use bold for user interface elements + +{{< table caption="Do and Don't - Bold interface elements" >}} +|Do|Don't| +|--- |--- | +|Click **Fork**.|Click "Fork".| +|Select **Other**.|Select "Other".| +{{< /table >}} + +### Use italics to define or introduce new terms + +{{< table caption="Do and Don't - Use italics for new terms" >}} +|Do|Don't| +|--- |--- | +|A _cluster_ is a set of nodes ...|A "cluster" is a set of nodes ...| +|These components form the _control plane_.|These components form the **control plane**.| +{{< /table >}} + +### Use code style for filenames, directories, paths, object field names and namespaces +{{< table caption="Do and Don't - Use code style for filenames, directories, paths, object field names and namespaces" >}} +|Do|Don't| +|--- |--- | +|Open the `envars.yaml` file.|Open the envars.yaml file.| +|Go to the `/docs/tutorials` directory.|Go to the /docs/tutorials directory.| +|Open the `/_data/concepts.yaml` file.|Open the /\_data/concepts.yaml file.| +{{< /table >}} + + +### Use punctuation inside quotes +{{< table caption="Do and Don't - Use code style for filenames, directories, paths, object field names and namespaces" >}} +|Do|Don't| +|--- |--- | +|events are recorded with an associated "stage."|events are recorded with an associated "stage".| +|The copy is called a "fork."|The copy is called a "fork".| +{{< /table >}} + +Exception: When the quoted word is a user input. + +Example: +* My user ID is “IM47g”. +* Did you try the password “mycatisawesome”? + +## Inline code formatting + + +### Use code style for inline code and commands + +For inline code in an HTML document, use the `` tag. In a Markdown +document, use the backtick (`` ` ``). + +{{< table caption="Do and Don't - Use code style for filenames, directories, paths, object field names and namespaces" >}} +|Do|Don't| +|--- |--- | +|The `kubectl run` command creates a Deployment.|The "kubectl run" command creates a Deployment.| +|For declarative management, use `kubectl apply`.|For declarative management, use "kubectl apply".| +|Use single backticks to enclose inline code. For example, `var example = true`.|Use two asterisks (`**`) or an underscore (`_`) to enclose inline code. For example, **var example = true**.| +|Use triple backticks (\`\`\`) before and after a multi-line block of code for fenced code blocks.|Use multi-line blocks of code to create diagrams, flowcharts, or other illustrations.| +|Use meaningful variable names that have a context.|Use variable names such as 'foo','bar', and 'baz' that are not meaningful and lack context.| +|Remove trailing spaces in the code.|Add trailing spaces in the code, where these are important, because a screen reader will read out the spaces as well.| +{{< /table >}} + +### Starting a sentence with a component tool or component name + +{{< table caption="Do and Don't - Starting a sentence with a component tool or component name" >}} +|Do|Don't| +|--- |--- | +|The `kubeadm` tool bootstraps and provisions machines in a cluster.|`kubeadm` tool bootstraps and provisions machines in a cluster.| +|The kube-scheduler is the default scheduler for Kubernetes.|kube-scheduler is the default scheduler for Kubernetes.| +{{< /table >}} + +### Use normal style for string and integer field values + +For field values of type string or integer, use normal style without quotation marks. + +{{< table caption="Do and Don't - Use normal style for string and integer field values" >}} +|Do|Don't| +|--- |--- | +|Set the value of `imagePullPolicy` to `Always`.|Set the value of `imagePullPolicy` to "Always".| +|Set the value of `image` to `nginx:1.16`.|Set the value of `image` to nginx:1.16.| +|Set the value of the `replicas` field to `2`.|Set the value of the `replicas` field to 2.| +{{< /table >}} + +## Code snippet formatting + + +### Don't include the command prompt + +{{< table caption="Do and Don't - Don't include the command prompt" >}} +|Do|Don't| +|--- |--- | +|kubectl get pods|$ kubectl get pods| +{{< /table >}} + +### Separate commands from output + +Verify that the Pod is running on your chosen node: + +``` +kubectl get pods --output=wide +``` + +The output is similar to this: + +``` +NAME READY STATUS RESTARTS AGE IP NODE +nginx 1/1 Running 0 13s 10.200.0.4 worker0 +``` + +## Velero.io word list + + +A list of Velero-specific terms and words to be used consistently across the site. + +{{< table caption="Velero.io word list" >}} +|Trem|Usage| +|--- |--- | +|Kubernetes|Kubernetes should always be capitalized.| +|Docker|Docker should always be capitalized.| +|Velero|Velero should always be capitalized.| +|VMware|VMware should always be correctly capitalized.| +|On-premises|On-premises or on-prem rather than on-premise or other variations.| +|Backup|Backup rather than back up, back-up or other variations.| +|Plugin|Plugin rather than plug-in or other variations.| +|Allowlist|Use allowlist instead of whitelist.| +|Denylist|Use denylist instead of blacklist.| +{{< /table >}} + +## Markdown elements + +### Headings +People accessing this documentation may use a screen reader or other assistive technology (AT). [Screen readers](https://en.wikipedia.org/wiki/Screen_reader) are linear output devices, they output items on a page one at a time. If there is a lot of content on a page, you can use headings to give the page an internal structure. A good page structure helps all readers to easily navigate the page or filter topics of interest. + +{{< table caption="Do and Don't - Headings" >}} +|Do|Don't| +|--- |--- | +|Include a title on each page or blog post.|Include more than one title headings (#) in a page.| +|Use ordered headings to provide a meaningful high-level outline of your content.|Use headings level 4 through 6, unless it is absolutely necessary. If your content is that detailed, it may need to be broken into separate articles.| +|Use sentence case for headings. For example, **Extend kubectl with plugins**|Use title case for headings. For example, **Extend Kubectl With Plugins**| +{{< /table >}} + +### Paragraphs + +{{< table caption="Do and Don't - Paragraphs" >}} + +|Do|Don't| +|--- |--- | +|Try to keep paragraphs under 6 sentences.|Write long-winded paragraphs.| +|Use three hyphens (`---`) to create a horizontal rule for breaks in paragraph content.|Use horizontal rules for decoration.| +{{< /table >}} + +### Links + +{{< table caption="Do and Don't - Links" >}} +|Do|Don't| +|--- |--- | +|Write hyperlinks that give you context for the content they link to. For example: Certain ports are open on your machines. See [check required ports](#check-required-ports) for more details.|Use ambiguous terms such as “click here”. For example: Certain ports are open on your machines. See [here](#check-required-ports) for more details.| +|Write Markdown-style links: `[link text](URL)`. For example: `[community meeting agenda](https://hackmd.io/Jq6F5zqZR7S80CeDWUklkA)` and the output is [community meeting agenda](https://hackmd.io/Jq6F5zqZR7S80CeDWUklkA).|Write HTML-style links: `Visit our tutorial!`| +{{< /table >}} + + +### Lists + +Group items in a list that are related to each other and need to appear in a specific order or to indicate a correlation between multiple items. When a screen reader comes across a list—whether it is an ordered or unordered list—it will be announced to the user that there is a group of list items. The user can then use the arrow keys to move up and down between the various items in the list. +Website navigation links can also be marked up as list items; after all they are nothing but a group of related links. + + - End each item in a list with a period if one or more items in the list are complete sentences. For the sake of consistency, normally either all items or none should be complete sentences. + + - Ordered lists that are part of an incomplete introductory sentence can be in lowercase and punctuated as if each item was a part of the introductory sentence. + + - Use the number one (`1.`) for ordered lists. + + - Use (`+`), (`*`), or (`-`) for unordered lists - be consistent within the same document. + + - Leave a blank line after each list. + + - Indent nested lists with four spaces (for example, ⋅⋅⋅⋅). + + - List items may consist of multiple paragraphs. Each subsequent paragraph in a list item must be indented by either four spaces or one tab. + +### Tables + +The semantic purpose of a data table is to present tabular data. Sighted users can quickly scan the table but a screen reader goes through line by line. A table [caption](https://www.w3schools.com/tags/tag_caption.asp) is used to create a descriptive title for a data table. Assistive technologies (AT) use the HTML table caption element to identify the table contents to the user within the page structure. + +If you need to create a table, create the table in markdown and use the table [Hugo shortcode](https://gohugo.io/content-management/shortcodes/) to include a caption. + +``` +{{}} +Parameter | Description | Default +:---------|:------------|:------- +`timeout` | The timeout for requests | `30s` +`logLevel` | The log level for log output | `INFO` +{{< /table */>}} + +``` +**Note:** This shortcode does not support markdown reference-style links. Use inline-style links in tables. See more information about [markdown link styles](https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet#links). diff --git a/site/content/docs/v1.7/support-process.md b/site/content/docs/v1.7/support-process.md new file mode 100644 index 000000000..d07268c52 --- /dev/null +++ b/site/content/docs/v1.7/support-process.md @@ -0,0 +1,44 @@ +--- +title: "Support Process" +layout: docs +--- + +## Weekly Rotation + +The Velero maintainers use a weekly rotation to manage community support. Each week, a different maintainer is the point person for responding to incoming support issues via Slack, GitHub, and the Google group. The point person is *not* expected to be on-call 24x7. Instead, they choose one or more hour(s) per day to be available/responding to incoming issues. They will communicate to the community what that time slot will be each week. + +## Start of Week + +We will update the public Slack channel's topic to indicate that you are the point person for the week, and what hours you'll be available. + +## During the Week + +### Where we will monitor +- `#velero` public Slack channel in Kubernetes org +- [all Velero-related repos][0] in GitHub (`velero`, `velero-plugin-for-[aws|gcp|microsoft-azure|csi]`, `helm-charts`) +- [Project Velero Google Group][1] + +### GitHub issue flow + +Generally speaking, new GitHub issues will fall into one of several categories. We use the following process for each: + +1. **Feature request** + - Label the issue with `Enhancement/User` or `Enhancement/Dev` + - Leave the issue in the `New Issues` swimlane for triage by product mgmt +1. **Bug** + - Label the issue with `Bug` + - Leave the issue in the `New Issues` swimlane for triage by product mgmt +1. **User question/problem** that does not clearly fall into one of the previous categories + - When you start investigating/responding, label the issue with `Investigating` + - Add comments as you go, so both the user and future support people have as much context as possible + - Use the `Needs Info` label to indicate an issue is waiting for information from the user. Remove/re-add the label as needed. + - If you resolve the issue with the user, close it out + - If the issue ends up being a feature request or a bug, update the title and follow the appropriate process for it + - If the reporter becomes unresponsive after multiple pings, close out the issue due to inactivity and comment that the user can always reach out again as needed + +## End of Week + +We ensure all GitHub issues worked on during the week on are labeled with `Investigating` and `Needs Info` (if appropriate), and have updated comments so the next person can pick them up. + +[0]: https://github.com/vmware-tanzu?q=velero&type=&language= +[1]: https://groups.google.com/forum/#!forum/projectvelero diff --git a/site/content/docs/v1.7/supported-providers.md b/site/content/docs/v1.7/supported-providers.md new file mode 100644 index 000000000..5e75dc1e2 --- /dev/null +++ b/site/content/docs/v1.7/supported-providers.md @@ -0,0 +1,70 @@ +--- +title: "Providers" +layout: docs +--- + +Velero supports a variety of storage providers for different backup and snapshot operations. Velero has a plugin system which allows anyone to add compatibility for additional backup and volume storage platforms without modifying the Velero codebase. + +## Velero supported providers + +{{< table caption="Velero supported providers" >}} + +| Provider | Object Store | Volume Snapshotter | Plugin Provider Repo | Setup Instructions | +|-----------------------------------|---------------------|------------------------------|-----------------------------------------|-------------------------------| +| [Amazon Web Services (AWS)](https://aws.amazon.com) | AWS S3 | AWS EBS | [Velero plugin for AWS](https://github.com/vmware-tanzu/velero-plugin-for-aws) | [AWS Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-aws#setup) | +| [Google Cloud Platform (GCP)](https://cloud.google.com) | Google Cloud Storage| Google Compute Engine Disks | [Velero plugin for GCP](https://github.com/vmware-tanzu/velero-plugin-for-gcp) | [GCP Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-gcp#setup) | +| [Microsoft Azure](https://azure.com) | Azure Blob Storage | Azure Managed Disks | [Velero plugin for Microsoft Azure](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure) | [Azure Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#setup) | +| [VMware vSphere](https://www.vmware.com/ca/products/vsphere.html) | 🚫 | vSphere Volumes | [VMware vSphere](https://github.com/vmware-tanzu/velero-plugin-for-vsphere) | [vSphere Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-vsphere#velero-plugin-for-vsphere-installation-and-configuration-details) | +| [Container Storage Interface (CSI)](https://kubernetes.io/blog/2019/01/15/container-storage-interface-ga/)| 🚫 | CSI Volumes | [Velero plugin for CSI](https://github.com/vmware-tanzu/velero-plugin-for-csi/) | [CSI Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-csi#kinds-of-plugins-included) | +{{< /table >}} + +Contact: [#Velero Slack](https://kubernetes.slack.com/messages/velero), [GitHub Issues](https://github.com/vmware-tanzu/velero/issues) + +## Community supported providers +{{< table caption="Community supported providers" >}} + +| Provider | Object Store | Volume Snapshotter | Plugin Documentation | Contact | +|---------------------------|------------------------------|------------------------------------|------------------------|---------------------------------| +| [AlibabaCloud](https://www.alibabacloud.com/) | Alibaba Cloud OSS | Alibaba Cloud | [AlibabaCloud](https://github.com/AliyunContainerService/velero-plugin) | [GitHub Issue](https://github.com/AliyunContainerService/velero-plugin/issues) | +| [DigitalOcean](https://www.digitalocean.com/) | DigitalOcean Object Storage | DigitalOcean Volumes Block Storage | [StackPointCloud](https://github.com/StackPointCloud/ark-plugin-digitalocean) | | +| [Hewlett Packard](https://www.hpe.com/us/en/storage.html) | 🚫 | HPE Storage | [Hewlett Packard](https://github.com/hpe-storage/velero-plugin) | [Slack](https://slack.hpedev.io/), [GitHub Issue](https://github.com/hpe-storage/velero-plugin/issues) | +| [OpenEBS](https://openebs.io/) | 🚫 | OpenEBS CStor Volume | [OpenEBS](https://github.com/openebs/velero-plugin) | [Slack](https://openebs-community.slack.com/), [GitHub Issue](https://github.com/openebs/velero-plugin/issues) | +| [OpenStack](https://www.openstack.org/) | Swift | Cinder | [OpenStack](https://github.com/Lirt/velero-plugin-for-openstack) | [GitHub Issue](https://github.com/Lirt/velero-plugin-for-openstack/issues) | +| [Portworx](https://portworx.com/) | 🚫 | Portworx Volume | [Portworx](https://docs.portworx.com/scheduler/kubernetes/ark.html) | [Slack](https://portworx.slack.com/messages/px-k8s), [GitHub Issue](https://github.com/portworx/ark-plugin/issues) | +| [Storj](https://storj.io) | Storj Object Storage | 🚫 | [Storj](https://github.com/storj-thirdparty/velero-plugin) | [GitHub Issue](https://github.com/storj-thirdparty/velero-plugin/issues) | +{{< /table >}} + +## S3-Compatible object store providers + +Velero's AWS Object Store plugin uses [Amazon's Go SDK][0] to connect to the AWS S3 API. Some third-party storage providers also support the S3 API, and users have reported the following providers work with Velero: + +_Note that these storage providers are not regularly tested by the Velero team._ + + * [IBM Cloud][1] + * [Oracle Cloud][2] + * [Minio][3] + * [DigitalOcean][4] + * [NooBaa][5] + * [Tencent Cloud][7] + * Ceph RADOS v12.2.7 + * Quobyte + * [Cloudian HyperStore][38] + +_Some storage providers, like Quobyte, may need a different [signature algorithm version][6]._ + +## Non-supported volume snapshots + +In the case you want to take volume snapshots but didn't find a plugin for your provider, Velero has support for snapshotting using restic. Please see the [restic integration][30] documentation. + +[0]: https://github.com/aws/aws-sdk-go/aws +[1]: contributions/ibm-config.md +[2]: contributions/oracle-config.md +[3]: contributions/minio.md +[4]: https://github.com/StackPointCloud/ark-plugin-digitalocean +[5]: http://www.noobaa.com/ +[6]: https://github.com/vmware-tanzu/velero-plugin-for-aws/blob/main/backupstoragelocation.md +[7]: contributions/tencent-config.md +[25]: https://github.com/hpe-storage/velero-plugin +[30]: restic.md +[36]: https://github.com/vmware-tanzu/velero-plugin-for-gcp#setup +[38]: https://www.cloudian.com/ diff --git a/site/content/docs/v1.7/tilt.md b/site/content/docs/v1.7/tilt.md new file mode 100644 index 000000000..42f68cfe4 --- /dev/null +++ b/site/content/docs/v1.7/tilt.md @@ -0,0 +1,210 @@ +--- +title: "Rapid iterative Velero development with Tilt " +layout: docs +--- + +## Overview +This document describes how to use [Tilt](https://tilt.dev) with any cluster for a simplified +workflow that offers easy deployments and rapid iterative builds. + +This setup allows for continuing deployment of the Velero server and, if specified, any provider plugin or the restic daemonset. +It does this work by: + +1. Deploying the necessary Kubernetes resources, such as the Velero CRDs and Velero deployment +1. Building a local binary for Velero and (if specified) provider plugins as a `local_resource` +1. Invoking `docker_build` to live update any binary into the container/init container and trigger a re-start + +Tilt will look for configuration files under `velero/tilt-resources`. Most of the +files in this directory are gitignored so you may configure your setup according to your needs. + +## Prerequisites +1. [Docker](https://docs.docker.com/install/) v19.03 or newer +1. A Kubernetes cluster v1.12 or greater (does not have to be Kind) +1. [Tilt](https://docs.tilt.dev/install.html) v0.12.0 or newer +1. Clone the [Velero project](https://github.com/vmware-tanzu/velero) repository + locally +1. Access to an S3 object storage +1. Clone any [provider plugin(s)](https://velero.io/plugins/) you want to make changes to and deploy (optional, must be configured to be deployed by the Velero Tilt's setup, [more info below](#provider-plugins)) + +Note: To properly configure any plugin you use, please follow the plugin's documentation. + +## Getting started + +### tl;dr +- Copy all sample files under `velero/tilt-resources/examples` into `velero/tilt-resources`. +- Configure the `velero_v1_backupstoragelocation.yaml` file, and the `cloud` file for the storage credentials/secret. + +- Run `tilt up`. + +### Create a Tilt settings file +Create a configuration file named `tilt-settings.json` and place it in your local copy of `velero/tilt-resources`. Alternatively, +you may copy and paste the sample file found in `velero/tilt-resources/examples`. + +Here is an example: + +```json +{ + "default_registry": "", + "enable_providers": [ + "aws", + "gcp", + "azure", + "csi" + ], + "providers": { + "aws": "../velero-plugin-for-aws", + "gcp": "../velero-plugin-for-gcp", + "azure": "../velero-plugin-for-microsoft-azure", + "csi": "../velero-plugin-for-csi" + }, + "allowed_contexts": [ + "development" + ], + "enable_restic": false, + "create_backup_locations": true, + "setup-minio": true, + "enable_debug": false, + "debug_continue_on_start": true +} +``` + +#### tilt-settings.json fields +**default_registry** (String, default=""): The image registry to use if you need to push images. See the [Tilt +*documentation](https://docs.tilt.dev/api.html#api.default_registry) for more details. + +**provider_repos** (Array[]String, default=[]): A list of paths to all the provider plugins you want to make changes to. Each provider must have a +`tilt-provider.json` file describing how to build the provider. + +**enable_providers** (Array[]String, default=[]): A list of the provider plugins to enable. See [provider plugins](provider-plugins) +for more details. Note: when not making changes to a plugin, it is not necessary to load them into +Tilt: an existing image and version might be specified in the Velero deployment instead, and Tilt will load that. + +**allowed_contexts** (Array, default=[]): A list of kubeconfig contexts Tilt is allowed to use. See the Tilt documentation on +*[allow_k8s_contexts](https://docs.tilt.dev/api.html#api.allow_k8s_contexts) for more details. Note: Kind is automatically allowed. + +**enable_restic** (Bool, default=false): Indicate whether to deploy the restic Daemonset. If set to `true`, Tilt will look for a `velero/tilt-resources/restic.yaml` file +containing the configuration of the Velero restic DaemonSet. + +**create_backup_locations** (Bool, default=false): Indicate whether to create one or more backup storage locations. If set to `true`, Tilt will look for a `velero/tilt-resources/velero_v1_backupstoragelocation.yaml` file +containing at least one configuration for a Velero backup storage location. + +**setup-minio** (Bool, default=false): Configure this to `true` if you want to configure backup storage locations in a Minio instance running inside your cluster. + +**enable_debug** (Bool, default=false): Configure this to `true` if you want to debug the velero process using [Delve](https://github.com/go-delve/delve). + +**debug_continue_on_start** (Bool, default=true): Configure this to `true` if you want the velero process to continue on start when in debug mode. See [Delve CLI documentation](https://github.com/go-delve/delve/blob/master/Documentation/usage/dlv.md). + +### Create Kubernetes resource files to deploy +All needed Kubernetes resource files are provided as ready to use samples in the `velero/tilt-resources/examples` directory. You only have to move them to the `velero/tilt-resources` level. + +Because the Velero Kubernetes deployment as well as the restic DaemonSet contain the configuration +for any plugin to be used, files for these resources are expected to be provided by the user so you may choose +which provider plugin to load as a init container. Currently, the sample files provided are configured with all the +plugins supported by Velero, feel free to remove any of them as needed. + +For Velero to operate fully, it also needs at least one backup +storage location. A sample file is provided that needs to be modified with the specific +configuration for your object storage. See the next sub-section for more details on this. + +### Configure a backup storage location +You will have to configure the `velero/tilt-resources/velero_v1_backupstoragelocation.yaml` with the proper values according to your storage provider. Read the [plugin documentation](https://velero.io/plugins/) +to learn what field/value pairs are required for your particular provider's backup storage location configuration. + +Below are some ways to configure a backup storage location for Velero. +#### As a storage with a service provider +Follow the provider documentation to provision the storage. We have a [list of all known object storage providers](supported-providers/) with corresponding plugins for Velero. + +#### Using MinIO as an object storage +Note: to use MinIO as an object storage, you will need to use the [`AWS` plugin](https://github.com/vmware-tanzu/velero-plugin-for-aws), and configure the storage location with the `spec.provider` set to `aws` and the `spec.config.region` set to `minio`. Example: +``` +spec: + config: + region: minio + s3ForcePathStyle: "true" + s3Url: http://minio.velero.svc:9000 + objectStorage: + bucket: velero + provider: aws +``` + +Here are two ways to use MinIO as the storage: + +1) As a MinIO instance running inside your cluster (don't do this for production!) + + In the `tilt-settings.json` file, set `"setup-minio": true`. This will configure a Kubernetes deployment containing a running +instance of MinIO inside your cluster. There are [extra steps](contributions/minio/#expose-minio-outside-your-cluster-with-a-service) +necessary to expose MinIO outside the cluster. + + To access this storage, you will need to expose MinIO outside the cluster by forwarding the MinIO port to the local machine using kubectl port-forward -n svc/minio 9000. Update the BSL configuration to use that as its "public URL" by adding `publicUrl: http://localhost:9000` to the BSL config. This is necessary to do things like download a backup file. + + Note: with this setup, when your cluster is terminated so is the storage and any backup/restore in it. + +1) As a standalone MinIO instance running locally in a Docker container + + See [these instructions](https://github.com/vmware-tanzu/velero/discussions/3381) to run MinIO locally on your computer, as a standalone as opposed to running it on a Pod. + +Please see our [locations documentation](locations/) to learn more how backup locations work. + +### Configure the provider credentials (secret) +Whatever object storage provider you use, configure the credentials for in the `velero/tilt-resources/cloud` file. Read the [plugin documentation](https://velero.io/plugins/) +to learn what field/value pairs are required for your provider's credentials. The Tilt file will invoke Kustomize to create the secret under the hard-coded key `secret.cloud-credentials.data.cloud` in the Velero namespace. + +There is a sample credentials file properly formatted for a MinIO storage credentials in `velero/tilt-resources/examples/cloud`. + +### Configure debugging with Delve +If you would like to debug the Velero process, you can enable debug mode by setting the field `enable_debug` to `true` in your `tilt-resources/tile-settings.json` file. +This will enable you to debug the process using [Delve](https://github.com/go-delve/delve). +By enabling debug mode, the Velero executable will be built in debug mode (using the flags `-gcflags="-N -l"` which disables optimizations and inlining), and the process will be started in the Velero deployment using [`dlv exec`](https://github.com/go-delve/delve/blob/master/Documentation/usage/dlv_exec.md). + +The debug server will accept connections on port 2345 and Tilt is configured to forward this port to the local machine. +Once Tilt is [running](#run-tilt) and the Velero resource is ready, you can connect to the debug server to begin debugging. +To connect to the session, you can use the Delve CLI locally by running `dlv connect 127.0.0.1:2345`. See the [Delve CLI documentation](https://github.com/go-delve/delve/tree/master/Documentation/cli) for more guidance on how to use Delve. +Delve can also be used within a number of [editors and IDEs](https://github.com/go-delve/delve/blob/master/Documentation/EditorIntegration.md). + +By default, the Velero process will continue on start when in debug mode. +This means that the process will run until a breakpoint is set. +You can disable this by setting the field `debug_continue_on_start` to `false` in your `tilt-resources/tile-settings.json` file. +When this setting is disabled, the Velero process will not continue to run until a `continue` instruction is issued through your Delve session. + +When exiting your debug session, the CLI and editor integrations will typically ask if the remote process should be stopped. +It is important to leave the remote process running and just disconnect from the debugging session. +By stopping the remote process, that will cause the Velero container to stop and the pod to restart. +If backups are in progress, these will be left in a stale state as they are not resumed when the Velero pod restarts. + +### Run Tilt! +To launch your development environment, run: + +``` bash +tilt up +``` + +This will output the address to a web browser interface where you can monitor Tilt's status and the logs for each Tilt resource. After a brief amount of time, you should have a running development environment, and you should now be able to +create backups/restores and fully operate Velero. + +Note: Running `tilt down` after exiting out of Tilt [will delete all resources](https://docs.tilt.dev/cli/tilt_down.html) specified in the Tiltfile. + +Tip: Create an alias to `velero/_tuiltbuild/local/velero` and you won't have to run `make local` to get a refreshed version of the Velero CLI, just use the alias. + +Please see the documentation for [how Velero works](how-velero-works/). + +## Provider plugins +A provider must supply a `tilt-provider.json` file describing how to build it. Here is an example: + +```json +{ + "plugin_name": "velero-plugin-for-aws", + "context": ".", + "image": "velero/velero-plugin-for-aws", + "live_reload_deps": [ + "velero-plugin-for-aws" + ], + "go_main": "./velero-plugin-for-aws" +} +``` + +## Live updates +Each provider plugin configured to be deployed by Velero's Tilt setup has a `live_reload_deps` list. This defines the files and/or directories that Tilt +should monitor for changes. When a dependency is modified, Tilt rebuilds the provider's binary **on your local +machine**, copies the binary to the init container, and triggers a restart of the Velero container. This is significantly faster +than rebuilding the container image for each change. It also helps keep the size of each development image as small as +possible (the container images do not need the entire go toolchain, source code, module dependencies, etc.). diff --git a/site/content/docs/v1.7/troubleshooting.md b/site/content/docs/v1.7/troubleshooting.md new file mode 100644 index 000000000..7b035006f --- /dev/null +++ b/site/content/docs/v1.7/troubleshooting.md @@ -0,0 +1,219 @@ +--- +title: "Troubleshooting" +layout: docs +--- + +These tips can help you troubleshoot known issues. If they don't help, you can [file an issue][4], or talk to us on the [#velero channel][25] on the Kubernetes Slack server. + +## Debug installation/ setup issues + +- [Debug installation/setup issues][2] + +## Debug restores + +- [Debug restores][1] + +## General troubleshooting information + +You can use the `velero bug` command to open a [Github issue][4] by launching a browser window with some prepopulated values. Values included are OS, CPU architecture, `kubectl` client and server versions (if available) and the `velero` client version. This information isn't submitted to Github until you click the `Submit new issue` button in the Github UI, so feel free to add, remove or update whatever information you like. + +Some general commands for troubleshooting that may be helpful: + +* `velero backup describe ` - describe the details of a backup +* `velero backup logs ` - fetch the logs for this specific backup. Useful for viewing failures and warnings, including resources that could not be backed up. +* `velero restore describe ` - describe the details of a restore +* `velero restore logs ` - fetch the logs for this specific restore. Useful for viewing failures and warnings, including resources that could not be restored. +* `kubectl logs deployment/velero -n velero` - fetch the logs of the Velero server pod. This provides the output of the Velero server processes. + +### Getting velero debug logs + +You can increase the verbosity of the Velero server by editing your Velero deployment to look like this: + + +``` +kubectl edit deployment/velero -n velero +... + containers: + - name: velero + image: velero/velero:latest + command: + - /velero + args: + - server + - --log-level # Add this line + - debug # Add this line +... +``` + +## Known issue with restoring LoadBalancer Service + +Because of how Kubernetes handles Service objects of `type=LoadBalancer`, when you restore these objects you might encounter an issue with changed values for Service UIDs. Kubernetes automatically generates the name of the cloud resource based on the Service UID, which is different when restored, resulting in a different name for the cloud load balancer. If the DNS CNAME for your application points to the DNS name of your cloud load balancer, you'll need to update the CNAME pointer when you perform a Velero restore. + +Alternatively, you might be able to use the Service's `spec.loadBalancerIP` field to keep connections valid, if your cloud provider supports this value. See [the Kubernetes documentation about Services of Type LoadBalancer](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). + +## Miscellaneous issues + +### Velero reports `custom resource not found` errors when starting up. + +Velero's server will not start if the required Custom Resource Definitions are not found in Kubernetes. Run `velero install` again to install any missing custom resource definitions. + +### `velero backup logs` returns a `SignatureDoesNotMatch` error + +Downloading artifacts from object storage utilizes temporary, signed URLs. In the case of S3-compatible +providers, such as Ceph, there may be differences between their implementation and the official S3 +API that cause errors. + +Here are some things to verify if you receive `SignatureDoesNotMatch` errors: + + * Make sure your S3-compatible layer is using [signature version 4][5] (such as Ceph RADOS v12.2.7) + * For Ceph, try using a native Ceph account for credentials instead of external providers such as OpenStack Keystone + +## Velero (or a pod it was backing up) restarted during a backup and the backup is stuck InProgress + +Velero cannot resume backups that were interrupted. Backups stuck in the `InProgress` phase can be deleted with `kubectl delete backup -n `. +Backups in the `InProgress` phase have not uploaded any files to object storage. + +## Velero is not publishing prometheus metrics + +Steps to troubleshoot: + +- Confirm that your velero deployment has metrics publishing enabled. The [latest Velero helm charts][6] have been setup with [metrics enabled by default][7]. +- Confirm that the Velero server pod exposes the port on which the metrics server listens on. By default, this value is 8085. + +```yaml + ports: + - containerPort: 8085 + name: metrics + protocol: TCP +``` + +- Confirm that the metric server is listening for and responding to connections on this port. This can be done using [port-forwarding][9] as shown below + +```bash +$ kubectl -n port-forward 8085:8085 +Forwarding from 127.0.0.1:8085 -> 8085 +Forwarding from [::1]:8085 -> 8085 +. +. +. +``` + +Now, visiting http://localhost:8085/metrics on a browser should show the metrics that are being scraped from Velero. + +- Confirm that the Velero server pod has the necessary [annotations][8] for prometheus to scrape metrics. +- Confirm, from the Prometheus UI, that the Velero pod is one of the targets being scraped from Prometheus. + + +## Is Velero using the correct cloud credentials? + +Cloud provider credentials are given to Velero to store and retrieve backups from the object store and to perform volume snapshotting operations. + +These credentials are either passed to Velero at install time using: +1. `--secret-file` flag to the `velero install` command. OR +1. `--set-file credentials.secretContents.cloud` flag to the `helm install` command. + +Or, they are specified when creating a `BackupStorageLocation` using the `--credential` flag. + +### Troubleshooting credentials provided during install + +If using the credentials provided at install time, they are stored in the cluster as a Kubernetes secret named `cloud-credentials` in the same namespace in which Velero is installed. + +Follow the below troubleshooting steps to confirm that Velero is using the correct credentials: +1. Confirm that the `cloud-credentials` secret exists and has the correct content. + ```bash + $ kubectl -n velero get secrets cloud-credentials + NAME TYPE DATA AGE + cloud-credentials Opaque 1 11h + $ kubectl -n velero get secrets cloud-credentials -ojsonpath={.data.cloud} | base64 --decode + + ``` + +1. Confirm that velero deployment is mounting the `cloud-credentials` secret. + ```bash + $ kubectl -n velero get deploy velero -ojson | jq .spec.template.spec.containers[0].volumeMounts + [ + { + "mountPath": "/plugins", + "name": "plugins" + }, + { + "mountPath": "/scratch", + "name": "scratch" + }, + { + "mountPath": "/credentials", + "name": "cloud-credentials" + } + ] + ``` + + If [restic-integration][3] is enabled, then, confirm that the restic daemonset is also mounting the `cloud-credentials` secret. + ```bash + $ kubectl -n velero get ds restic -ojson |jq .spec.template.spec.containers[0].volumeMounts + [ + { + "mountPath": "/host_pods", + "mountPropagation": "HostToContainer", + "name": "host-pods" + }, + { + "mountPath": "/scratch", + "name": "scratch" + }, + { + "mountPath": "/credentials", + "name": "cloud-credentials" + } + ] + ``` + +1. Confirm if the correct credentials are mounted into the Velero pod. + ```bash + $ kubectl -n velero exec -ti deploy/velero -- bash + nobody@velero-69f9c874c-l8mqp:/$ cat /credentials/cloud + + ``` + +### Troubleshooting `BackupStorageLocation` credentials + +Follow the below troubleshooting steps to confirm that Velero is using the correct credentials if using credentials specific to a [`BackupStorageLocation`][10]: +1. Confirm that the object storage provider plugin being used supports multiple credentials. + + If the logs from the Velero deployment contain the error message `"config has invalid keys credentialsFile"`, the version of your object storage plugin does not yet support multiple credentials. + + The object storage plugins [maintained by the Velero team][11] support this feature, so please update your plugin to the latest version if you see the above error message. + + If you are using a plugin from a different provider, please contact them for further advice. + +1. Confirm that the secret and key referenced by the `BackupStorageLocation` exists in the Velero namespace and has the correct content: + ```bash + # Determine which secret and key the BackupStorageLocation is using + BSL_SECRET=$(kubectl get backupstoragelocations.velero.io -n velero -o yaml -o jsonpath={.spec.credential.name}) + BSL_SECRET_KEY=$(kubectl get backupstoragelocations.velero.io -n velero -o yaml -o jsonpath={.spec.credential.key}) + + # Confirm that the secret exists + kubectl -n velero get secret $BSL_SECRET + + # Print the content of the secret and ensure it is correct + kubectl -n velero get secret $BSL_SECRET -ojsonpath={.data.$BSL_SECRET_KEY} | base64 --decode + ``` + If the secret can't be found, the secret does not exist within the Velero namespace and must be created. + + If no output is produced when printing the contents of the secret, the key within the secret may not exist or may have no content. + Ensure that the key exists within the secret's data by checking the output from `kubectl -n velero describe secret $BSL_SECRET`. + If it does not exist, follow the instructions for [editing a Kubernetes secret][12] to add the base64 encoded credentials data. + + +[1]: debugging-restores.md +[2]: debugging-install.md +[3]: restic.md +[4]: https://github.com/vmware-tanzu/velero/issues +[5]: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html +[6]: https://github.com/vmware-tanzu/helm-charts/blob/main/charts/velero +[7]: https://github.com/vmware-tanzu/helm-charts/blob/main/charts/velero/values.yaml#L44 +[8]: https://github.com/vmware-tanzu/helm-charts/blob/main/charts/velero/values.yaml#L49-L52 +[9]: https://kubectl.docs.kubernetes.io/pages/container_debugging/port_forward_to_pods.html +[10]: locations.md +[11]: /plugins +[12]: https://kubernetes.io/docs/concepts/configuration/secret/#editing-a-secret +[25]: https://kubernetes.slack.com/messages/velero diff --git a/site/content/docs/v1.7/uninstalling.md b/site/content/docs/v1.7/uninstalling.md new file mode 100644 index 000000000..f0b7b8d17 --- /dev/null +++ b/site/content/docs/v1.7/uninstalling.md @@ -0,0 +1,11 @@ +--- +title: "Uninstalling Velero" +layout: docs +--- + +If you would like to completely uninstall Velero from your cluster, the following commands will remove all resources created by `velero install`: + +```bash +kubectl delete namespace/velero clusterrolebinding/velero +kubectl delete crds -l component=velero +``` diff --git a/site/content/docs/main/upgrade-to-1.6.md b/site/content/docs/v1.7/upgrade-to-1.7.md similarity index 68% rename from site/content/docs/main/upgrade-to-1.6.md rename to site/content/docs/v1.7/upgrade-to-1.7.md index 0797184ec..c46b881c1 100644 --- a/site/content/docs/main/upgrade-to-1.6.md +++ b/site/content/docs/v1.7/upgrade-to-1.7.md @@ -1,23 +1,26 @@ --- -title: "Upgrading to Velero 1.6" +title: "Upgrading to Velero 1.7" layout: docs --- ## Prerequisites -- Velero [v1.5.x][5] installed. +- Velero [v1.6.x][6] installed. -If you're not yet running at least Velero v1.5, see the following: +If you're not yet running at least Velero v1.6, see the following: - [Upgrading to v1.1][1] - [Upgrading to v1.2][2] - [Upgrading to v1.3][3] - [Upgrading to v1.4][4] - [Upgrading to v1.5][5] +- [Upgrading to v1.6][6] + +Before upgrading, check the [Velero compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatabilty-matrix) to make sure your version of Kubernetes is supported by the new version of Velero. ## Instructions -1. Install the Velero v1.6 command-line interface (CLI) by following the [instructions here][0]. +1. Install the Velero v1.7 command-line interface (CLI) by following the [instructions here][0]. Verify that you've properly installed it by running: @@ -29,7 +32,7 @@ If you're not yet running at least Velero v1.5, see the following: ```bash Client: - Version: v1.6.2 + Version: v1.7.0 Git commit: ``` @@ -41,18 +44,21 @@ If you're not yet running at least Velero v1.5, see the following: **NOTE:** You could change the default CRD API version (v1beta1 _or_ v1) if Velero CLI can't discover the Kubernetes preferred CRD API version. The Kubernetes version < 1.16 preferred CRD API version is v1beta1; the Kubernetes version >= 1.16 preferred CRD API version is v1. - **NOTE:** If you are upgrading Velero in Kubernetes 1.14.x or earlier, you will need to use `kubectl apply`'s `--validate=false` option when applying the CRD configuration above. See [issue 2077][6] and [issue 2311][7] for more context. + **NOTE:** If you are upgrading Velero in Kubernetes 1.14.x or earlier, you will need to use `kubectl apply`'s `--validate=false` option when applying the CRD configuration above. See [issue 2077][10] and [issue 2311][11] for more context. -1. Update the container image used by the Velero deployment and, optionally, the restic daemon set: +1. Update the container image used by the Velero deployment, plugin and, optionally, the restic daemon set: ```bash + # set the container and image of the init container for plugin accordingly, + # if you are using other plugin kubectl set image deployment/velero \ - velero=velero/velero:v1.6.2 \ + velero=velero/velero:v1.7.0 \ + velero-velero-plugin-for-aws=velero/velero-plugin-for-aws:v1.3.0 \ --namespace velero # optional, if using the restic daemon set kubectl set image daemonset/restic \ - restic=velero/velero:v1.6.2 \ + restic=velero/velero:v1.7.0 \ --namespace velero ``` @@ -66,16 +72,16 @@ If you're not yet running at least Velero v1.5, see the following: ```bash Client: - Version: v1.6.2 + Version: v1.7.0 Git commit: Server: - Version: v1.6.2 + Version: v1.7.0 ``` ## Notes ### Default backup storage location -We have deprecated the way to indicate the default backup storage location. Previously, that was indicated according to the backup storage location name set on the velero server-side via the flag `velero server --default-backup-storage-location`. Now we configure the default backup storage location on the velero client-side. Please refer to the [About locations][9] on how to indicate which backup storage location is the default one. +We have deprecated the way to indicate the default backup storage location. Previously, that was indicated according to the backup storage location name set on the velero server-side via the flag `velero server --default-backup-storage-location`. Now we configure the default backup storage location on the velero client-side. Please refer to the [About locations][9] on how to indicate which backup storage location is the default one. After upgrading, if there is a previously created backup storage location with the name that matches what was defined on the server side as the default, it will be automatically set as the `default`. @@ -85,7 +91,7 @@ After upgrading, if there is a previously created backup storage location with t [3]: https://velero.io/docs/v1.3.2/upgrade-to-1.3/ [4]: https://velero.io/docs/v1.4/upgrade-to-1.4/ [5]: https://velero.io/docs/v1.5/upgrade-to-1.5 -[6]: https://github.com/vmware-tanzu/velero/releases/tag/v1.4.2 -[7]: https://github.com/vmware-tanzu/velero/issues/2077 -[8]: https://github.com/vmware-tanzu/velero/issues/2311 -[9]: https://velero.io/docs/v1.6/locations +[6]: https://velero.io/docs/v1.6/upgrade-to-1.6 +[9]: https://velero.io/docs/v1.7/locations +[10]: https://github.com/vmware-tanzu/velero/issues/2077 +[11]: https://github.com/vmware-tanzu/velero/issues/2311 diff --git a/site/content/docs/v1.7/velero-install.md b/site/content/docs/v1.7/velero-install.md new file mode 100644 index 000000000..8cc6d4d2e --- /dev/null +++ b/site/content/docs/v1.7/velero-install.md @@ -0,0 +1,49 @@ +--- +title: "Velero Install CLI" +layout: docs +--- + +This document serves as a guide to using the `velero install` CLI command to install `velero` server components into your kubernetes cluster. + +_NOTE_: `velero install` will, by default, use the CLI's version information to determine the version of the server components to deploy. This behavior may be overridden by using the `--image` flag. Refer to [Building Server Component Container Images][1]. + +## Usage + +This section explains some of the basic flags supported by the `velero install` CLI command. For a complete explanation of the flags, please run `velero install --help` + +```bash +velero install \ + --plugins + --provider \ + --bucket \ + --secret-file \ + --velero-pod-cpu-request \ + --velero-pod-mem-request \ + --velero-pod-cpu-limit \ + --velero-pod-mem-limit \ + [--use-restic] \ + [--default-volumes-to-restic] \ + [--restic-pod-cpu-request ] \ + [--restic-pod-mem-request ] \ + [--restic-pod-cpu-limit ] \ + [--restic-pod-mem-limit ] +``` + +The values for the resource requests and limits flags follow the same format as [Kubernetes resource requirements][3] +For plugin container images, please refer to our [supported providers][2] page. + +## Examples + +This section provides examples that serve as a starting point for more customized installations. + +```bash +velero install --provider gcp --plugins velero/velero-plugin-for-gcp:v1.0.0 --bucket mybucket --secret-file ./gcp-service-account.json + +velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --provider aws --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-restic + +velero install --provider azure --plugins velero/velero-plugin-for-microsoft-azure:v1.0.0 --bucket $BLOB_CONTAINER --secret-file ./credentials-velero --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] +``` + +[1]: build-from-source.md#making-images-and-updating-velero +[2]: supported-providers.md +[3]: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ diff --git a/site/content/docs/v1.7/vendoring-dependencies.md b/site/content/docs/v1.7/vendoring-dependencies.md new file mode 100644 index 000000000..9fc1bcac1 --- /dev/null +++ b/site/content/docs/v1.7/vendoring-dependencies.md @@ -0,0 +1,21 @@ +--- +title: "Vendoring dependencies" +layout: docs +--- + +## Overview + +We are using [dep][0] to manage dependencies. You can install it by following [these +instructions][1]. + +## Adding a new dependency + +Run `dep ensure`. If you want to see verbose output, you can append `-v` as in +`dep ensure -v`. + +## Updating an existing dependency + +Run `dep ensure -update [ ...]` to update one or more dependencies. + +[0]: https://github.com/golang/dep +[1]: https://golang.github.io/dep/docs/installation.html diff --git a/site/content/docs/v1.7/website-guidelines.md b/site/content/docs/v1.7/website-guidelines.md new file mode 100644 index 000000000..c5373f4d8 --- /dev/null +++ b/site/content/docs/v1.7/website-guidelines.md @@ -0,0 +1,45 @@ +--- +title: "Website Guidelines" +layout: docs +--- + +## Running the website locally + +When making changes to the website, please run the site locally before submitting a PR and manually verify your changes. + +At the root of the project, run: + +```bash +make serve-docs +``` + +This runs all the Hugo dependencies in a container. + +Alternatively, for quickly loading the website, under the `velero/site/` directory run: + +```bash +hugo serve +``` + +For more information on how to run the website locally, please see our [Hugo documentation](https://gohugo.io/getting-started/). + +## Adding a blog post + +To add a blog post, create a new markdown (.MD) file in the `/site/content/posts/` folder. A blog post requires the following front matter. + +```yaml +title: "Title of the blog" +excerpt: Brief summary of thee blog post that appears as a preview on velero.io/blogs +author_name: Jane Smith +slug: URL-For-Blog +# Use different categories that apply to your blog. This is used to connect related blogs on the site +categories: ['velero','release'] +# Image to use for blog. The path is relative to the site/static/ folder +image: /img/posts/example-image.jpg +# Tag should match author to drive author pages. Tags can have multiple values. +tags: ['Velero Team', 'Nolan Brubaker'] +``` + +Include the `author_name` value in tags field so the page that lists the author's posts will work properly, for example https://velero.io/tags/carlisia-thompson/. + +Ideally each blog will have a unique image to use on the blog home page, but if you do not include an image, the default Velero logo will be used instead. Use an image that is less than 70KB and add it to the `/site/static/img/posts` folder. diff --git a/site/content/docs/v1.8/_index.md b/site/content/docs/v1.8/_index.md new file mode 100644 index 000000000..74eece82f --- /dev/null +++ b/site/content/docs/v1.8/_index.md @@ -0,0 +1,58 @@ +--- +toc: "false" +cascade: + version: v1.8 + toc: "true" +--- +![100] + +[![Build Status][1]][2] + +## Overview + +Velero (formerly Heptio Ark) gives you tools to back up and restore your Kubernetes cluster resources and persistent volumes. You can run Velero with a cloud provider or on-premises. Velero lets you: + +* Take backups of your cluster and restore in case of loss. +* Migrate cluster resources to other clusters. +* Replicate your production cluster to development and testing clusters. + +Velero consists of: + +* A server that runs on your cluster +* A command-line client that runs locally + +## Documentation + +This site is our documentation home with installation instructions, plus information about customizing Velero for your needs, architecture, extending Velero, contributing to Velero and more. + +Please use the version selector at the top of the site to ensure you are using the appropriate documentation for your version of Velero. + +## Troubleshooting + +If you encounter issues, review the [troubleshooting docs][30], [file an issue][4], or talk to us on the [#velero channel][25] on the Kubernetes Slack server. + +## Contributing + +If you are ready to jump in and test, add code, or help with documentation, follow the instructions on our [Start contributing](https://velero.io/docs/v1.8.0/start-contributing/) documentation for guidance on how to setup Velero for development. + +## Changelog + +See [the list of releases][6] to find out about feature changes. + +[1]: https://github.com/vmware-tanzu/velero/workflows/Main%20CI/badge.svg +[2]: https://github.com/vmware-tanzu/velero/actions?query=workflow%3A"Main+CI" + +[4]: https://github.com/vmware-tanzu/velero/issues +[6]: https://github.com/vmware-tanzu/velero/releases + +[9]: https://kubernetes.io/docs/setup/ +[10]: https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-with-homebrew-on-macos +[11]: https://kubernetes.io/docs/tasks/tools/install-kubectl/#tabset-1 +[12]: https://github.com/kubernetes/kubernetes/blob/main/cluster/addons/dns/README.md +[14]: https://github.com/kubernetes/kubernetes +[24]: https://groups.google.com/forum/#!forum/projectvelero +[25]: https://kubernetes.slack.com/messages/velero + +[30]: troubleshooting.md + +[100]: img/velero.png diff --git a/site/content/docs/v1.8/api-types/README.md b/site/content/docs/v1.8/api-types/README.md new file mode 100644 index 000000000..54c23544d --- /dev/null +++ b/site/content/docs/v1.8/api-types/README.md @@ -0,0 +1,21 @@ +--- +title: "Table of Contents" +layout: docs +--- + +## API types + +Here we list the API types that have some functionality that you can only configure via json/yaml vs the `velero` cli +(hooks) + +* [Backup][1] +* [Restore][2] +* [Schedule][3] +* [BackupStorageLocation][4] +* [VolumeSnapshotLocation][5] + +[1]: backup.md +[2]: restore.md +[3]: schedule.md +[4]: backupstoragelocation.md +[5]: volumesnapshotlocation.md diff --git a/site/content/docs/v1.8/api-types/_index.md b/site/content/docs/v1.8/api-types/_index.md new file mode 100644 index 000000000..e60832500 --- /dev/null +++ b/site/content/docs/v1.8/api-types/_index.md @@ -0,0 +1,19 @@ +--- +layout: docs +title: API types +--- + +Here's a list the API types that have some functionality that you can only configure via json/yaml vs the `velero` cli +(hooks) + +* [Backup][1] +* [Restore][2] +* [Schedule][3] +* [BackupStorageLocation][4] +* [VolumeSnapshotLocation][5] + +[1]: backup.md +[2]: restore.md +[3]: schedule.md +[4]: backupstoragelocation.md +[5]: volumesnapshotlocation.md diff --git a/site/content/docs/v1.8/api-types/backup.md b/site/content/docs/v1.8/api-types/backup.md new file mode 100644 index 000000000..5620c9ff6 --- /dev/null +++ b/site/content/docs/v1.8/api-types/backup.md @@ -0,0 +1,148 @@ +--- +title: "Backup API Type" +layout: docs +--- + +## Use + +Use the `Backup` API type to request the Velero server to perform a backup. Once created, the +Velero Server immediately starts the backup process. + +## API GroupVersion + +Backup belongs to the API group version `velero.io/v1`. + +## Definition + +Here is a sample `Backup` object with each of the fields documented: + +```yaml +# Standard Kubernetes API Version declaration. Required. +apiVersion: velero.io/v1 +# Standard Kubernetes Kind declaration. Required. +kind: Backup +# Standard Kubernetes metadata. Required. +metadata: + # Backup name. May be any valid Kubernetes object name. Required. + name: a + # Backup namespace. Must be the namespace of the Velero server. Required. + namespace: velero +# Parameters about the backup. Required. +spec: + # Array of namespaces to include in the backup. If unspecified, all namespaces are included. + # Optional. + includedNamespaces: + - '*' + # Array of namespaces to exclude from the backup. Optional. + excludedNamespaces: + - some-namespace + # Array of resources to include in the backup. Resources may be shortcuts (for example 'po' for 'pods') + # or fully-qualified. If unspecified, all resources are included. Optional. + includedResources: + - '*' + # Array of resources to exclude from the backup. Resources may be shortcuts (for example 'po' for 'pods') + # or fully-qualified. Optional. + excludedResources: + - storageclasses.storage.k8s.io + # Whether or not to include cluster-scoped resources. Valid values are true, false, and + # null/unset. If true, all cluster-scoped resources are included (subject to included/excluded + # resources and the label selector). If false, no cluster-scoped resources are included. If unset, + # all cluster-scoped resources are included if and only if all namespaces are included and there are + # no excluded namespaces. Otherwise, if there is at least one namespace specified in either + # includedNamespaces or excludedNamespaces, then the only cluster-scoped resources that are backed + # up are those associated with namespace-scoped resources included in the backup. For example, if a + # PersistentVolumeClaim is included in the backup, its associated PersistentVolume (which is + # cluster-scoped) would also be backed up. + includeClusterResources: null + # Individual objects must match this label selector to be included in the backup. Optional. + labelSelector: + matchLabels: + app: velero + component: server + # Whether or not to snapshot volumes. This only applies to PersistentVolumes for Azure, GCE, and + # AWS. Valid values are true, false, and null/unset. If unset, Velero performs snapshots as long as + # a persistent volume provider is configured for Velero. + snapshotVolumes: null + # Where to store the tarball and logs. + storageLocation: aws-primary + # The list of locations in which to store volume snapshots created for this backup. + volumeSnapshotLocations: + - aws-primary + - gcp-primary + # The amount of time before this backup is eligible for garbage collection. If not specified, + # a default value of 30 days will be used. The default can be configured on the velero server + # by passing the flag --default-backup-ttl. + ttl: 24h0m0s + # Whether restic should be used to take a backup of all pod volumes by default. + defaultVolumesToRestic: true + # Actions to perform at different times during a backup. The only hook supported is + # executing a command in a container in a pod using the pod exec API. Optional. + hooks: + # Array of hooks that are applicable to specific resources. Optional. + resources: + - + # Name of the hook. Will be displayed in backup log. + name: my-hook + # Array of namespaces to which this hook applies. If unspecified, the hook applies to all + # namespaces. Optional. + includedNamespaces: + - '*' + # Array of namespaces to which this hook does not apply. Optional. + excludedNamespaces: + - some-namespace + # Array of resources to which this hook applies. The only resource supported at this time is + # pods. + includedResources: + - pods + # Array of resources to which this hook does not apply. Optional. + excludedResources: [] + # This hook only applies to objects matching this label selector. Optional. + labelSelector: + matchLabels: + app: velero + component: server + # An array of hooks to run before executing custom actions. Only "exec" hooks are supported. + pre: + - + # The type of hook. This must be "exec". + exec: + # The name of the container where the command will be executed. If unspecified, the + # first container in the pod will be used. Optional. + container: my-container + # The command to execute, specified as an array. Required. + command: + - /bin/uname + - -a + # How to handle an error executing the command. Valid values are Fail and Continue. + # Defaults to Fail. Optional. + onError: Fail + # How long to wait for the command to finish executing. Defaults to 30 seconds. Optional. + timeout: 10s + # An array of hooks to run after all custom actions and additional items have been + # processed. Only "exec" hooks are supported. + post: + # Same content as pre above. +# Status about the Backup. Users should not set any data here. +status: + # The version of this Backup. The only version supported is 1. + version: 1 + # The date and time when the Backup is eligible for garbage collection. + expiration: null + # The current phase. Valid values are New, FailedValidation, InProgress, Completed, PartiallyFailed, Failed. + phase: "" + # An array of any validation errors encountered. + validationErrors: null + # Date/time when the backup started being processed. + startTimestamp: 2019-04-29T15:58:43Z + # Date/time when the backup finished being processed. + completionTimestamp: 2019-04-29T15:58:56Z + # Number of volume snapshots that Velero tried to create for this backup. + volumeSnapshotsAttempted: 2 + # Number of volume snapshots that Velero successfully created for this backup. + volumeSnapshotsCompleted: 1 + # Number of warnings that were logged by the backup. + warnings: 2 + # Number of errors that were logged by the backup. + errors: 0 + +``` diff --git a/site/content/docs/v1.8/api-types/backupstoragelocation.md b/site/content/docs/v1.8/api-types/backupstoragelocation.md new file mode 100644 index 000000000..b6c58ece7 --- /dev/null +++ b/site/content/docs/v1.8/api-types/backupstoragelocation.md @@ -0,0 +1,54 @@ +--- +title: "Velero Backup Storage Locations" +layout: docs +--- + +## Backup Storage Location + +Velero can store backups in a number of locations. These are represented in the cluster via the `BackupStorageLocation` CRD. + +Velero must have at least one `BackupStorageLocation`. By default, this is expected to be named `default`, however the name can be changed by specifying `--default-backup-storage-location` on `velero server`. Backups that do not explicitly specify a storage location will be saved to this `BackupStorageLocation`. + +A sample YAML `BackupStorageLocation` looks like the following: + +```yaml +apiVersion: velero.io/v1 +kind: BackupStorageLocation +metadata: + name: default + namespace: velero +spec: + backupSyncPeriod: 2m0s + provider: aws + objectStorage: + bucket: myBucket + credential: + name: secret-name + key: key-in-secret + config: + region: us-west-2 + profile: "default" +``` + +### Parameter Reference + +The configurable parameters are as follows: + +#### Main config parameters + +{{< table caption="Main config parameters" >}} +| Key | Type | Default | Meaning | +| --- | --- | --- | --- | +| `provider` | String | Required Field | The name for whichever object storage provider will be used to store the backups. See [your object storage provider's plugin documentation](../supported-providers) for the appropriate value to use. | +| `objectStorage` | ObjectStorageLocation | Required Field | Specification of the object storage for the given provider. | +| `objectStorage/bucket` | String | Required Field | The storage bucket where backups are to be uploaded. | +| `objectStorage/prefix` | String | Optional Field | The directory inside a storage bucket where backups are to be uploaded. | +| `objectStorage/caCert` | String | Optional Field | A base64 encoded CA bundle to be used when verifying TLS connections | +| `config` | map[string]string | None (Optional) | Provider-specific configuration keys/values to be passed to the object store plugin. See [your object storage provider's plugin documentation](../supported-providers) for details. | +| `accessMode` | String | `ReadWrite` | How Velero can access the backup storage location. Valid values are `ReadWrite`, `ReadOnly`. | +| `backupSyncPeriod` | metav1.Duration | Optional Field | How frequently Velero should synchronize backups in object storage. Default is Velero's server backup sync period. Set this to `0s` to disable sync. | +| `validationFrequency` | metav1.Duration | Optional Field | How frequently Velero should validate the object storage . Default is Velero's server validation frequency. Set this to `0s` to disable validation. Default 1 minute. | +| `credential` | [corev1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#secretkeyselector-v1-core) | Optional Field | The credential information to be used with this location. | +| `credential/name` | String | Optional Field | The name of the secret within the Velero namespace which contains the credential information. | +| `credential/key` | String | Optional Field | The key to use within the secret. | +{{< /table >}} diff --git a/site/content/docs/v1.8/api-types/restore.md b/site/content/docs/v1.8/api-types/restore.md new file mode 100644 index 000000000..817e549bc --- /dev/null +++ b/site/content/docs/v1.8/api-types/restore.md @@ -0,0 +1,168 @@ +--- +title: "Restore API Type" +layout: docs +--- + +## Use + +The `Restore` API type is used as a request for the Velero server to perform a Restore. Once created, the +Velero Server immediately starts the Restore process. + +## API GroupVersion + +Restore belongs to the API group version `velero.io/v1`. + +## Definition + +Here is a sample `Restore` object with each of the fields documented: + +```yaml +# Standard Kubernetes API Version declaration. Required. +apiVersion: velero.io/v1 +# Standard Kubernetes Kind declaration. Required. +kind: Restore +# Standard Kubernetes metadata. Required. +metadata: + # Restore name. May be any valid Kubernetes object name. Required. + name: a-very-special-backup-0000111122223333 + # Restore namespace. Must be the namespace of the Velero server. Required. + namespace: velero +# Parameters about the restore. Required. +spec: + # BackupName is the unique name of the Velero backup to restore from. + backupName: a-very-special-backup + # Array of namespaces to include in the restore. If unspecified, all namespaces are included. + # Optional. + includedNamespaces: + - '*' + # Array of namespaces to exclude from the restore. Optional. + excludedNamespaces: + - some-namespace + # Array of resources to include in the restore. Resources may be shortcuts (for example 'po' for 'pods') + # or fully-qualified. If unspecified, all resources are included. Optional. + includedResources: + - '*' + # Array of resources to exclude from the restore. Resources may be shortcuts (for example 'po' for 'pods') + # or fully-qualified. Optional. + excludedResources: + - storageclasses.storage.k8s.io + # Whether or not to include cluster-scoped resources. Valid values are true, false, and + # null/unset. If true, all cluster-scoped resources are included (subject to included/excluded + # resources and the label selector). If false, no cluster-scoped resources are included. If unset, + # all cluster-scoped resources are included if and only if all namespaces are included and there are + # no excluded namespaces. Otherwise, if there is at least one namespace specified in either + # includedNamespaces or excludedNamespaces, then the only cluster-scoped resources that are backed + # up are those associated with namespace-scoped resources included in the restore. For example, if a + # PersistentVolumeClaim is included in the restore, its associated PersistentVolume (which is + # cluster-scoped) would also be backed up. + includeClusterResources: null + # Individual objects must match this label selector to be included in the restore. Optional. + labelSelector: + matchLabels: + app: velero + component: server + # NamespaceMapping is a map of source namespace names to + # target namespace names to restore into. Any source namespaces not + # included in the map will be restored into namespaces of the same name. + namespaceMapping: + namespace-backup-from: namespace-to-restore-to + # RestorePVs specifies whether to restore all included PVs + # from snapshot (via the cloudprovider). + restorePVs: true + # ScheduleName is the unique name of the Velero schedule + # to restore from. If specified, and BackupName is empty, Velero will + # restore from the most recent successful backup created from this schedule. + scheduleName: my-scheduled-backup-name + # Actions to perform during or post restore. The only hooks currently supported are + # adding an init container to a pod before it can be restored and executing a command in a + # restored pod's container. Optional. + hooks: + # Array of hooks that are applicable to specific resources. Optional. + resources: + # Name is the name of this hook. + - name: restore-hook-1 + # Array of namespaces to which this hook applies. If unspecified, the hook applies to all + # namespaces. Optional. + includedNamespaces: + - ns1 + # Array of namespaces to which this hook does not apply. Optional. + excludedNamespaces: + - ns3 + # Array of resources to which this hook applies. The only resource supported at this time is + # pods. + includedResources: + - pods + # Array of resources to which this hook does not apply. Optional. + excludedResources: [] + # This hook only applies to objects matching this label selector. Optional. + labelSelector: + matchLabels: + app: velero + component: server + # An array of hooks to run during or after restores. Currently only "init" and "exec" hooks + # are supported. + postHooks: + # The type of the hook. This must be "init" or "exec". + - init: + # An array of container specs to be added as init containers to pods to which this hook applies to. + initContainers: + - name: restore-hook-init1 + image: alpine:latest + # Mounting volumes from the podSpec to which this hooks applies to. + volumeMounts: + - mountPath: /restores/pvc1-vm + # Volume name from the podSpec + name: pvc1-vm + command: + - /bin/ash + - -c + - echo -n "FOOBARBAZ" >> /restores/pvc1-vm/foobarbaz + - name: restore-hook-init2 + image: alpine:latest + # Mounting volumes from the podSpec to which this hooks applies to. + volumeMounts: + - mountPath: /restores/pvc2-vm + # Volume name from the podSpec + name: pvc2-vm + command: + - /bin/ash + - -c + - echo -n "DEADFEED" >> /restores/pvc2-vm/deadfeed + - exec: + # The container name where the hook will be executed. Defaults to the first container. + # Optional. + container: foo + # The command that will be executed in the container. Required. + command: + - /bin/bash + - -c + - "psql < /backup/backup.sql" + # How long to wait for a container to become ready. This should be long enough for the + # container to start plus any preceding hooks in the same container to complete. The wait + # timeout begins when the container is restored and may require time for the image to pull + # and volumes to mount. If not set the restore will wait indefinitely. Optional. + waitTimeout: 5m + # How long to wait once execution begins. Defaults to 30 seconds. Optional. + execTimeout: 1m + # How to handle execution failures. Valid values are `Fail` and `Continue`. Defaults to + # `Continue`. With `Continue` mode, execution failures are logged only. With `Fail` mode, + # no more restore hooks will be executed in any container in any pod and the status of the + # Restore will be `PartiallyFailed`. Optional. + onError: Continue +# RestoreStatus captures the current status of a Velero restore. Users should not set any data here. +status: + # The current phase. Valid values are New, FailedValidation, InProgress, Completed, PartiallyFailed, Failed. + phase: "" + # An array of any validation errors encountered. + validationErrors: null + # Number of warnings that were logged by the restore. + warnings: 2 + # Errors is a count of all error messages that were generated + # during execution of the restore. The actual errors are stored in object + # storage. + errors: 0 + # FailureReason is an error that caused the entire restore + # to fail. + failureReason: + +``` diff --git a/site/content/docs/v1.8/api-types/schedule.md b/site/content/docs/v1.8/api-types/schedule.md new file mode 100644 index 000000000..9c5dc26ce --- /dev/null +++ b/site/content/docs/v1.8/api-types/schedule.md @@ -0,0 +1,142 @@ +--- +title: "Schedule API Type" +layout: docs +--- + +## Use + +The `Schedule` API type is used as a repeatable request for the Velero server to perform a backup for a given cron notation. Once created, the +Velero Server will start the backup process. It will then wait for the next valid point of the given cron expression and execute the backup +process on a repeating basis. + +## API GroupVersion + +Schedule belongs to the API group version `velero.io/v1`. + +## Definition + +Here is a sample `Schedule` object with each of the fields documented: + +```yaml +# Standard Kubernetes API Version declaration. Required. +apiVersion: velero.io/v1 +# Standard Kubernetes Kind declaration. Required. +kind: Schedule +# Standard Kubernetes metadata. Required. +metadata: + # Schedule name. May be any valid Kubernetes object name. Required. + name: a + # Schedule namespace. Must be the namespace of the Velero server. Required. + namespace: velero +# Parameters about the scheduled backup. Required. +spec: + # Schedule is a Cron expression defining when to run the Backup + schedule: 0 7 * * * + # Template is the spec that should be used for each backup triggered by this schedule. + template: + # Array of namespaces to include in the scheduled backup. If unspecified, all namespaces are included. + # Optional. + includedNamespaces: + - '*' + # Array of namespaces to exclude from the scheduled backup. Optional. + excludedNamespaces: + - some-namespace + # Array of resources to include in the scheduled backup. Resources may be shortcuts (for example 'po' for 'pods') + # or fully-qualified. If unspecified, all resources are included. Optional. + includedResources: + - '*' + # Array of resources to exclude from the scheduled backup. Resources may be shortcuts (for example 'po' for 'pods') + # or fully-qualified. Optional. + excludedResources: + - storageclasses.storage.k8s.io + # Whether or not to include cluster-scoped resources. Valid values are true, false, and + # null/unset. If true, all cluster-scoped resources are included (subject to included/excluded + # resources and the label selector). If false, no cluster-scoped resources are included. If unset, + # all cluster-scoped resources are included if and only if all namespaces are included and there are + # no excluded namespaces. Otherwise, if there is at least one namespace specified in either + # includedNamespaces or excludedNamespaces, then the only cluster-scoped resources that are backed + # up are those associated with namespace-scoped resources included in the scheduled backup. For example, if a + # PersistentVolumeClaim is included in the backup, its associated PersistentVolume (which is + # cluster-scoped) would also be backed up. + includeClusterResources: null + # Individual objects must match this label selector to be included in the scheduled backup. Optional. + labelSelector: + matchLabels: + app: velero + component: server + # Whether or not to snapshot volumes. This only applies to PersistentVolumes for Azure, GCE, and + # AWS. Valid values are true, false, and null/unset. If unset, Velero performs snapshots as long as + # a persistent volume provider is configured for Velero. + snapshotVolumes: null + # Where to store the tarball and logs. + storageLocation: aws-primary + # The list of locations in which to store volume snapshots created for backups under this schedule. + volumeSnapshotLocations: + - aws-primary + - gcp-primary + # The amount of time before backups created on this schedule are eligible for garbage collection. If not specified, + # a default value of 30 days will be used. The default can be configured on the velero server + # by passing the flag --default-backup-ttl. + ttl: 24h0m0s + # Whether restic should be used to take a backup of all pod volumes by default. + defaultVolumesToRestic: true + # The labels you want on backup objects, created from this schedule (instead of copying the labels you have on schedule object itself). + # When this field is set, the labels from the Schedule resource are not copied to the Backup resource. + metadata: + labels: + labelname: somelabelvalue + # Actions to perform at different times during a backup. The only hook supported is + # executing a command in a container in a pod using the pod exec API. Optional. + hooks: + # Array of hooks that are applicable to specific resources. Optional. + resources: + - + # Name of the hook. Will be displayed in backup log. + name: my-hook + # Array of namespaces to which this hook applies. If unspecified, the hook applies to all + # namespaces. Optional. + includedNamespaces: + - '*' + # Array of namespaces to which this hook does not apply. Optional. + excludedNamespaces: + - some-namespace + # Array of resources to which this hook applies. The only resource supported at this time is + # pods. + includedResources: + - pods + # Array of resources to which this hook does not apply. Optional. + excludedResources: [] + # This hook only applies to objects matching this label selector. Optional. + labelSelector: + matchLabels: + app: velero + component: server + # An array of hooks to run before executing custom actions. Only "exec" hooks are supported. + pre: + - + # The type of hook. This must be "exec". + exec: + # The name of the container where the command will be executed. If unspecified, the + # first container in the pod will be used. Optional. + container: my-container + # The command to execute, specified as an array. Required. + command: + - /bin/uname + - -a + # How to handle an error executing the command. Valid values are Fail and Continue. + # Defaults to Fail. Optional. + onError: Fail + # How long to wait for the command to finish executing. Defaults to 30 seconds. Optional. + timeout: 10s + # An array of hooks to run after all custom actions and additional items have been + # processed. Only "exec" hooks are supported. + post: + # Same content as pre above. +status: + # The current phase of the latest scheduled backup. Valid values are New, FailedValidation, InProgress, Completed, PartiallyFailed, Failed. + phase: "" + # Date/time of the last backup for a given schedule + lastBackup: + # An array of any validation errors encountered. + validationErrors: +``` diff --git a/site/content/docs/v1.8/api-types/volumesnapshotlocation.md b/site/content/docs/v1.8/api-types/volumesnapshotlocation.md new file mode 100644 index 000000000..28ac33222 --- /dev/null +++ b/site/content/docs/v1.8/api-types/volumesnapshotlocation.md @@ -0,0 +1,40 @@ +--- +title: "Velero Volume Snapshot Location" +layout: docs +--- + +## Volume Snapshot Location + +A volume snapshot location is the location in which to store the volume snapshots created for a backup. + +Velero can be configured to take snapshots of volumes from multiple providers. Velero also allows you to configure multiple possible `VolumeSnapshotLocation` per provider, although you can only select one location per provider at backup time. + +Each VolumeSnapshotLocation describes a provider + location. These are represented in the cluster via the `VolumeSnapshotLocation` CRD. Velero must have at least one `VolumeSnapshotLocation` per cloud provider. + +A sample YAML `VolumeSnapshotLocation` looks like the following: + +```yaml +apiVersion: velero.io/v1 +kind: VolumeSnapshotLocation +metadata: + name: aws-default + namespace: velero +spec: + provider: aws + config: + region: us-west-2 + profile: "default" +``` + +### Parameter Reference + +The configurable parameters are as follows: + +#### Main config parameters + +{{< table caption="Main config parameters" >}} +| Key | Type | Default | Meaning | +| --- | --- | --- | --- | +| `provider` | String | Required Field | The name for whichever storage provider will be used to create/store the volume snapshots. See [your volume snapshot provider's plugin documentation](../supported-providers) for the appropriate value to use. | +| `config` | map string string | None (Optional) | Provider-specific configuration keys/values to be passed to the volume snapshotter plugin. See [your volume snapshot provider's plugin documentation](../supported-providers) for details. | +{{< /table >}} diff --git a/site/content/docs/v1.8/backup-hooks.md b/site/content/docs/v1.8/backup-hooks.md new file mode 100644 index 000000000..de6a58c53 --- /dev/null +++ b/site/content/docs/v1.8/backup-hooks.md @@ -0,0 +1,92 @@ +--- +title: "Backup Hooks" +layout: docs +--- + +Velero supports executing commands in containers in pods during a backup. + +## Backup Hooks + +When performing a backup, you can specify one or more commands to execute in a container in a pod +when that pod is being backed up. The commands can be configured to run *before* any custom action +processing ("pre" hooks), or after all custom actions have been completed and any additional items +specified by custom action have been backed up ("post" hooks). Note that hooks are _not_ executed within a shell +on the containers. + +There are two ways to specify hooks: annotations on the pod itself, and in the Backup spec. + +### Specifying Hooks As Pod Annotations + +You can use the following annotations on a pod to make Velero execute a hook when backing up the pod: + +#### Pre hooks + +* `pre.hook.backup.velero.io/container` + * The container where the command should be executed. Defaults to the first container in the pod. Optional. +* `pre.hook.backup.velero.io/command` + * The command to execute. If you need multiple arguments, specify the command as a JSON array, such as `["/usr/bin/uname", "-a"]` +* `pre.hook.backup.velero.io/on-error` + * What to do if the command returns a non-zero exit code. Defaults to Fail. Valid values are Fail and Continue. Optional. +* `pre.hook.backup.velero.io/timeout` + * How long to wait for the command to execute. The hook is considered in error if the command exceeds the timeout. Defaults to 30s. Optional. + + +#### Post hooks + +* `post.hook.backup.velero.io/container` + * The container where the command should be executed. Defaults to the first container in the pod. Optional. +* `post.hook.backup.velero.io/command` + * The command to execute. If you need multiple arguments, specify the command as a JSON array, such as `["/usr/bin/uname", "-a"]` +* `post.hook.backup.velero.io/on-error` + * What to do if the command returns a non-zero exit code. Defaults to Fail. Valid values are Fail and Continue. Optional. +* `post.hook.backup.velero.io/timeout` + * How long to wait for the command to execute. The hook is considered in error if the command exceeds the timeout. Defaults to 30s. Optional. + +### Specifying Hooks in the Backup Spec + +Please see the documentation on the [Backup API Type][1] for how to specify hooks in the Backup +spec. + +## Hook Example with fsfreeze + +This examples walks you through using both pre and post hooks for freezing a file system. Freezing the +file system is useful to ensure that all pending disk I/O operations have completed prior to taking a snapshot. + +This example uses [examples/nginx-app/with-pv.yaml][2]. Follow the [steps for your provider][3] to +setup this example. + +### Annotations + +The Velero [example/nginx-app/with-pv.yaml][2] serves as an example of adding the pre and post hook annotations directly +to your declarative deployment. Below is an example of what updating an object in place might look like. + +```shell +kubectl annotate pod -n nginx-example -l app=nginx \ + pre.hook.backup.velero.io/command='["/sbin/fsfreeze", "--freeze", "/var/log/nginx"]' \ + pre.hook.backup.velero.io/container=fsfreeze \ + post.hook.backup.velero.io/command='["/sbin/fsfreeze", "--unfreeze", "/var/log/nginx"]' \ + post.hook.backup.velero.io/container=fsfreeze +``` + +Now test the pre and post hooks by creating a backup. You can use the Velero logs to verify that the pre and post +hooks are running and exiting without error. + +```shell +velero backup create nginx-hook-test + +velero backup get nginx-hook-test +velero backup logs nginx-hook-test | grep hookCommand +``` + +## Using Multiple Commands + +To use multiple commands, wrap your target command in a shell and separate them with `;`, `&&`, or other shell conditional constructs. + +```shell + pre.hook.backup.velero.io/command='["/bin/bash", "-c", "echo hello > hello.txt && echo goodbye > goodbye.txt"]' +``` + + +[1]: api-types/backup.md +[2]: https://github.com/vmware-tanzu/velero/blob/v1.8.0/examples/nginx-app/with-pv.yaml +[3]: cloud-common.md diff --git a/site/content/docs/v1.8/backup-reference.md b/site/content/docs/v1.8/backup-reference.md new file mode 100644 index 000000000..60e019188 --- /dev/null +++ b/site/content/docs/v1.8/backup-reference.md @@ -0,0 +1,70 @@ +--- +title: "Backup Reference" +layout: docs +--- + +## Exclude Specific Items from Backup + +It is possible to exclude individual items from being backed up, even if they match the resource/namespace/label selectors defined in the backup spec. To do this, label the item as follows: + +```bash +kubectl label -n / velero.io/exclude-from-backup=true +``` + +## Specify Backup Orders of Resources of Specific Kind + +To backup resources of specific Kind in a specific order, use option --ordered-resources to specify a mapping Kinds to an ordered list of specific resources of that Kind. Resource names are separated by commas and their names are in format 'namespace/resourcename'. For cluster scope resource, simply use resource name. Key-value pairs in the mapping are separated by semi-colon. Kind name is in plural form. + +```bash +velero backup create backupName --include-cluster-resources=true --ordered-resources 'pods=ns1/pod1,ns1/pod2;persistentvolumes=pv4,pv8' --include-namespaces=ns1 +velero backup create backupName --ordered-resources 'statefulsets=ns1/sts1,ns1/sts0' --include-namespaces=ns1 +``` +## Schedule a Backup + +The **schedule** operation allows you to create a backup of your data at a specified time, defined by a [Cron expression](https://en.wikipedia.org/wiki/Cron). + +``` +velero schedule create NAME --schedule="* * * * *" [flags] +``` + +Cron schedules use the following format. + +``` +# ┌───────────── minute (0 - 59) +# │ ┌───────────── hour (0 - 23) +# │ │ ┌───────────── day of the month (1 - 31) +# │ │ │ ┌───────────── month (1 - 12) +# │ │ │ │ ┌───────────── day of the week (0 - 6) (Sunday to Saturday; +# │ │ │ │ │ 7 is also Sunday on some systems) +# │ │ │ │ │ +# │ │ │ │ │ +# * * * * * +``` + +For example, the command below creates a backup that runs every day at 3am. + +``` +velero schedule create example-schedule --schedule="0 3 * * *" +``` + +This command will create the backup, `example-schedule`, within Velero, but the backup will not be taken until the next scheduled time, 3am. Backups created by a schedule are saved with the name `-`, where `` is formatted as *YYYYMMDDhhmmss*. For a full list of available configuration flags use the Velero CLI help command. + +``` +velero schedule create --help +``` + +Once you create the scheduled backup, you can then trigger it manually using the `velero backup` command. + +``` +velero backup create --from-schedule example-schedule +``` + +This command will immediately trigger a new backup based on your template for `example-schedule`. This will not affect the backup schedule, and another backup will trigger at the scheduled time. + +## Kubernetes API Pagination + +By default, Velero will paginate the LIST API call for each resource type in the Kubernetes API when collecting items into a backup. The `--client-page-size` flag for the Velero server configures the size of each page. + +Depending on the cluster's scale, tuning the page size can improve backup performance. You can experiment with higher values, noting their impact on the relevant `apiserver_request_duration_seconds_*` metrics from the Kubernetes apiserver. + +Pagination can be entirely disabled by setting `--client-page-size` to `0`. This will request all items in a single unpaginated LIST call. diff --git a/site/content/docs/v1.8/basic-install.md b/site/content/docs/v1.8/basic-install.md new file mode 100644 index 000000000..07c4ef687 --- /dev/null +++ b/site/content/docs/v1.8/basic-install.md @@ -0,0 +1,73 @@ +--- +title: "Basic Install" +layout: docs +--- + +Use this doc to get a basic installation of Velero. +Refer [this document](customize-installation.md) to customize your installation. + +## Prerequisites + +- Access to a Kubernetes cluster, v1.16 or later, with DNS and container networking enabled. +- `kubectl` installed locally + +Velero uses object storage to store backups and associated artifacts. It also optionally integrates with supported block storage systems to snapshot your persistent volumes. Before beginning the installation process, you should identify the object storage provider and optional block storage provider(s) you'll be using from the list of [compatible providers][0]. + +Velero supports storage providers for both cloud-provider environments and on-premises environments. For more details on on-premises scenarios, see the [on-premises documentation][2]. + +### Velero on Windows + +Velero does not officially support Windows. In testing, the Velero team was able to backup stateless Windows applications only. The restic integration and backups of stateful applications or PersistentVolumes were not supported. + +If you want to perform your own testing of Velero on Windows, you must deploy Velero as a Windows container. Velero does not provide official Windows images, but its possible for you to build your own Velero Windows container image to use. Note that you must build this image on a Windows node. + +## Install the CLI + +### Option 1: MacOS - Homebrew + +On macOS, you can use [Homebrew](https://brew.sh) to install the `velero` client: + +```bash +brew install velero +``` + +### Option 2: GitHub release + +1. Download the [latest release][1]'s tarball for your client platform. +1. Extract the tarball: + + ```bash + tar -xvf .tar.gz + ``` + +1. Move the extracted `velero` binary to somewhere in your `$PATH` (`/usr/local/bin` for most users). + +### Option 3: Windows - Chocolatey + +On Windows, you can use [Chocolatey](https://chocolatey.org/install) to install the [velero](https://chocolatey.org/packages/velero) client: + +```powershell +choco install velero +``` + +## Install and configure the server components + +There are two supported methods for installing the Velero server components: + +- the `velero install` CLI command +- the [Helm chart](https://vmware-tanzu.github.io/helm-charts/) + +Velero uses storage provider plugins to integrate with a variety of storage systems to support backup and snapshot operations. The steps to install and configure the Velero server components along with the appropriate plugins are specific to your chosen storage provider. To find installation instructions for your chosen storage provider, follow the documentation link for your provider at our [supported storage providers][0] page + +_Note: if your object storage provider is different than your volume snapshot provider, follow the installation instructions for your object storage provider first, then return here and follow the instructions to [add your volume snapshot provider][4]._ + +## Command line Autocompletion + +Please refer to [this part of the documentation][5]. + +[0]: supported-providers.md +[1]: https://github.com/vmware-tanzu/velero/releases/latest +[2]: on-premises.md +[3]: overview-plugins.md +[4]: customize-installation.md#install-an-additional-volume-snapshot-provider +[5]: customize-installation.md#optional-velero-cli-configurations \ No newline at end of file diff --git a/site/content/docs/v1.8/build-from-source.md b/site/content/docs/v1.8/build-from-source.md new file mode 100644 index 000000000..df9f738cf --- /dev/null +++ b/site/content/docs/v1.8/build-from-source.md @@ -0,0 +1,200 @@ +--- +title: "Build from source" +layout: docs +--- + +## Prerequisites + +* Access to a Kubernetes cluster, version 1.7 or later. +* A DNS server on the cluster +* `kubectl` installed +* [Go][5] installed (minimum version 1.8) + +## Get the source + +### Option 1) Get latest (recommended) + +```bash +mkdir $HOME/go +export GOPATH=$HOME/go +go get github.com/vmware-tanzu/velero +``` + +Where `go` is your [import path][4] for Go. + +For Go development, it is recommended to add the Go import path (`$HOME/go` in this example) to your path. + +### Option 2) Release archive + +Download the archive named `Source code` from the [release page][22] and extract it in your Go import path as `src/github.com/vmware-tanzu/velero`. + +Note that the Makefile targets assume building from a git repository. When building from an archive, you will be limited to the `go build` commands described below. + +## Build + +There are a number of different ways to build `velero` depending on your needs. This section outlines the main possibilities. + +When building by using `make`, it will place the binaries under `_output/bin/$GOOS/$GOARCH`. For example, you will find the binary for darwin here: `_output/bin/darwin/amd64/velero`, and the binary for linux here: `_output/bin/linux/amd64/velero`. `make` will also splice version and git commit information in so that `velero version` displays proper output. + +Note: `velero install` will also use the version information to determine which tagged image to deploy. If you would like to overwrite what image gets deployed, use the `image` flag (see below for instructions on how to build images). + +### Build the binary + +To build the `velero` binary on your local machine, compiled for your OS and architecture, run one of these two commands: + +```bash +go build ./cmd/velero +``` + +```bash +make local +``` + +### Cross compiling + +To build the velero binary targeting linux/amd64 within a build container on your local machine, run: + +```bash +make build +``` + +For any specific platform, run `make build--`. + +For example, to build for the Mac, run `make build-darwin-amd64`. + +Velero's `Makefile` has a convenience target, `all-build`, that builds the following platforms: + +* linux-amd64 +* linux-arm +* linux-arm64 +* linux-ppc64le +* darwin-amd64 +* windows-amd64 + +## Making images and updating Velero + +If after installing Velero you would like to change the image used by its deployment to one that contains your code changes, you may do so by updating the image: + +```bash +kubectl -n velero set image deploy/velero velero=myimagerepo/velero:$VERSION +``` + +To build a Velero container image, you need to configure `buildx` first. + +### Buildx + +Docker Buildx is a CLI plugin that extends the docker command with the full support of the features provided by Moby BuildKit builder toolkit. It provides the same user experience as docker build with many new features like creating scoped builder instances and building against multiple nodes concurrently. + +More information in the [docker docs][23] and in the [buildx github][24] repo. + +### Image building + +Set the `$REGISTRY` environment variable. For example, if you want to build the `gcr.io/my-registry/velero:main` image, set `$REGISTRY` to `gcr.io/my-registry`. If this variable is not set, the default is `velero`. + +Optionally, set the `$VERSION` environment variable to change the image tag or `$BIN` to change which binary to build a container image for. Then, run: + +```bash +make container +``` +_Note: To build build container images for both `velero` and `velero-restic-restore-helper`, run: `make all-containers`_ + +### Publishing container images to a registry + +To publish container images to a registry, the following one time setup is necessary: + +1. If you are building cross platform container images + ```bash + $ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + ``` +1. Create and bootstrap a new docker buildx builder + ```bash + $ docker buildx create --use --name builder + builder + $ docker buildx inspect --bootstrap + [+] Building 2.6s (1/1) FINISHED + => [internal] booting buildkit 2.6s + => => pulling image moby/buildkit:buildx-stable-1 1.9s + => => creating container buildx_buildkit_builder0 0.7s + Name: builder + Driver: docker-container + + Nodes: + Name: builder0 + Endpoint: unix:///var/run/docker.sock + Status: running + Platforms: linux/amd64, linux/arm64, linux/ppc64le, linux/s390x, linux/386, linux/arm/v7, linux/arm/v6 + ``` + NOTE: Without the above setup, the output of `docker buildx inspect --bootstrap` will be: + ```bash + $ docker buildx inspect --bootstrap + Name: default + Driver: docker + + Nodes: + Name: default + Endpoint: default + Status: running + Platforms: linux/amd64, linux/arm64, linux/ppc64le, linux/s390x, linux/386, linux/arm/v7, linux/arm/v6 + ``` + And the `REGISTRY=myrepo BUILDX_OUTPUT_TYPE=registry make container` will fail with the below error: + ```bash + $ REGISTRY=ashishamarnath BUILDX_PLATFORMS=linux/arm64 BUILDX_OUTPUT_TYPE=registry make container + auto-push is currently not implemented for docker driver + make: *** [container] Error 1 + ``` + +Having completed the above one time setup, now the output of `docker buildx inspect --bootstrap` should be like + +```bash +$ docker buildx inspect --bootstrap +Name: builder +Driver: docker-container + +Nodes: +Name: builder0 +Endpoint: unix:///var/run/docker.sock +Status: running +Platforms: linux/amd64, linux/arm64, linux/riscv64, linux/ppc64le, linux/s390x, linux/386, linux/arm/v7, linux/arm/v +``` + +Now build and push the container image by running the `make container` command with `$BUILDX_OUTPUT_TYPE` set to `registry` +```bash +$ REGISTRY=myrepo BUILDX_OUTPUT_TYPE=registry make container +``` + +### Cross platform building + +Docker `buildx` platforms supported: +* `linux/amd64` +* `linux/arm64` +* `linux/arm/v7` +* `linux/ppc64le` + +For any specific platform, run `BUILDX_PLATFORMS=/ make container` + +For example, to build an image for arm64, run: + +```bash +BUILDX_PLATFORMS=linux/arm64 make container +``` +_Note: By default, `$BUILDX_PLATFORMS` is set to `linux/amd64`_ + +With `buildx`, you can also build all supported platforms at the same time and push a multi-arch image to the registry. For example: + +```bash +REGISTRY=myrepo VERSION=foo BUILDX_PLATFORMS=linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le BUILDX_OUTPUT_TYPE=registry make all-containers +``` +_Note: when building for more than 1 platform at the same time, you need to set `BUILDX_OUTPUT_TYPE` to `registry` as local multi-arch images are not supported [yet][25]._ + +Note: if you want to update the image but not change its name, you will have to trigger Kubernetes to pick up the new image. One way of doing so is by deleting the Velero deployment pod: + +```bash +kubectl -n velero delete pods -l deploy=velero +``` + +[4]: https://blog.golang.org/organizing-go-code +[5]: https://golang.org/doc/install +[22]: https://github.com/vmware-tanzu/velero/releases +[23]: https://docs.docker.com/buildx/working-with-buildx/ +[24]: https://github.com/docker/buildx +[25]: https://github.com/moby/moby/pull/38738 diff --git a/site/content/docs/v1.8/code-standards.md b/site/content/docs/v1.8/code-standards.md new file mode 100644 index 000000000..22a2261d5 --- /dev/null +++ b/site/content/docs/v1.8/code-standards.md @@ -0,0 +1,151 @@ +--- +title: "Code Standards" +layout: docs +toc: "true" +--- + +## Opening PRs + +When opening a pull request, please fill out the checklist supplied the template. This will help others properly categorize and review your pull request. + +## Adding a changelog + +Authors are expected to include a changelog file with their pull requests. The changelog file +should be a new file created in the `changelogs/unreleased` folder. The file should follow the +naming convention of `pr-username` and the contents of the file should be your text for the +changelog. + + velero/changelogs/unreleased <- folder + 000-username <- file + +Add that to the PR. + +If a PR does not warrant a changelog, the CI check for a changelog can be skipped by applying a `changelog-not-required` label on the PR. If you are making a PR on a release branch, you should still make a new file in the `changelogs/unreleased` folder on the release branch for your change. + +## Copyright header + +Whenever a source code file is being modified, the copyright notice should be updated to our standard copyright notice. That is, it should read “Copyright the Velero contributors.” + +For new files, the entire copyright and license header must be added. + +Please note that doc files do not need a copyright header. + +## Code + +- Log messages are capitalized. + +- Error messages are kept lower-cased. + +- Wrap/add a stack only to errors that are being directly returned from non-velero code, such as an API call to the Kubernetes server. + + ```bash + errors.WithStack(err) + ``` + +- Prefer to use the utilities in the Kubernetes package [`sets`](https://godoc.org/github.com/kubernetes/apimachinery/pkg/util/sets). + + ```bash + k8s.io/apimachinery/pkg/util/sets + ``` + +## Imports + +For imports, we use the following convention: + +`` + +Example: + + import ( + corev1api "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + corev1listers "k8s.io/client-go/listers/core/v1" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" + ) + +## Mocks + +We use a package to generate mocks for our interfaces. + +Example: if you want to change this mock: https://github.com/vmware-tanzu/velero/blob/v1.8.0/pkg/restic/mocks/restorer.go + +Run: + +```bash +go get github.com/vektra/mockery/.../ +cd pkg/restic +mockery -name=Restorer +``` + +Might need to run `make update` to update the imports. + +## Kubernetes Labels + +When generating label values, be sure to pass them through the `label.GetValidName()` helper function. + +This will help ensure that the values are the proper length and format to be stored and queried. + +In general, UIDs are safe to persist as label values. + +This function is not relevant to annotation values, which do not have restrictions. + +## DCO Sign off + +All authors to the project retain copyright to their work. However, to ensure +that they are only submitting work that they have rights to, we are requiring +everyone to acknowledge this by signing their work. + +Any copyright notices in this repo should specify the authors as "the Velero contributors". + +To sign your work, just add a line like this at the end of your commit message: + +``` +Signed-off-by: Joe Beda +``` + +This can easily be done with the `--signoff` option to `git commit`. + +By doing this you state that you can certify the following (from https://developercertificate.org/): + +``` +Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2004, 2006 The Linux Foundation and its contributors. +1 Letterman Drive +Suite D4700 +San Francisco, CA, 94129 + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + + +Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. +``` diff --git a/site/content/docs/v1.8/contributions/ibm-config.md b/site/content/docs/v1.8/contributions/ibm-config.md new file mode 100644 index 000000000..4aaf823a2 --- /dev/null +++ b/site/content/docs/v1.8/contributions/ibm-config.md @@ -0,0 +1,101 @@ +--- +title: "Use IBM Cloud Object Storage as Velero's storage destination." +layout: docs +--- +You can deploy Velero on IBM [Public][5] or [Private][4] clouds, or even on any other Kubernetes cluster, but anyway you can use IBM Cloud Object Store as a destination for Velero's backups. + +To set up IBM Cloud Object Storage (COS) as Velero's destination, you: + +* Download an official release of Velero +* Create your COS instance +* Create an S3 bucket +* Define a service that can store data in the bucket +* Configure and start the Velero server + +## Download Velero + +1. Download the [latest official release's](https://github.com/vmware-tanzu/velero/releases) tarball for your client platform. + + _We strongly recommend that you use an [official release](https://github.com/vmware-tanzu/velero/releases) of +Velero. The tarballs for each release contain the `velero` command-line client. The code in the main branch +of the Velero repository is under active development and is not guaranteed to be stable!_ + +1. Extract the tarball: + + ```bash + tar -xvf .tar.gz -C /dir/to/extract/to + ``` + + The directory you extracted is called the "Velero directory" in subsequent steps. + +1. Move the `velero` binary from the Velero directory to somewhere in your PATH. + +## Create COS instance +If you don’t have a COS instance, you can create a new one, according to the detailed instructions in [Creating a new resource instance][1]. + +## Create an S3 bucket +Velero requires an object storage bucket to store backups in. See instructions in [Create some buckets to store your data][2]. + +## Define a service that can store data in the bucket. +The process of creating service credentials is described in [Service credentials][3]. +Several comments: + +1. The Velero service will write its backup into the bucket, so it requires the “Writer” access role. + +2. Velero uses an AWS S3 compatible API. Which means it authenticates using a signature created from a pair of access and secret keys — a set of HMAC credentials. You can create these HMAC credentials by specifying `{“HMAC”:true}` as an optional inline parameter. See [HMAC credentials][31] guide. + +3. After successfully creating a Service credential, you can view the JSON definition of the credential. Under the `cos_hmac_keys` entry there are `access_key_id` and `secret_access_key`. Use them in the next step. + +4. Create a Velero-specific credentials file (`credentials-velero`) in your local directory: + + ``` + [default] + aws_access_key_id= + aws_secret_access_key= + ``` + + Where the access key id and secret are the values that you got above. + +## Install and start Velero + +Install Velero, including all prerequisites, into the cluster and start the deployment. This will create a namespace called `velero`, and place a deployment named `velero` in it. + +```bash +velero install \ + --provider aws \ + --bucket \ + --secret-file ./credentials-velero \ + --use-volume-snapshots=false \ + --backup-location-config region=,s3ForcePathStyle="true",s3Url= +``` + +Velero does not have a volume snapshot plugin for IBM Cloud, so creating volume snapshots is disabled. + +Additionally, you can specify `--use-restic` to enable [restic support][16], and `--wait` to wait for the deployment to be ready. + +(Optional) Specify [CPU and memory resource requests and limits][15] for the Velero/restic pods. + +Once the installation is complete, remove the default `VolumeSnapshotLocation` that was created by `velero install`, since it's specific to AWS and won't work for IBM Cloud: + +```bash +kubectl -n velero delete volumesnapshotlocation.velero.io default +``` + +For more complex installation needs, use either the Helm chart, or add `--dry-run -o yaml` options for generating the YAML representation for the installation. + +## Installing the nginx example (optional) + +If you run the nginx example, in file `examples/nginx-app/with-pv.yaml`: + +Uncomment `storageClassName: ` and replace with your `StorageClass` name. + +[0]: namespace.md +[1]: https://cloud.ibm.com/docs/cloud-object-storage/getting-started.html +[2]: https://cloud.ibm.com/docs/cloud-object-storage/getting-started.html#create-buckets +[3]: https://cloud.ibm.com/docs/cloud-object-storage/iam?topic=cloud-object-storage-service-credentials +[31]: https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-uhc-hmac-credentials-main +[4]: https://www.ibm.com/docs/en/cloud-private +[5]: https://cloud.ibm.com/docs/containers/container_index.html#container_index +[14]: http://docs.aws.amazon.com/IAM/latest/UserGuide/introduction.html +[15]: customize-installation.md#customize-resource-requests-and-limits +[16]: restic.md diff --git a/site/content/docs/v1.8/contributions/img-for-tencent/15ccaacf00640a04ae29ceed4c86195b.png b/site/content/docs/v1.8/contributions/img-for-tencent/15ccaacf00640a04ae29ceed4c86195b.png new file mode 100644 index 000000000..61859ca50 Binary files /dev/null and b/site/content/docs/v1.8/contributions/img-for-tencent/15ccaacf00640a04ae29ceed4c86195b.png differ diff --git a/site/content/docs/v1.8/contributions/img-for-tencent/1d53b0115644d43657c2a5ece805c9b4.png b/site/content/docs/v1.8/contributions/img-for-tencent/1d53b0115644d43657c2a5ece805c9b4.png new file mode 100644 index 000000000..57de0cde0 Binary files /dev/null and b/site/content/docs/v1.8/contributions/img-for-tencent/1d53b0115644d43657c2a5ece805c9b4.png differ diff --git a/site/content/docs/v1.8/contributions/img-for-tencent/69194157ccd5e377d1e7d914fd8c0336.png b/site/content/docs/v1.8/contributions/img-for-tencent/69194157ccd5e377d1e7d914fd8c0336.png new file mode 100644 index 000000000..f363eb1a7 Binary files /dev/null and b/site/content/docs/v1.8/contributions/img-for-tencent/69194157ccd5e377d1e7d914fd8c0336.png differ diff --git a/site/content/docs/v1.8/contributions/img-for-tencent/9015313121ed7987558c88081b052574.png b/site/content/docs/v1.8/contributions/img-for-tencent/9015313121ed7987558c88081b052574.png new file mode 100644 index 000000000..921b54a16 Binary files /dev/null and b/site/content/docs/v1.8/contributions/img-for-tencent/9015313121ed7987558c88081b052574.png differ diff --git a/site/content/docs/v1.8/contributions/img-for-tencent/ceaca9ce6bc92bdce987c63d2fe71561.png b/site/content/docs/v1.8/contributions/img-for-tencent/ceaca9ce6bc92bdce987c63d2fe71561.png new file mode 100644 index 000000000..36805831b Binary files /dev/null and b/site/content/docs/v1.8/contributions/img-for-tencent/ceaca9ce6bc92bdce987c63d2fe71561.png differ diff --git a/site/content/docs/v1.8/contributions/img-for-tencent/e8c2ab4e5e31d1370c62fad25059a8a8.png b/site/content/docs/v1.8/contributions/img-for-tencent/e8c2ab4e5e31d1370c62fad25059a8a8.png new file mode 100644 index 000000000..c64f47a81 Binary files /dev/null and b/site/content/docs/v1.8/contributions/img-for-tencent/e8c2ab4e5e31d1370c62fad25059a8a8.png differ diff --git a/site/content/docs/v1.8/contributions/img-for-tencent/e932223585c0b19891cc085ad7f438e1.png b/site/content/docs/v1.8/contributions/img-for-tencent/e932223585c0b19891cc085ad7f438e1.png new file mode 100644 index 000000000..188c2db52 Binary files /dev/null and b/site/content/docs/v1.8/contributions/img-for-tencent/e932223585c0b19891cc085ad7f438e1.png differ diff --git a/site/content/docs/v1.8/contributions/img-for-tencent/eb2bbabae48b188748f5278bedf177f1.png b/site/content/docs/v1.8/contributions/img-for-tencent/eb2bbabae48b188748f5278bedf177f1.png new file mode 100644 index 000000000..3eaee3553 Binary files /dev/null and b/site/content/docs/v1.8/contributions/img-for-tencent/eb2bbabae48b188748f5278bedf177f1.png differ diff --git a/site/content/docs/v1.8/contributions/img-for-tencent/effe8a0a7ce3aa8e422db00bfdddc375.png b/site/content/docs/v1.8/contributions/img-for-tencent/effe8a0a7ce3aa8e422db00bfdddc375.png new file mode 100644 index 000000000..28e1bbaad Binary files /dev/null and b/site/content/docs/v1.8/contributions/img-for-tencent/effe8a0a7ce3aa8e422db00bfdddc375.png differ diff --git a/site/content/docs/v1.8/contributions/img-for-tencent/f0fff5228527edc72d6e71a50d5dc966.png b/site/content/docs/v1.8/contributions/img-for-tencent/f0fff5228527edc72d6e71a50d5dc966.png new file mode 100644 index 000000000..b732cf252 Binary files /dev/null and b/site/content/docs/v1.8/contributions/img-for-tencent/f0fff5228527edc72d6e71a50d5dc966.png differ diff --git a/site/content/docs/v1.8/contributions/minio.md b/site/content/docs/v1.8/contributions/minio.md new file mode 100644 index 000000000..3c683473b --- /dev/null +++ b/site/content/docs/v1.8/contributions/minio.md @@ -0,0 +1,296 @@ +--- +title: "Quick start evaluation install with Minio" +layout: docs +--- + +The following example sets up the Velero server and client, then backs up and restores a sample application. + +For simplicity, the example uses Minio, an S3-compatible storage service that runs locally on your cluster. +For additional functionality with this setup, see the section below on how to [expose Minio outside your cluster][1]. + +**NOTE** The example lets you explore basic Velero functionality. Configuring Minio for production is out of scope. + +See [Set up Velero on your platform][3] for how to configure Velero for a production environment. + +If you encounter issues with installing or configuring, see [Debugging Installation Issues](debugging-install.md). + +## Prerequisites + +* Access to a Kubernetes cluster, version 1.7 or later. **Note:** restic support requires Kubernetes version 1.10 or later, or an earlier version with the mount propagation feature enabled. Restic support is not required for this example, but may be of interest later. See [Restic Integration][17]. +* A DNS server on the cluster +* `kubectl` installed +* Sufficient disk space to store backups in Minio. You will need sufficient disk space available to handle any +backups plus at least 1GB additional. Minio will not operate if less than 1GB of free disk space is available. + +## Install the CLI + +### Option 1: MacOS - Homebrew + +On macOS, you can use [Homebrew](https://brew.sh) to install the `velero` client: + +```bash +brew install velero +``` + +### Option 2: GitHub release + +1. Download the [latest official release's](https://github.com/vmware-tanzu/velero/releases) tarball for your client platform. + + _We strongly recommend that you use an [official release](https://github.com/vmware-tanzu/velero/releases) of +Velero. The tarballs for each release contain the `velero` command-line client. The code in the main branch +of the Velero repository is under active development and is not guaranteed to be stable!_ + +1. Extract the tarball: + + ```bash + tar -xvf .tar.gz -C /dir/to/extract/to + ``` + + The directory you extracted is called the "Velero directory" in subsequent steps. + +1. Move the `velero` binary from the Velero directory to somewhere in your PATH. + +## Set up server + +These instructions start the Velero server and a Minio instance that is accessible from within the cluster only. See [Expose Minio outside your cluster](#expose-minio-outside-your-cluster-with-a-service) for information about configuring your cluster for outside access to Minio. Outside access is required to access logs and run `velero describe` commands. + +1. Create a Velero-specific credentials file (`credentials-velero`) in your Velero directory: + + ``` + [default] + aws_access_key_id = minio + aws_secret_access_key = minio123 + ``` + +1. Start the server and the local storage service. In the Velero directory, run: + + ``` + kubectl apply -f examples/minio/00-minio-deployment.yaml + ``` + _Note_: The example Minio yaml provided uses "empty dir". Your node needs to have enough space available to store the + data being backed up plus 1GB of free space. If the node does not have enough space, you can modify the example yaml to + use a Persistent Volume instead of "empty dir" + + ``` + velero install \ + --provider aws \ + --plugins velero/velero-plugin-for-aws:v1.2.1 \ + --bucket velero \ + --secret-file ./credentials-velero \ + --use-volume-snapshots=false \ + --backup-location-config region=minio,s3ForcePathStyle="true",s3Url=http://minio.velero.svc:9000 + ``` + + This example assumes that it is running within a local cluster without a volume provider capable of snapshots, so no `VolumeSnapshotLocation` is created (`--use-volume-snapshots=false`). You may need to update AWS plugin version to one that is [compatible](https://github.com/vmware-tanzu/velero-plugin-for-aws#compatibility) with the version of Velero you are installing. + + Additionally, you can specify `--use-restic` to enable restic support, and `--wait` to wait for the deployment to be ready. + + This example also assumes you have named your Minio bucket "velero". + + +1. Deploy the example nginx application: + + ```bash + kubectl apply -f examples/nginx-app/base.yaml + ``` + +1. Check to see that both the Velero and nginx deployments are successfully created: + + ``` + kubectl get deployments -l component=velero --namespace=velero + kubectl get deployments --namespace=nginx-example + ``` + +## Back up + +1. Create a backup for any object that matches the `app=nginx` label selector: + + ``` + velero backup create nginx-backup --selector app=nginx + ``` + + Alternatively if you want to backup all objects *except* those matching the label `backup=ignore`: + + ``` + velero backup create nginx-backup --selector 'backup notin (ignore)' + ``` + +1. (Optional) Create regularly scheduled backups based on a cron expression using the `app=nginx` label selector: + + ``` + velero schedule create nginx-daily --schedule="0 1 * * *" --selector app=nginx + ``` + + Alternatively, you can use some non-standard shorthand cron expressions: + + ``` + velero schedule create nginx-daily --schedule="@daily" --selector app=nginx + ``` + + See the [cron package's documentation][30] for more usage examples. + +1. Simulate a disaster: + + ``` + kubectl delete namespace nginx-example + ``` + +1. To check that the nginx deployment and service are gone, run: + + ``` + kubectl get deployments --namespace=nginx-example + kubectl get services --namespace=nginx-example + kubectl get namespace/nginx-example + ``` + + You should get no results. + + NOTE: You might need to wait for a few minutes for the namespace to be fully cleaned up. + +## Restore + +1. Run: + + ``` + velero restore create --from-backup nginx-backup + ``` + +1. Run: + + ``` + velero restore get + ``` + + After the restore finishes, the output looks like the following: + + ``` + NAME BACKUP STATUS WARNINGS ERRORS CREATED SELECTOR + nginx-backup-20170727200524 nginx-backup Completed 0 0 2017-07-27 20:05:24 +0000 UTC + ``` + +NOTE: The restore can take a few moments to finish. During this time, the `STATUS` column reads `InProgress`. + +After a successful restore, the `STATUS` column is `Completed`, and `WARNINGS` and `ERRORS` are 0. All objects in the `nginx-example` namespace should be just as they were before you deleted them. + +If there are errors or warnings, you can look at them in detail: + +``` +velero restore describe +``` + +For more information, see [the debugging information][18]. + +## Clean up + +If you want to delete any backups you created, including data in object storage and persistent +volume snapshots, you can run: + +``` +velero backup delete BACKUP_NAME +``` + +This asks the Velero server to delete all backup data associated with `BACKUP_NAME`. You need to do +this for each backup you want to permanently delete. A future version of Velero will allow you to +delete multiple backups by name or label selector. + +Once fully removed, the backup is no longer visible when you run: + +``` +velero backup get BACKUP_NAME +``` + +To completely uninstall Velero, minio, and the nginx example app from your Kubernetes cluster: + +``` +kubectl delete namespace/velero clusterrolebinding/velero +kubectl delete crds -l component=velero +kubectl delete -f examples/nginx-app/base.yaml +``` + +## Expose Minio outside your cluster with a Service + +When you run commands to get logs or describe a backup, the Velero server generates a pre-signed URL to download the requested items. To access these URLs from outside the cluster -- that is, from your Velero client -- you need to make Minio available outside the cluster. You can: + +- Change the Minio Service type from `ClusterIP` to `NodePort`. +- Set up Ingress for your cluster, keeping Minio Service type `ClusterIP`. + +You can also specify a `publicUrl` config field for the pre-signed URL in your backup storage location config. + +### Expose Minio with Service of type NodePort + +The Minio deployment by default specifies a Service of type `ClusterIP`. You can change this to `NodePort` to easily expose a cluster service externally if you can reach the node from your Velero client. + +You must also get the Minio URL, which you can then specify as the value of the `publicUrl` field in your backup storage location config. + +1. In `examples/minio/00-minio-deployment.yaml`, change the value of Service `spec.type` from `ClusterIP` to `NodePort`. + +1. Get the Minio URL: + + - if you're running Minikube: + + ```shell + minikube service minio --namespace=velero --url + ``` + + - in any other environment: + 1. Get the value of an external IP address or DNS name of any node in your cluster. You must be able to reach this address from the Velero client. + 1. Append the value of the NodePort to get a complete URL. You can get this value by running: + + ```shell + kubectl -n velero get svc/minio -o jsonpath='{.spec.ports[0].nodePort}' + ``` + +1. Edit your `BackupStorageLocation` YAML, adding `publicUrl: ` as a field under `spec.config`. You must include the `http://` or `https://` prefix. + +## Accessing logs with an HTTPS endpoint + +If you're using Minio with HTTPS, you may see unintelligible text in the output of `velero describe`, or `velero logs` commands. + +To fix this, you can add a public URL to the `BackupStorageLocation`. + +In a terminal, run the following: + +```shell +kubectl patch -n velero backupstoragelocation default --type merge -p '{"spec":{"config":{"publicUrl":"https://:9000"}}}' +``` + +If your certificate is self-signed, see the [documentation on self-signed certificates][32]. + +## Expose Minio outside your cluster with Kubernetes in Docker (KinD): + +Kubernetes in Docker does not have support for NodePort services (see [this issue](https://github.com/kubernetes-sigs/kind/issues/99)). In this case, you can use a port forward to access the Minio bucket. + +In a terminal, run the following: + +```shell +MINIO_POD=$(kubectl get pods -n velero -l component=minio -o jsonpath='{.items[0].metadata.name}') + +kubectl port-forward $MINIO_POD -n velero 9000:9000 +``` + +Then, in another terminal: + +```shell +kubectl edit backupstoragelocation default -n velero +``` + +Add `publicUrl: http://localhost:9000` under the `spec.config` section. + + +### Work with Ingress + +Configuring Ingress for your cluster is out of scope for the Velero documentation. If you have already set up Ingress, however, it makes sense to continue with it while you run the example Velero configuration with Minio. + +In this case: + +1. Keep the Service type as `ClusterIP`. + +1. Edit your `BackupStorageLocation` YAML, adding `publicUrl: ` as a field under `spec.config`. + +[1]: #expose-minio-with-service-of-type-nodeport +[3]: ../customize-installation.md +[17]: ../restic.md +[18]: ../debugging-restores.md +[26]: https://github.com/vmware-tanzu/velero/releases +[30]: https://godoc.org/github.com/robfig/cron +[32]: ../self-signed-certificates.md diff --git a/site/content/docs/v1.8/contributions/oracle-config.md b/site/content/docs/v1.8/contributions/oracle-config.md new file mode 100644 index 000000000..b348091ca --- /dev/null +++ b/site/content/docs/v1.8/contributions/oracle-config.md @@ -0,0 +1,248 @@ +--- +title: "Use Oracle Cloud as a Backup Storage Provider for Velero" +layout: docs +--- + +## Introduction + +[Velero](https://velero.io/) is a tool used to backup and migrate Kubernetes applications. Here are the steps to use [Oracle Cloud Object Storage](https://docs.cloud.oracle.com/iaas/Content/Object/Concepts/objectstorageoverview.htm) as a destination for Velero backups. + +1. [Download Velero](#download-velero) +2. [Create A Customer Secret Key](#create-a-customer-secret-key) +3. [Create An Oracle Object Storage Bucket](#create-an-oracle-object-storage-bucket) +4. [Install Velero](#install-velero) +5. [Clean Up](#clean-up) +6. [Examples](#examples) +7. [Additional Reading](#additional-reading) + +## Download Velero + +1. Download the [latest release](https://github.com/vmware-tanzu/velero/releases/) of Velero to your development environment. This includes the `velero` CLI utility and example Kubernetes manifest files. For example: + + ``` + wget https://github.com/vmware-tanzu/velero/releases/download/v1.0.0/velero-v1.0.0-linux-amd64.tar.gz + ``` + + **NOTE:** Its strongly recommend that you use an official release of Velero. The tarballs for each release contain the velero command-line client. The code in the main branch of the Velero repository is under active development and is not guaranteed to be stable! + +2. Untar the release in your `/usr/bin` directory: `tar -xzvf .tar.gz` + + You may choose to rename the directory `velero` for the sake of simplicity: `mv velero-v1.0.0-linux-amd64 velero` + +3. Add it to your PATH: `export PATH=/usr/local/bin/velero:$PATH` + +4. Run `velero` to confirm the CLI has been installed correctly. You should see an output like this: + +``` +$ velero +Velero is a tool for managing disaster recovery, specifically for Kubernetes +cluster resources. It provides a simple, configurable, and operationally robust +way to back up your application state and associated data. + +If you're familiar with kubectl, Velero supports a similar model, allowing you to +execute commands such as 'velero get backup' and 'velero create schedule'. The same +operations can also be performed as 'velero backup get' and 'velero schedule create'. + +Usage: + velero [command] +``` + + + +## Create A Customer Secret Key + +1. Oracle Object Storage provides an API to enable interoperability with Amazon S3. To use this Amazon S3 Compatibility API, you need to generate the signing key required to authenticate with Amazon S3. This special signing key is an Access Key/Secret Key pair. Follow these steps to [create a Customer Secret Key](https://docs.cloud.oracle.com/iaas/Content/Identity/Tasks/managingcredentials.htm#To4). Refer to this link for more information about [Working with Customer Secret Keys](https://docs.cloud.oracle.com/iaas/Content/Identity/Tasks/managingcredentials.htm#s3). + +2. Create a Velero credentials file with your Customer Secret Key: + + ``` + $ vi credentials-velero + + [default] + aws_access_key_id=bae031188893d1eb83719648790ac850b76c9441 + aws_secret_access_key=MmY9heKrWiNVCSZQ2Mf5XTJ6Ys93Bw2d2D6NMSTXZlk= + ``` + + + +## Create An Oracle Object Storage Bucket + +Create an Oracle Cloud Object Storage bucket called `velero` in the root compartment of your Oracle Cloud tenancy. Refer to this page for [more information about creating a bucket with Object Storage](https://docs.cloud.oracle.com/iaas/Content/Object/Tasks/managingbuckets.htm#usingconsole). + + + +## Install Velero + +You will need the following information to install Velero into your Kubernetes cluster with Oracle Object Storage as the Backup Storage provider: + +``` +velero install \ + --provider [provider name] \ + --bucket [bucket name] \ + --prefix [tenancy name] \ + --use-volume-snapshots=false \ + --secret-file [secret file location] \ + --backup-location-config region=[region],s3ForcePathStyle="true",s3Url=[storage API endpoint] +``` + +- `--provider` This example uses the S3-compatible API, so use `aws` as the provider. +- `--bucket` The name of the bucket created in Oracle Object Storage - in our case this is named `velero`. +- ` --prefix` The name of your Oracle Cloud tenancy - in our case this is named `oracle-cloudnative`. +- `--use-volume-snapshots=false` Velero does not have a volume snapshot plugin for Oracle Cloud, so creating volume snapshots is disabled. +- `--secret-file` The path to your `credentials-velero` file. +- `--backup-location-config` The path to your Oracle Object Storage bucket. This consists of your `region` which corresponds to your Oracle Cloud region name ([List of Oracle Cloud Regions](https://docs.cloud.oracle.com/iaas/Content/General/Concepts/regions.htm?Highlight=regions)) and the `s3Url`, the S3-compatible API endpoint for Oracle Object Storage based on your region: `https://oracle-cloudnative.compat.objectstorage.[region name].oraclecloud.com` + +For example: + +``` +velero install \ + --provider aws \ + --bucket velero \ + --prefix oracle-cloudnative \ + --use-volume-snapshots=false \ + --secret-file /Users/mboxell/bin/velero/credentials-velero \ + --backup-location-config region=us-phoenix-1,s3ForcePathStyle="true",s3Url=https://oracle-cloudnative.compat.objectstorage.us-phoenix-1.oraclecloud.com +``` + +This will create a `velero` namespace in your cluster along with a number of CRDs, a ClusterRoleBinding, ServiceAccount, Secret, and Deployment for Velero. If your pod fails to successfully provision, you can troubleshoot your installation by running: `kubectl logs [velero pod name]`. + + + +## Clean Up + +To remove Velero from your environment, delete the namespace, ClusterRoleBinding, ServiceAccount, Secret, and Deployment and delete the CRDs, run: + +``` +kubectl delete namespace/velero clusterrolebinding/velero +kubectl delete crds -l component=velero +``` + +This will remove all resources created by `velero install`. + + + +## Examples + +After creating the Velero server in your cluster, try this example: + +### Basic example (without PersistentVolumes) + +1. Start the sample nginx app: `kubectl apply -f examples/nginx-app/base.yaml` + + This will create an `nginx-example` namespace with a `nginx-deployment` deployment, and `my-nginx` service. + + ``` + $ kubectl apply -f examples/nginx-app/base.yaml + namespace/nginx-example created + deployment.apps/nginx-deployment created + service/my-nginx created + ``` + + You can see the created resources by running `kubectl get all` + + ``` + $ kubectl get all + NAME READY STATUS RESTARTS AGE + pod/nginx-deployment-67594d6bf6-4296p 1/1 Running 0 20s + pod/nginx-deployment-67594d6bf6-f9r5s 1/1 Running 0 20s + + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + service/my-nginx LoadBalancer 10.96.69.166 80:31859/TCP 21s + + NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE + deployment.apps/nginx-deployment 2 2 2 2 21s + + NAME DESIRED CURRENT READY AGE + replicaset.apps/nginx-deployment-67594d6bf6 2 2 2 21s + ``` + +2. Create a backup: `velero backup create nginx-backup --include-namespaces nginx-example` + + ``` + $ velero backup create nginx-backup --include-namespaces nginx-example + Backup request "nginx-backup" submitted successfully. + Run `velero backup describe nginx-backup` or `velero backup logs nginx-backup` for more details. + ``` + + At this point you can navigate to appropriate bucket, called `velero`, in the Oracle Cloud Object Storage console to see the resources backed up using Velero. + +3. Simulate a disaster by deleting the `nginx-example` namespace: `kubectl delete namespaces nginx-example` + + ``` + $ kubectl delete namespaces nginx-example + namespace "nginx-example" deleted + ``` + + Wait for the namespace to be deleted. To check that the nginx deployment, service, and namespace are gone, run: + + ``` + kubectl get deployments --namespace=nginx-example + kubectl get services --namespace=nginx-example + kubectl get namespace/nginx-example + ``` + + This should return: `No resources found.` + +4. Restore your lost resources: `velero restore create --from-backup nginx-backup` + + ``` + $ velero restore create --from-backup nginx-backup + Restore request "nginx-backup-20190604102710" submitted successfully. + Run `velero restore describe nginx-backup-20190604102710` or `velero restore logs nginx-backup-20190604102710` for more details. + ``` + + Running `kubectl get namespaces` will show that the `nginx-example` namespace has been restored along with its contents. + +5. Run: `velero restore get` to view the list of restored resources. After the restore finishes, the output looks like the following: + + ``` + $ velero restore get + NAME BACKUP STATUS WARNINGS ERRORS CREATED SELECTOR + nginx-backup-20190604104249 nginx-backup Completed 0 0 2019-06-04 10:42:39 -0700 PDT + ``` + + NOTE: The restore can take a few moments to finish. During this time, the `STATUS` column reads `InProgress`. + + After a successful restore, the `STATUS` column shows `Completed`, and `WARNINGS` and `ERRORS` will show `0`. All objects in the `nginx-example` namespace should be just as they were before you deleted them. + + If there are errors or warnings, for instance if the `STATUS` column displays `FAILED` instead of `InProgress`, you can look at them in detail with `velero restore describe ` + + +6. Clean up the environment with `kubectl delete -f examples/nginx-app/base.yaml` + + ``` + $ kubectl delete -f examples/nginx-app/base.yaml + namespace "nginx-example" deleted + deployment.apps "nginx-deployment" deleted + service "my-nginx" deleted + ``` + + If you want to delete any backups you created, including data in object storage, you can run: `velero backup delete BACKUP_NAME` + + ``` + $ velero backup delete nginx-backup + Are you sure you want to continue (Y/N)? Y + Request to delete backup "nginx-backup" submitted successfully. + The backup will be fully deleted after all associated data (disk snapshots, backup files, restores) are removed. + ``` + + This asks the Velero server to delete all backup data associated with `BACKUP_NAME`. You need to do this for each backup you want to permanently delete. A future version of Velero will allow you to delete multiple backups by name or label selector. + + Once fully removed, the backup is no longer visible when you run: `velero backup get BACKUP_NAME` or more generally `velero backup get`: + + ``` + $ velero backup get nginx-backup + An error occurred: backups.velero.io "nginx-backup" not found + ``` + + ``` + $ velero backup get + NAME STATUS CREATED EXPIRES STORAGE LOCATION SELECTOR + ``` + + + +## Additional Reading + +* [Official Velero Documentation](https://velero.io/docs/v1.8.0/) +* [Oracle Cloud Infrastructure Documentation](https://docs.cloud.oracle.com/) diff --git a/site/content/docs/v1.8/contributions/tencent-config.md b/site/content/docs/v1.8/contributions/tencent-config.md new file mode 100644 index 000000000..50edfce49 --- /dev/null +++ b/site/content/docs/v1.8/contributions/tencent-config.md @@ -0,0 +1,168 @@ +--- +title: "Use Tencent Cloud Object Storage as Velero's storage destination." +layout: docs +--- + + +You can deploy Velero on Tencent [TKE](https://cloud.tencent.com/document/product/457), or an other Kubernetes cluster, and use Tencent Cloud Object Store as a destination for Velero’s backups. + + +## Prerequisites + +- Registered [Tencent Cloud Account](https://cloud.tencent.com/register). +- [Tencent Cloud COS](https://console.cloud.tencent.com/cos) service, referred to as COS, has been launched +- A Kubernetes cluster has been created, cluster version v1.12 or later, and the cluster can use DNS and Internet services normally. If you need to create a TKE cluster, refer to the Tencent [create a cluster](https://cloud.tencent.com/document/product/457/32189) documentation. + +## Create a Tencent Cloud COS bucket + +Create an object bucket for Velero to store backups in the Tencent Cloud COS console. For how to create, please refer to Tencent Cloud COS [Create a bucket](https://cloud.tencent.com/document/product/436/13309) usage instructions. + +Set access to the bucket through the object storage console, the bucket needs to be **read** and **written**, so the account is granted data reading, data writing permissions. For how to configure, see the [permission access settings](https://cloud.tencent.com/document/product/436/13315.E5.8D.95.E4.B8.AA.E6.8E.88.E6.9D.83) Tencent user instructions. + +## Get bucket access credentials + +Velero uses an AWS S3-compatible API to access Tencent Cloud COS storage, which requires authentication using a pair of access key IDs and key-created signatures. + +In the S3 API parameter, the "access_key_id" field is the access key ID and the "secret_access_key" field is the key. + +In the [Tencent Cloud Access Management Console](https://console.cloud.tencent.com/cam/capi), Create and acquire Tencent Cloud Keys "SecretId" and "SecretKey" for COS authorized account. **Where the "SecretId" value corresponds to the value of S3 API parameter "access_key_id" field, the "SecretKey" value corresponds to the value of S3 API parameter "secret_access_key" field**. + +Create the credential profile "credentials-velero" required by Velero in the local directory based on the above correspondence: + +```bash +[default] +aws_access_key_id= +aws_secret_access_key= +``` + +## Install Velero Resources + +You need to install the Velero CLI first, see [Install the CLI](https://velero.io/docs/v1.5/basic-install/#install-the-cli) for how to install. + +Follow the Velero installation command below to create velero and restic workloads and other necessary resource objects. + +```bash +velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.1.0 --bucket \ +--secret-file ./credentials-velero \ +--use-restic \ +--default-volumes-to-restic \ +--backup-location-config \ +region=ap-guangzhou,s3ForcePathStyle="true",s3Url=https://cos.ap-guangzhou.myqcloud.com +``` + +Description of the parameters: + +- `--provider`: Declares the type of plug-in provided by "aws". + +- `--plugins`: Use the AWS S3 compatible API plug-in "velero-plugin-for-aws". + +- `--bucket`: The bucket name created at Tencent Cloud COS. + +- `--secret-file`: Access tencent cloud COS access credential file for the "credentials-velero" credential file created above. + +- `--use-restic`: Back up and restore persistent volume data using the open source free backup tool [restic](https://github.com/restic/restic). However, 'hostPath' volumes are not supported, see the [restic limit](https://velero.io/docs/v1.5/restic/#limitations) for details), an integration that complements Velero's backup capabilities and is recommended to be turned on. + +- `--default-volumes-to-restic`: Enable the use of Restic to back up all Pod volumes, provided that the `--use-restic`parameter needs to be turned on. + +- `--backup-location-config`: Back up the bucket access-related configuration: + + `region`: Tencent cloud COS bucket area, for example, if the created region is Guangzhou, the Region parameter value is "ap-guangzhou". + + `s3ForcePathStyle`: Use the S3 file path format. + + `s3Url`: Tencent Cloud COS-compatible S3 API access address,Note that instead of creating a COS bucket for public network access domain name, you must use a format of "https://cos.`region`.myqcloud.com" URL, for example, if the region is Guangzhou, the parameter value is "https://cos.ap-guangzhou.myqcloud.com.". + +There are other installation parameters that can be viewed using `velero install --help`, such as setting `--use-volume-snapshots-false` to close the storage volume data snapshot backup if you do not want to back up the storage volume data. + +After executing the installation commands above, the installation process looks like this: + +{{< figure src="/docs/main/contributions/img-for-tencent/9015313121ed7987558c88081b052574.png" width="100%">}} + +After the installation command is complete, wait for the velero and restic workloads to be ready to see if the configured storage location is available. + +Executing the 'velero backup-location get' command to view the storage location status and display "Available" indicates that access to Tencent Cloud COS is OK, as shown in the following image: + +{{< figure src="/docs/main/contributions/img-for-tencent/69194157ccd5e377d1e7d914fd8c0336.png" width="100%">}} + +At this point, The installation using Tencent Cloud COS as Velero storage location is complete, If you need more installation information about Velero, You can see the official website [Velero documentation](https://velero.io/docs/) . + +## Velero backup and restore example + +In the cluster, use the helm tool to create a minio test service with a persistent volume, and the minio installation method can be found in the [minio installation](https://github.com/minio/charts), in which case can bound a load balancer for the minio service to access the management page using a public address in the browser. + +{{< figure src="/docs/main/contributions/img-for-tencent/f0fff5228527edc72d6e71a50d5dc966.png" width="100%">}} + +Sign in to the minio web management page and upload some image data for the test, as shown below: + +{{< figure src="/docs/main/contributions/img-for-tencent/e932223585c0b19891cc085ad7f438e1.png" width="100%">}} + +With Velero Backup, you can back up all objects in the cluster directly, or filter objects by type, namespace, and/or label. This example uses the following command to back up all resources under the 'default' namespace. + +``` +velero backup create default-backup --include-namespaces +``` + +Use the `velero backup get` command to see if the backup task is complete, and when the backup task status is "Completed," the backup task is completed without any errors, as shown in the following below: + +{{< figure src="/docs/main/contributions/img-for-tencent/eb2bbabae48b188748f5278bedf177f1.png" width="100%">}} + +At this point delete all of MinIO's resources, including its PVC persistence volume, as shown below:: + +{{< figure src="/docs/main/contributions/img-for-tencent/15ccaacf00640a04ae29ceed4c86195b.png" width="100%">}} + +After deleting the MinIO resource, use your backup to restore the deleted MinIO resource, and temporarily update the backup storage location to read-only mode (this prevents the backup object from being created or deleted in the backup storage location during the restore process):: + +```bash +kubectl patch backupstoragelocation default --namespace velero \ + --type merge \ + --patch '{"spec":{"accessMode":"ReadOnly"}}' + +``` + +Modifying access to Velero's storage location is "ReadOnly," as shown in the following image: + +{{< figure src="/docs/main/contributions/img-for-tencent/e8c2ab4e5e31d1370c62fad25059a8a8.png" width="100%">}} + +Now use the backup "default-backup" that Velero just created to create the restore task: + +```bash +velero restore create --from-backup +``` + +You can also use `velero restore get` to see the status of the restore task, and if the restore status is "Completed," the restore task is complete, as shown in the following image: + +{{< figure src="/docs/main/contributions/img-for-tencent/effe8a0a7ce3aa8e422db00bfdddc375.png" width="100%">}} + +When the restore is complete, you can see that the previously deleted minio-related resources have been restored successfully, as shown in the following image: + +{{< figure src="/docs/main/contributions/img-for-tencent/1d53b0115644d43657c2a5ece805c9b4.png" width="100%">}} + +Log in to minio's management page on your browser and you can see that the previously uploaded picture data is still there, indicating that the persistent volume's data was successfully restored, as shown below: + +{{< figure src="/docs/main/contributions/img-for-tencent/ceaca9ce6bc92bdce987c63d2fe71561.png" width="100%">}} + +When the restore is complete, don't forget to restore the backup storage location to read and write mode so that the next backup task can be used successfully: + +```bash +kubectl patch backupstoragelocation default --namespace velero \ + --type merge \ + --patch '{"spec":{"accessMode":"ReadWrite"}}' +``` + + + +## Uninstall Velero Resources + +To uninstall velero resources in a cluster, you can do so using the following command: + +```bash +kubectl delete namespace/velero clusterrolebinding/velero +kubectl delete crds -l component=velero +``` + + + +## Additional Reading + +- [Official Velero Documentation](https://velero.io/docs/) +- [Tencent Cloud Documentation](https://cloud.tencent.com/document/product) diff --git a/site/content/docs/v1.8/csi.md b/site/content/docs/v1.8/csi.md new file mode 100644 index 000000000..49f4f77e9 --- /dev/null +++ b/site/content/docs/v1.8/csi.md @@ -0,0 +1,77 @@ +--- +title: "Container Storage Interface Snapshot Support in Velero" +layout: docs +--- + +_This feature is under development. Documentation may not be up-to-date and features may not work as expected._ + +Integrating Container Storage Interface (CSI) snapshot support into Velero enables Velero to backup and restore CSI-backed volumes using the [Kubernetes CSI Snapshot Beta APIs](https://kubernetes.io/docs/concepts/storage/volume-snapshots/). + +By supporting CSI snapshot APIs, Velero can support any volume provider that has a CSI driver, without requiring a Velero-specific plugin to be available. + +## Prerequisites + +The following are the prerequisites for using Velero to take Container Storage Interface (CSI) snapshots: + + 1. The cluster is Kubernetes version 1.17 or greater. + 1. The cluster is running a CSI driver capable of support volume snapshots at the [v1beta1 API level](https://kubernetes.io/blog/2019/12/09/kubernetes-1-17-feature-cis-volume-snapshot-beta/). + 1. When restoring CSI volumesnapshots across clusters, the name of the CSI driver in the destination cluster is the same as that on the source cluster to ensure cross cluster portability of CSI volumesnapshots + +## Installing Velero with CSI support + +Ensure that the Velero server is running with the `EnableCSI` feature flag. See [Enabling Features][1] for more information. +Also, the Velero [CSI plugin][2] ([Docker Hub][3]) is necessary to integrate with the CSI volume snapshot APIs. + +Both of these can be added with the `velero install` command. + +```bash +velero install \ +--features=EnableCSI \ +--plugins=,velero/velero-plugin-for-csi:v0.1.0 \ +... +``` + +To include the status of CSI objects associated with a Velero backup in `velero backup describe` output, run `velero client config set features=EnableCSI`. +See [Enabling Features][1] for more information about managing client-side feature flags. + +## Implementation Choices + +This section documents some of the choices made during implementation of the Velero [CSI plugin][2]: + +1. Volumesnapshots created by the plugin will be retained only for the lifetime of the backup even if the `DeletionPolicy` on the volumesnapshotclass is set to `Retain`. To accomplish this, during deletion of the backup the prior to deleting the volumesnapshot, volumesnapshotcontent object will be patched to set its `DeletionPolicy` to `Delete`. Thus deleting volumesnapshot object will result in cascade delete of the volumesnapshotcontent and the snapshot in the storage provider. +1. Volumesnapshotcontent objects created during a velero backup that are dangling, unbound to a volumesnapshot object, will also be discovered, through labels, and deleted on backup deletion. +1. The Velero CSI plugin, to backup CSI backed PVCs, will choose the VolumeSnapshotClass in the cluster that has the same driver name and also has the `velero.io/csi-volumesnapshot-class` label set on it, like +```yaml +velero.io/csi-volumesnapshot-class: "true" +``` + +## Roadmap + +Velero's support level for CSI volume snapshotting will follow upstream Kubernetes support for the feature, and will reach general availability sometime +after volume snapshotting is GA in upstream Kubernetes. Beta support is expected to launch in Velero v1.4. + +## How it Works - Overview + +Velero's CSI support does not rely on the Velero VolumeSnapshotter plugin interface. + +Instead, Velero uses a collection of BackupItemAction plugins that act first against PersistentVolumeClaims. + +When this BackupItemAction sees PersistentVolumeClaims pointing to a PersistentVolume backed by a CSI driver, it will choose the VolumeSnapshotClass with the same driver name that has the `velero.io/csi-volumesnapshot-class` label to create a CSI VolumeSnapshot object with the PersistentVolumeClaim as a source. +This VolumeSnapshot object resides in the same namespace as the PersistentVolumeClaim that was used as a source. + +From there, the CSI external-snapshotter controller will see the VolumeSnapshot and create a VolumeSnapshotContent object, a cluster-scoped resource that will point to the actual, disk-based snapshot in the storage system. +The external-snapshotter plugin will call the CSI driver's snapshot method, and the driver will call the storage system's APIs to generate the snapshot. +Once an ID is generated and the storage system marks the snapshot as usable for restore, the VolumeSnapshotContent object will be updated with a `status.snapshotHandle` and the `status.readyToUse` field will be set. + +Velero will include the generated VolumeSnapshot and VolumeSnapshotContent objects in the backup tarball, as well as upload all VolumeSnapshots and VolumeSnapshotContents objects in a JSON file to the object storage system. +When Velero synchronizes backups into a new cluster, VolumeSnapshotContent objects will be synced into the cluster as well, so that Velero can manage backup expiration appropriately. + +The `DeletionPolicy` on the VolumeSnapshotContent will be the same as the `DeletionPolicy` on the VolumeSnapshotClass that was used to create the VolumeSnapshot. Setting a `DeletionPolicy` of `Retain` on the VolumeSnapshotClass will preserve the volume snapshot in the storage system for the lifetime of the Velero backup and will prevent the deletion of the volume snapshot, in the storage system, in the event of a disaster where the namespace with the VolumeSnapshot object may be lost. + +When the Velero backup expires, the VolumeSnapshot objects will be deleted and the VolumeSnapshotContent objects will be updated to have a `DeletionPolicy` of `Delete`, to free space on the storage system. + +For more details on how each plugin works, see the [CSI plugin repo][2]'s documentation. + +[1]: customize-installation.md#enable-server-side-features +[2]: https://github.com/vmware-tanzu/velero-plugin-for-csi/ +[3]: https://hub.docker.com/repository/docker/velero/velero-plugin-for-csi diff --git a/site/content/docs/v1.8/custom-plugins.md b/site/content/docs/v1.8/custom-plugins.md new file mode 100644 index 000000000..c4dbc03e1 --- /dev/null +++ b/site/content/docs/v1.8/custom-plugins.md @@ -0,0 +1,121 @@ +--- +title: "Plugins" +layout: docs +--- + +Velero has a plugin architecture that allows users to add their own custom functionality to Velero backups & restores without having to modify/recompile the core Velero binary. To add custom functionality, users simply create their own binary containing implementations of Velero's plugin kinds (described below), plus a small amount of boilerplate code to expose the plugin implementations to Velero. This binary is added to a container image that serves as an init container for the Velero server pod and copies the binary into a shared emptyDir volume for the Velero server to access. + +Multiple plugins, of any type, can be implemented in this binary. + +A fully-functional [sample plugin repository][1] is provided to serve as a convenient starting point for plugin authors. + +## Plugin Naming + +A plugin is identified by a prefix + name. + +**Note: Please don't use `velero.io` as the prefix for a plugin not supported by the Velero team.** The prefix should help users identify the entity developing the plugin, so please use a prefix that identify yourself. + +Whenever you define a Backup Storage Location or Volume Snapshot Location, this full name will be the value for the `provider` specification. + +For example: `oracle.io/oracle`. + +``` +apiVersion: velero.io/v1 +kind: BackupStorageLocation +spec: + provider: oracle.io/oracle +``` + +``` +apiVersion: velero.io/v1 +kind: VolumeSnapshotLocation +spec: + provider: oracle.io/oracle +``` + +When naming your plugin, keep in mind that the full name needs to conform to these rules: +- have two parts, prefix + name, separated by '/' +- none of the above parts can be empty +- the prefix is a valid DNS subdomain name +- a plugin with the same prefix + name cannot not already exist + +### Some examples: + +``` +- example.io/azure +- 1.2.3.4/5678 +- example-with-dash.io/azure +``` + +You will need to give your plugin(s) the full name when registering them by calling the appropriate `RegisterX` function: + +## Plugin Kinds + +Velero supports the following kinds of plugins: + +- **Object Store** - persists and retrieves backups, backup logs and restore logs +- **Item Snapshotter** - creates snapshots for Kubernetes objects during backup and restores the object from snapshots during restore. ItemSnapshotters + are typically used with the [Astrolabe](https://github.com/vmware-tanzu/astrolabe) framework. +- **Backup Item Action** - executes arbitrary logic for individual items prior to storing them in a backup file +- **Restore Item Action** - executes arbitrary logic for individual items prior to restoring them into a cluster +- **Delete Item Action** - executes arbitrary logic based on individual items within a backup prior to deleting the backup +## Deprecated plugin kinds +- **Volume Snapshotter** - creates volume snapshots (during backup) and restores volumes from snapshots (during restore) VolumeSnapshotters +are deprecated and will be replaced with ItemSnapshotter/Astrolabe plugins. + +## Plugin Logging + +Velero provides a [logger][2] that can be used by plugins to log structured information to the main Velero server log or +per-backup/restore logs. It also passes a `--log-level` flag to each plugin binary, whose value is the value of the same +flag from the main Velero process. This means that if you turn on debug logging for the Velero server via `--log-level=debug`, +plugins will also emit debug-level logs. See the [sample repository][1] for an example of how to use the logger within your plugin. + +## Plugin Configuration + +Velero uses a ConfigMap-based convention for providing configuration to plugins. If your plugin needs to be configured at runtime, +define a ConfigMap like the following: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + # any name can be used; Velero uses the labels (below) + # to identify it rather than the name + name: my-plugin-config + + # must be in the namespace where the velero deployment + # is running + namespace: velero + + labels: + # this value-less label identifies the ConfigMap as + # config for a plugin (the built-in change storageclass + # restore item action plugin) + velero.io/plugin-config: "" + + # add a label whose key corresponds to the fully-qualified + # plugin name (for example mydomain.io/my-plugin-name), and whose + # value is the plugin type (BackupItemAction, RestoreItemAction, + # ObjectStore, or VolumeSnapshotter) + : + +data: + # add your configuration data here as key-value pairs +``` + +Then, in your plugin's implementation, you can read this ConfigMap to fetch the necessary configuration. See the [restic restore action][3] +for an example of this -- in particular, the `getPluginConfig(...)` function. + +## Feature Flags + +Velero will pass any known features flags as a comma-separated list of strings to the `--features` argument. + +Once parsed into a `[]string`, the features can then be registered using the `NewFeatureFlagSet` function and queried with `features.Enabled()`. + +## Environment Variables + +Velero adds the `LD_LIBRARY_PATH` into the list of environment variables to provide the convenience for plugins that requires C libraries/extensions in the runtime. + +[1]: https://github.com/vmware-tanzu/velero-plugin-example +[2]: https://github.com/vmware-tanzu/velero/blob/v1.8.0/pkg/plugin/logger.go +[3]: https://github.com/vmware-tanzu/velero/blob/v1.8.0/pkg/restore/restic_restore_action.go diff --git a/site/content/docs/v1.8/customize-installation.md b/site/content/docs/v1.8/customize-installation.md new file mode 100644 index 000000000..3b25817df --- /dev/null +++ b/site/content/docs/v1.8/customize-installation.md @@ -0,0 +1,364 @@ +--- +title: "Customize Velero Install" +layout: docs +--- + +## Plugins + +During install, Velero requires that at least one plugin is added (with the `--plugins` flag). Please see the documentation under [Plugins](overview-plugins.md) + +## Install in any namespace + +Velero is installed in the `velero` namespace by default. However, you can install Velero in any namespace. See [run in custom namespace][2] for details. + +## Use non-file-based identity mechanisms + +By default, `velero install` expects a credentials file for your `velero` IAM account to be provided via the `--secret-file` flag. + +If you are using an alternate identity mechanism, such as kube2iam/kiam on AWS, Workload Identity on GKE, etc., that does not require a credentials file, you can specify the `--no-secret` flag instead of `--secret-file`. + +## Enable restic integration + +By default, `velero install` does not install Velero's [restic integration][3]. To enable it, specify the `--use-restic` flag. + +If you've already run `velero install` without the `--use-restic` flag, you can run the same command again, including the `--use-restic` flag, to add the restic integration to your existing install. + +## Default Pod Volume backup to restic + +By default, `velero install` does not enable use of restic to take backups of all pod volumes. An annotation has to be applied on every pod which contains volumes to be backed up by restic. + +To backup all pod volumes using restic without having to apply annotation on the pod, run the `velero install` command with the `--default-volumes-to-restic` flag. + +Using this flag requires restic integration to be enabled with the `--use-restic` flag. Please refer to the [restic integration][3] page for more information. + +## Enable features + +New features in Velero will be released as beta features behind feature flags which are not enabled by default. A full listing of Velero feature flags can be found [here][11]. + +### Enable server side features + +Features on the Velero server can be enabled using the `--features` flag to the `velero install` command. This flag takes as value a comma separated list of feature flags to enable. As an example [CSI snapshotting of PVCs][10] can be enabled using `EnableCSI` feature flag in the `velero install` command as shown below: + +```bash +velero install --features=EnableCSI +``` + +Another example is enabling the support of multiple API group versions, as documented at [- -features=EnableAPIGroupVersions](enable-api-group-versions-feature.md). + +Feature flags, passed to `velero install` will be passed to the Velero deployment and also to the `restic` daemon set, if `--use-restic` flag is used. + +Similarly, features may be disabled by removing the corresponding feature flags from the `--features` flag. + +Enabling and disabling feature flags will require modifying the Velero deployment and also the restic daemonset. This may be done from the CLI by uninstalling and re-installing Velero, or by editing the `deploy/velero` and `daemonset/restic` resources in-cluster. + +```bash +$ kubectl -n velero edit deploy/velero +$ kubectl -n velero edit daemonset/restic +``` + +### Enable client side features + +For some features it may be necessary to use the `--features` flag to the Velero client. This may be done by passing the `--features` on every command run using the Velero CLI or the by setting the features in the velero client config file using the `velero client config set` command as shown below: + +```bash +velero client config set features=EnableCSI +``` + +This stores the config in a file at `$HOME/.config/velero/config.json`. + +All client side feature flags may be disabled using the below command + +```bash +velero client config set features= +``` + +### Colored CLI output + +Velero CLI uses colored output for some commands, such as `velero describe`. If +the environment in which Velero is run doesn't support colored output, the +colored output will be automatically disabled. However, you can manually disable +colors with config file: + +```bash +velero client config set colorized=false +``` + +Note that if you specify `--colorized=true` as a CLI option it will override +the config file setting. + + +## Customize resource requests and limits + +At installation, Velero sets default resource requests and limits for the Velero pod and the restic pod, if you using the [restic integration](/docs/main/restic/). + +{{< table caption="Velero Customize resource requests and limits defaults" >}} +|Setting|Velero pod defaults|restic pod defaults| +|--- |--- |--- | +|CPU request|500m|500m| +|Memory requests|128Mi|512Mi| +|CPU limit|1000m (1 CPU)|1000m (1 CPU)| +|Memory limit|512Mi|1024Mi| +{{< /table >}} + +### Install with custom resource requests and limits + +You can customize these resource requests and limit when you first install using the [velero install][6] CLI command. + +``` +velero install \ + --velero-pod-cpu-request \ + --velero-pod-mem-request \ + --velero-pod-cpu-limit \ + --velero-pod-mem-limit \ + [--use-restic] \ + [--default-volumes-to-restic] \ + [--restic-pod-cpu-request ] \ + [--restic-pod-mem-request ] \ + [--restic-pod-cpu-limit ] \ + [--restic-pod-mem-limit ] +``` + +### Update resource requests and limits after install + +After installation you can adjust the resource requests and limits in the Velero Deployment spec or restic DeamonSet spec, if you are using the restic integration. + +**Velero pod** + +Update the `spec.template.spec.containers.resources.limits` and `spec.template.spec.containers.resources.requests` values in the Velero deployment. + +```bash +kubectl patch deployment velero -n velero --patch \ +'{"spec":{"template":{"spec":{"containers":[{"name": "velero", "resources": {"limits":{"cpu": "1", "memory": "512Mi"}, "requests": {"cpu": "1", "memory": "128Mi"}}}]}}}}' +``` + +**restic pod** + +Update the `spec.template.spec.containers.resources.limits` and `spec.template.spec.containers.resources.requests` values in the restic DeamonSet spec. + +```bash +kubectl patch daemonset restic -n velero --patch \ +'{"spec":{"template":{"spec":{"containers":[{"name": "restic", "resources": {"limits":{"cpu": "1", "memory": "1024Mi"}, "requests": {"cpu": "1", "memory": "512Mi"}}}]}}}}' +``` + +Additionally, you may want to update the the default Velero restic pod operation timeout (default 240 minutes) to allow larger backups more time to complete. You can adjust this timeout by adding the `- --restic-timeout` argument to the Velero Deployment spec. + +**NOTE:** Changes made to this timeout value will revert back to the default value if you re-run the Velero install command. + +1. Open the Velero Deployment spec. + + ``` + kubectl edit deploy velero -n velero + ``` + +1. Add `- --restic-timeout` to `spec.template.spec.containers`. + + ```yaml + spec: + template: + spec: + containers: + - args: + - --restic-timeout=240m + ``` + +## Configure more than one storage location for backups or volume snapshots + +Velero supports any number of backup storage locations and volume snapshot locations. For more details, see [about locations](locations.md). + +However, `velero install` only supports configuring at most one backup storage location and one volume snapshot location. + +To configure additional locations after running `velero install`, use the `velero backup-location create` and/or `velero snapshot-location create` commands along with provider-specific configuration. Use the `--help` flag on each of these commands for more details. + +## Do not configure a backup storage location during install + +If you need to install Velero without a default backup storage location (without specifying `--bucket` or `--provider`), the `--no-default-backup-location` flag is required for confirmation. + +## Install an additional volume snapshot provider + +Velero supports using different providers for volume snapshots than for object storage -- for example, you can use AWS S3 for object storage, and Portworx for block volume snapshots. + +However, `velero install` only supports configuring a single matching provider for both object storage and volume snapshots. + +To use a different volume snapshot provider: + +1. Install the Velero server components by following the instructions for your **object storage** provider + +1. Add your volume snapshot provider's plugin to Velero (look in [your provider][0]'s documentation for the image name): + + ```bash + velero plugin add + ``` + +1. Add a volume snapshot location for your provider, following [your provider][0]'s documentation for configuration: + + ```bash + velero snapshot-location create \ + --provider \ + [--config ] + ``` + +## Generate YAML only + +By default, `velero install` generates and applies a customized set of Kubernetes configuration (YAML) to your cluster. + +To generate the YAML without applying it to your cluster, use the `--dry-run -o yaml` flags. + +This is useful for applying bespoke customizations, integrating with a GitOps workflow, etc. + +## Use a storage provider secured by a self-signed certificate + +If you intend to use Velero with a storage provider that is secured by a self-signed certificate, +you may need to instruct Velero to trust that certificate. See [use Velero with a storage provider secured by a self-signed certificate][9] for details. + +## Additional options + +Run `velero install --help` or see the [Helm chart documentation](https://vmware-tanzu.github.io/helm-charts/) for the full set of installation options. + +## Optional Velero CLI configurations + +### Enabling shell autocompletion + +**Velero CLI** provides autocompletion support for `Bash` and `Zsh`, which can save you a lot of typing. + +Below are the procedures to set up autocompletion for `Bash` (including the difference between `Linux` and `macOS`) and `Zsh`. + +#### Bash on Linux + +The **Velero CLI** completion script for `Bash` can be generated with the command `velero completion bash`. Sourcing the completion script in your shell enables velero autocompletion. + +However, the completion script depends on [**bash-completion**](https://github.com/scop/bash-completion), which means that you have to install this software first (you can test if you have bash-completion already installed by running `type _init_completion`). + +##### Install bash-completion + +`bash-completion` is provided by many package managers (see [here](https://github.com/scop/bash-completion#installation)). You can install it with `apt-get install bash-completion` or `yum install bash-completion`, etc. + +The above commands create `/usr/share/bash-completion/bash_completion`, which is the main script of bash-completion. Depending on your package manager, you have to manually source this file in your `~/.bashrc` file. + +To find out, reload your shell and run `type _init_completion`. If the command succeeds, you're already set, otherwise add the following to your `~/.bashrc` file: + +```shell +source /usr/share/bash-completion/bash_completion +``` + +Reload your shell and verify that bash-completion is correctly installed by typing `type _init_completion`. + +##### Enable Velero CLI autocompletion for Bash on Linux + +You now need to ensure that the **Velero CLI** completion script gets sourced in all your shell sessions. There are two ways in which you can do this: + +- Source the completion script in your `~/.bashrc` file: + + ```shell + echo 'source <(velero completion bash)' >>~/.bashrc + ``` + +- Add the completion script to the `/etc/bash_completion.d` directory: + + ```shell + velero completion bash >/etc/bash_completion.d/velero + ``` + +- If you have an alias for velero, you can extend shell completion to work with that alias: + + ```shell + echo 'alias v=velero' >>~/.bashrc + echo 'complete -F __start_velero v' >>~/.bashrc + ``` + +> `bash-completion` sources all completion scripts in `/etc/bash_completion.d`. + +Both approaches are equivalent. After reloading your shell, velero autocompletion should be working. + +#### Bash on macOS + +The **Velero CLI** completion script for Bash can be generated with `velero completion bash`. Sourcing this script in your shell enables velero completion. + +However, the velero completion script depends on [**bash-completion**](https://github.com/scop/bash-completion) which you thus have to previously install. + + +> There are two versions of bash-completion, v1 and v2. V1 is for Bash 3.2 (which is the default on macOS), and v2 is for Bash 4.1+. The velero completion script **doesn't work** correctly with bash-completion v1 and Bash 3.2. It requires **bash-completion v2** and **Bash 4.1+**. Thus, to be able to correctly use velero completion on macOS, you have to install and use Bash 4.1+ ([*instructions*](https://itnext.io/upgrading-bash-on-macos-7138bd1066ba)). The following instructions assume that you use Bash 4.1+ (that is, any Bash version of 4.1 or newer). + + +##### Install bash-completion + +> As mentioned, these instructions assume you use Bash 4.1+, which means you will install bash-completion v2 (in contrast to Bash 3.2 and bash-completion v1, in which case kubectl completion won't work). + +You can test if you have bash-completion v2 already installed with `type _init_completion`. If not, you can install it with Homebrew: + + ```shell + brew install bash-completion@2 + ``` + +As stated in the output of this command, add the following to your `~/.bashrc` file: + + ```shell + export BASH_COMPLETION_COMPAT_DIR="/usr/local/etc/bash_completion.d" + [[ -r "/usr/local/etc/profile.d/bash_completion.sh" ]] && . "/usr/local/etc/profile.d/bash_completion.sh" + ``` + +Reload your shell and verify that bash-completion v2 is correctly installed with `type _init_completion`. + +##### Enable Velero CLI autocompletion for Bash on macOS + +You now have to ensure that the velero completion script gets sourced in all your shell sessions. There are multiple ways to achieve this: + +- Source the completion script in your `~/.bashrc` file: + + ```shell + echo 'source <(velero completion bash)' >>~/.bashrc + + ``` + +- Add the completion script to the `/usr/local/etc/bash_completion.d` directory: + + ```shell + velero completion bash >/usr/local/etc/bash_completion.d/velero + ``` + +- If you have an alias for velero, you can extend shell completion to work with that alias: + + ```shell + echo 'alias v=velero' >>~/.bashrc + echo 'complete -F __start_velero v' >>~/.bashrc + ``` + +- If you installed velero with Homebrew (as explained [above](#install-with-homebrew-on-macos)), then the velero completion script should already be in `/usr/local/etc/bash_completion.d/velero`. In that case, you don't need to do anything. + +> The Homebrew installation of bash-completion v2 sources all the files in the `BASH_COMPLETION_COMPAT_DIR` directory, that's why the latter two methods work. + +In any case, after reloading your shell, velero completion should be working. + +#### Autocompletion on Zsh + +The velero completion script for Zsh can be generated with the command `velero completion zsh`. Sourcing the completion script in your shell enables velero autocompletion. + +To do so in all your shell sessions, add the following to your `~/.zshrc` file: + + ```shell + source <(velero completion zsh) + ``` + +If you have an alias for kubectl, you can extend shell completion to work with that alias: + + ```shell + echo 'alias v=velero' >>~/.zshrc + echo 'complete -F __start_velero v' >>~/.zshrc + ``` + +After reloading your shell, kubectl autocompletion should be working. + +If you get an error like `complete:13: command not found: compdef`, then add the following to the beginning of your `~/.zshrc` file: + + ```shell + autoload -Uz compinit + compinit + ``` + +[1]: https://github.com/vmware-tanzu/velero/releases/latest +[2]: namespace.md +[3]: restic.md +[4]: on-premises.md +[6]: velero-install.md#usage +[9]: self-signed-certificates.md +[10]: csi.md +[11]: https://github.com/vmware-tanzu/velero/blob/v1.8.0/pkg/apis/velero/v1/constants.go diff --git a/site/content/docs/v1.8/debugging-install.md b/site/content/docs/v1.8/debugging-install.md new file mode 100644 index 000000000..e6a75e525 --- /dev/null +++ b/site/content/docs/v1.8/debugging-install.md @@ -0,0 +1,74 @@ +--- +title: "Debugging Installation Issues" +layout: docs +--- + +## General + +### `invalid configuration: no configuration has been provided` +This typically means that no `kubeconfig` file can be found for the Velero client to use. Velero looks for a kubeconfig in the +following locations: +* the path specified by the `--kubeconfig` flag, if any +* the path specified by the `$KUBECONFIG` environment variable, if any +* `~/.kube/config` + +### Backups or restores stuck in `New` phase +This means that the Velero controllers are not processing the backups/restores, which usually happens because the Velero server is not running. Check the pod description and logs for errors: +``` +kubectl -n velero describe pods +kubectl -n velero logs deployment/velero +``` + + +## AWS + +### `NoCredentialProviders: no valid providers in chain` + +#### Using credentials +This means that the secret containing the AWS IAM user credentials for Velero has not been created/mounted properly +into the Velero server pod. Ensure the following: + +* The `cloud-credentials` secret exists in the Velero server's namespace +* The `cloud-credentials` secret has a single key, `cloud`, whose value is the contents of the `credentials-velero` file +* The `credentials-velero` file is formatted properly and has the correct values: + + ``` + [default] + aws_access_key_id= + aws_secret_access_key= + ``` + +* The `cloud-credentials` secret is defined as a volume for the Velero deployment +* The `cloud-credentials` secret is being mounted into the Velero server pod at `/credentials` + +#### Using kube2iam +This means that Velero can't read the content of the S3 bucket. Ensure the following: + +* A Trust Policy document exists that allows the role used by kube2iam to assume Velero's role, as stated in the AWS config documentation. +* The new Velero role has all the permissions listed in the documentation regarding S3. + + +## Azure + +### `Failed to refresh the Token` or `adal: Refresh request failed` +This means that the secrets containing the Azure service principal credentials for Velero has not been created/mounted +properly into the Velero server pod. Ensure the following: + +* The `cloud-credentials` secret exists in the Velero server's namespace +* The `cloud-credentials` secret has all of the expected keys and each one has the correct value (see [setup instructions][0]) +* The `cloud-credentials` secret is defined as a volume for the Velero deployment +* The `cloud-credentials` secret is being mounted into the Velero server pod at `/credentials` + + +## GCE/GKE + +### `open credentials/cloud: no such file or directory` +This means that the secret containing the GCE service account credentials for Velero has not been created/mounted properly +into the Velero server pod. Ensure the following: + +* The `cloud-credentials` secret exists in the Velero server's namespace +* The `cloud-credentials` secret has a single key, `cloud`, whose value is the contents of the `credentials-velero` file +* The `cloud-credentials` secret is defined as a volume for the Velero deployment +* The `cloud-credentials` secret is being mounted into the Velero server pod at `/credentials` + +[0]: azure-config.md#create-service-principal diff --git a/site/content/docs/v1.8/debugging-restores.md b/site/content/docs/v1.8/debugging-restores.md new file mode 100644 index 000000000..3320349cd --- /dev/null +++ b/site/content/docs/v1.8/debugging-restores.md @@ -0,0 +1,105 @@ +--- +title: "Debugging Restores" +layout: docs +--- + +## Example + +When Velero finishes a Restore, its status changes to "Completed" regardless of whether or not there are issues during the process. The number of warnings and errors are indicated in the output columns from `velero restore get`: + +``` +NAME BACKUP STATUS WARNINGS ERRORS CREATED SELECTOR +backup-test-20170726180512 backup-test Completed 155 76 2017-07-26 11:41:14 -0400 EDT +backup-test-20170726180513 backup-test Completed 121 14 2017-07-26 11:48:24 -0400 EDT +backup-test-2-20170726180514 backup-test-2 Completed 0 0 2017-07-26 13:31:21 -0400 EDT +backup-test-2-20170726180515 backup-test-2 Completed 0 1 2017-07-26 13:32:59 -0400 EDT +``` + +To delve into the warnings and errors into more detail, you can use `velero restore describe`: + +```bash +velero restore describe backup-test-20170726180512 +``` + +The output looks like this: + +``` +Name: backup-test-20170726180512 +Namespace: velero +Labels: +Annotations: + +Backup: backup-test + +Namespaces: + Included: * + Excluded: + +Resources: + Included: serviceaccounts + Excluded: nodes, events, events.events.k8s.io + Cluster-scoped: auto + +Namespace mappings: + +Label selector: + +Restore PVs: auto + +Preserve Service NodePorts: auto + +Phase: Completed + +Validation errors: + +Warnings: + Velero: + Cluster: + Namespaces: + velero: serviceaccounts "velero" already exists + serviceaccounts "default" already exists + kube-public: serviceaccounts "default" already exists + kube-system: serviceaccounts "attachdetach-controller" already exists + serviceaccounts "certificate-controller" already exists + serviceaccounts "cronjob-controller" already exists + serviceaccounts "daemon-set-controller" already exists + serviceaccounts "default" already exists + serviceaccounts "deployment-controller" already exists + serviceaccounts "disruption-controller" already exists + serviceaccounts "endpoint-controller" already exists + serviceaccounts "generic-garbage-collector" already exists + serviceaccounts "horizontal-pod-autoscaler" already exists + serviceaccounts "job-controller" already exists + serviceaccounts "kube-dns" already exists + serviceaccounts "namespace-controller" already exists + serviceaccounts "node-controller" already exists + serviceaccounts "persistent-volume-binder" already exists + serviceaccounts "pod-garbage-collector" already exists + serviceaccounts "replicaset-controller" already exists + serviceaccounts "replication-controller" already exists + serviceaccounts "resourcequota-controller" already exists + serviceaccounts "service-account-controller" already exists + serviceaccounts "service-controller" already exists + serviceaccounts "statefulset-controller" already exists + serviceaccounts "ttl-controller" already exists + default: serviceaccounts "default" already exists + +Errors: + Velero: + Cluster: + Namespaces: +``` + +## Structure + +Errors appear for incomplete or partial restores. Warnings appear for non-blocking issues, for example, the +restore looks "normal" and all resources referenced in the backup exist in some form, although some +of them may have been pre-existing. + +Both errors and warnings are structured in the same way: + +* `Velero`: A list of system-related issues encountered by the Velero server. For example, Velero couldn't read a directory. + +* `Cluster`: A list of issues related to the restore of cluster-scoped resources. + +* `Namespaces`: A map of namespaces to the list of issues related to the restore of their respective resources. diff --git a/site/content/docs/v1.8/development.md b/site/content/docs/v1.8/development.md new file mode 100644 index 000000000..82ace894d --- /dev/null +++ b/site/content/docs/v1.8/development.md @@ -0,0 +1,56 @@ +--- +title: "Development " +layout: docs +--- + +## Update generated files + +Run `make update` to regenerate files if you make the following changes: + +* Add/edit/remove command line flags and/or their help text +* Add/edit/remove commands or subcommands +* Add new API types +* Add/edit/remove plugin protobuf message or service definitions + +The following files are automatically generated from the source code: + +* The clientset +* Listers +* Shared informers +* Documentation +* Protobuf/gRPC types + +You can run `make verify` to ensure that all generated files (clientset, listers, shared informers, docs) are up to date. + +## Linting + +You can run `make lint` which executes golangci-lint inside the build image, or `make local-lint` which executes outside of the build image. +Both `make lint` and `make local-lint` will only run the linter against changes. + +Use `lint-all` to run the linter against the entire code base. + +The default linters are defined in the `Makefile` via the `LINTERS` variable. + +You can also override the default list of linters by running the command + +`$ make lint LINTERS=gosec` + +## Test + +To run unit tests, use `make test`. + +## Vendor dependencies + +If you need to add or update the vendored dependencies, see [Vendoring dependencies][11]. + +## Using the main branch + +If you are developing or using the main branch, note that you may need to update the Velero CRDs to get new changes as other development work is completed. + +```bash +velero install --crds-only --dry-run -o yaml | kubectl apply -f - +``` + +**NOTE:** You could change the default CRD API version (v1beta1 _or_ v1) if Velero CLI can't discover the Kubernetes preferred CRD API version. The Kubernetes version < 1.16 preferred CRD API version is v1beta1; the Kubernetes version >= 1.16 preferred CRD API version is v1. + +[11]: vendoring-dependencies.md diff --git a/site/content/docs/v1.8/disaster-case.md b/site/content/docs/v1.8/disaster-case.md new file mode 100644 index 000000000..6b73e6716 --- /dev/null +++ b/site/content/docs/v1.8/disaster-case.md @@ -0,0 +1,44 @@ +--- +title: "Disaster recovery" +layout: docs +--- + +*Using Schedules and Read-Only Backup Storage Locations* + +If you periodically back up your cluster's resources, you are able to return to a previous state in case of some unexpected mishap, such as a service outage. Doing so with Velero looks like the following: + +1. After you first run the Velero server on your cluster, set up a daily backup (replacing `` in the command as desired): + + ``` + velero schedule create --schedule "0 7 * * *" + ``` + + This creates a Backup object with the name `-`. The default backup retention period, expressed as TTL (time to live), is 30 days (720 hours); you can use the `--ttl ` flag to change this as necessary. See [how velero works][1] for more information about backup expiry. + +1. A disaster happens and you need to recreate your resources. + +1. Update your backup storage location to read-only mode (this prevents backup objects from being created or deleted in the backup storage location during the restore process): + + ```bash + kubectl patch backupstoragelocation \ + --namespace velero \ + --type merge \ + --patch '{"spec":{"accessMode":"ReadOnly"}}' + ``` + +1. Create a restore with your most recent Velero Backup: + + ``` + velero restore create --from-backup - + ``` + +1. When ready, revert your backup storage location to read-write mode: + + ```bash + kubectl patch backupstoragelocation \ + --namespace velero \ + --type merge \ + --patch '{"spec":{"accessMode":"ReadWrite"}}' + ``` + +[1]: how-velero-works.md#set-a-backup-to-expire diff --git a/site/content/docs/v1.8/enable-api-group-versions-feature.md b/site/content/docs/v1.8/enable-api-group-versions-feature.md new file mode 100644 index 000000000..5c40b3e71 --- /dev/null +++ b/site/content/docs/v1.8/enable-api-group-versions-feature.md @@ -0,0 +1,115 @@ +--- +title: "Enable API Group Versions Feature" +layout: docs +--- + +## Background + +Velero serves to both restore and migrate Kubernetes applications. Typically, backup and restore does not involve upgrading Kubernetes API group versions. However, when migrating from a source cluster to a destination cluster, it is not unusual to see the API group versions differing between clusters. + +**NOTE:** Kubernetes applications are made up of various resources. Common resources are pods, jobs, and deployments. Custom resources are created via custom resource definitions (CRDs). Every resource, whether custom or not, is part of a group, and each group has a version called the API group version. + +Kubernetes by default allows changing API group versions between clusters as long as the upgrade is a single version, for example, v1 -> v2beta1. Jumping multiple versions, for example, v1 -> v3, is not supported out of the box. This is where the Velero Enable API Group Version feature can help you during an upgrade. + +Currently, the Enable API Group Version feature is in beta and can be enabled by installing Velero with a [feature flag](customize-installation.md/#enable-server-side-features), `--features=EnableAPIGroupVersions`. + +For the most up-to-date information on Kubernetes API version compatibility, you should always review the [Kubernetes release notes](https://github.com/kubernetes/kubernetes/tree/master/CHANGELOG) for the source and destination cluster version to before starting an upgrade, migration, or restore. If there is a difference between Kubernetes API versions, use the Enable API Group Version feature to help mitigate compatibility issues. + +## How the Enable API Group Versions Feature Works + +When the Enable API Group Versions feature is enabled on the source cluster, Velero will not only back up Kubernetes preferred API group versions, but it will also back up all supported versions on the cluster. As an example, consider the resource `horizontalpodautoscalers` which falls under the `autoscaling` group. Without the feature flag enabled, only the preferred API group version for autoscaling, `v1` will be backed up. With the feature enabled, the remaining supported versions, `v2beta1` and `v2beta2` will also be backed up. Once the versions are stored in the backup tarball file, they will be available to be restored on the destination cluster. + +When the Enable API Group Versions feature is enabled on the destination cluster, Velero restore will choose the version to restore based on an API group version priority order. + +The version priorities are listed from highest to lowest priority below: + +- Priority 1: destination cluster preferred version +- Priority 2: source cluster preferred version +- Priority 3: non-preferred common supported version with the highest [Kubernetes version priority](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definition-versioning/#version-priority) + +The highest priority (Priority 1) will be the destination cluster's preferred API group version. If the destination preferred version is found in the backup tarball, it will be the API group version chosen for restoration for that resource. However, if the destination preferred version is not found in the backup tarball, the next version in the list will be selected: the source cluster preferred version (Priority 2). + +If the source cluster preferred version is found to be supported by the destination cluster, it will be chosen as the API group version to restore. However, if the source preferred version is not supported by the destination cluster, then the next version in the list will be considered: a non-preferred common supported version (Priority 3). + +In the case that there are more than one non-preferred common supported version, which version will be chosen? The answer requires understanding the [Kubernetes version priority order](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definition-versioning/#version-priority). Kubernetes prioritizes group versions by making the latest, most stable version the highest priority. The highest priority version is the Kubernetes preferred version. Here is a sorted version list example from the Kubernetes.io documentation: + +- v10 +- v2 +- v1 +- v11beta2 +- v10beta3 +- v3beta1 +- v12alpha1 +- v11alpha2 +- foo1 +- foo10 + +Of the non-preferred common versions, the version that has the highest Kubernetes version priority will be chosen. See the example for Priority 3 below. + +To better understand which API group version will be chosen, the following provides some concrete examples. The examples use the term "target cluster" which is synonymous to "destination cluster". + +![Priority 1 Case A example](/docs/main/img/gv_priority1-caseA.png) + +![Priority 1 Case B example](/docs/main/img/gv_priority1-caseB.png) + +![Priority 2 Case C example](/docs/main/img/gv_priority2-caseC.png) + +![Priority 3 Case D example](/docs/main/img/gv_priority3-caseD.png) + +## Procedure for Using the Enable API Group Versions Feature + +1. [Install Velero](basic-install.md) on source cluster with the [feature flag enabled](customize-installation.md/#enable-server-side-features). The flag is `--features=EnableAPIGroupVersions`. For the enable API group versions feature to work, the feature flag needs to be used for Velero installations on both the source and destination clusters. +2. Back up and restore following the [migration case instructions](migration-case.md). Note that "Cluster 1" in the instructions refers to the source cluster, and "Cluster 2" refers to the destination cluster. + +## Advanced Procedure for Customizing the Version Prioritization + +Optionally, users can create a config map to override the default API group prioritization for some or all of the resources being migrated. For each resource that is specified by the user, Velero will search for the version in both the backup tarball and the destination cluster. If there is a match, the user-specified API group version will be restored. If the backup tarball and the destination cluster does not have or support any of the user-specified versions, then the default version prioritization will be used. + +Here are the steps for creating a config map that allows users to override the default version prioritization. These steps must happen on the destination cluster before a Velero restore is initiated. + +1. Create a file called `restoreResourcesVersionPriority`. The file name will become a key in the `data` field of the config map. + - In the file, write a line for each resource group you'd like to override. Make sure each line follows the format `.=,` + - Note that the resource group and versions are separated by a single equal (=) sign. Each version is listed in order of user's priority separated by commas. + - Here is an example of the contents of a config map file: + + ```cm + rockbands.music.example.io=v2beta1,v2beta2 + orchestras.music.example.io=v2,v3alpha1 + subscriptions.operators.coreos.com=v2,v1 + ``` + +2. Apply config map with + + ```bash + kubectl create configmap enableapigroupversions --from-file=/restoreResourcesVersionPriority -n velero + ``` + +3. See the config map with + + ```bash + kubectl describe configmap enableapigroupversions -n velero + ``` + + The config map should look something like + + ```bash + Name: enableapigroupversions + Namespace: velero + Labels: + Annotations: + + Data + ==== + restoreResourcesVersionPriority: + ---- + rockbands.music.example.io=v2beta1,v2beta2 + orchestras.music.example.io=v2,v3alpha1 + subscriptions.operators.coreos.com=v2,v1 + Events: + ``` + +## Troubleshooting + +1. Refer to the [troubleshooting section](troubleshooting.md) of the docs as the techniques generally apply here as well. +2. The [debug logs](troubleshooting.md/#getting-velero-debug-logs) will contain information on which version was chosen to restore. +3. If no API group version could be found that both exists in the backup tarball file and is supported by the destination cluster, then the following error will be recorded (no need to activate debug level logging): `"error restoring rockbands.music.example.io/rockstars/beatles: the server could not find the requested resource"`. diff --git a/site/content/docs/v1.8/examples.md b/site/content/docs/v1.8/examples.md new file mode 100644 index 000000000..58a35fd5d --- /dev/null +++ b/site/content/docs/v1.8/examples.md @@ -0,0 +1,70 @@ +--- +title: "Examples" +layout: docs +--- + +After you set up the Velero server, you can clone the examples used in the following sections by running the following: +``` +git clone https://github.com/vmware-tanzu/velero.git +cd velero +``` + +## Basic example (without PersistentVolumes) + +1. Start the sample nginx app: + + ```bash + kubectl apply -f examples/nginx-app/base.yaml + ``` + +1. Create a backup: + + ```bash + velero backup create nginx-backup --include-namespaces nginx-example + ``` + +1. Simulate a disaster: + + ```bash + kubectl delete namespaces nginx-example + ``` + + Wait for the namespace to be deleted. + +1. Restore your lost resources: + + ```bash + velero restore create --from-backup nginx-backup + ``` + +## Snapshot example (with PersistentVolumes) + +> NOTE: For Azure, you must run Kubernetes version 1.7.2 or later to support PV snapshotting of managed disks. + +1. Start the sample nginx app: + + ```bash + kubectl apply -f examples/nginx-app/with-pv.yaml + ``` + +1. Create a backup with PV snapshotting: + + ```bash + velero backup create nginx-backup --include-namespaces nginx-example + ``` + +1. Simulate a disaster: + + ```bash + kubectl delete namespaces nginx-example + ``` + + Because the default [reclaim policy][1] for dynamically-provisioned PVs is "Delete", these commands should trigger your cloud provider to delete the disk that backs the PV. Deletion is asynchronous, so this may take some time. **Before continuing to the next step, check your cloud provider to confirm that the disk no longer exists.** + +1. Restore your lost resources: + + ```bash + velero restore create --from-backup nginx-backup + ``` + +[1]: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming diff --git a/site/content/docs/v1.8/how-velero-works.md b/site/content/docs/v1.8/how-velero-works.md new file mode 100644 index 000000000..250ab2dab --- /dev/null +++ b/site/content/docs/v1.8/how-velero-works.md @@ -0,0 +1,87 @@ +--- +title: "How Velero Works" +layout: docs +--- + +Each Velero operation -- on-demand backup, scheduled backup, restore -- is a custom resource, defined with a Kubernetes [Custom Resource Definition (CRD)][20] and stored in [etcd][22]. Velero also includes controllers that process the custom resources to perform backups, restores, and all related operations. + +You can back up or restore all objects in your cluster, or you can filter objects by type, namespace, and/or label. + +Velero is ideal for the disaster recovery use case, as well as for snapshotting your application state, prior to performing system operations on your cluster, like upgrades. + +## On-demand backups + +The **backup** operation: + +1. Uploads a tarball of copied Kubernetes objects into cloud object storage. + +1. Calls the cloud provider API to make disk snapshots of persistent volumes, if specified. + +You can optionally specify backup hooks to be executed during the backup. For example, you might +need to tell a database to flush its in-memory buffers to disk before taking a snapshot. [More about backup hooks][10]. + +Note that cluster backups are not strictly atomic. If Kubernetes objects are being created or edited at the time of backup, they might not be included in the backup. The odds of capturing inconsistent information are low, but it is possible. + +## Scheduled backups + +The **schedule** operation allows you to back up your data at recurring intervals. You can create a scheduled backup at any time, and the first backup is then performed at the schedule's specified interval. These intervals are specified by a Cron expression. + +Velero saves backups created from a schedule with the name `-`, where `` is formatted as *YYYYMMDDhhmmss*. For more information see the [Backup Reference documentation](backup-reference.md). + +## Restores + +The **restore** operation allows you to restore all of the objects and persistent volumes from a previously created backup. You can also restore only a filtered subset of objects and persistent volumes. Velero supports multiple namespace remapping--for example, in a single restore, objects in namespace "abc" can be recreated under namespace "def", and the objects in namespace "123" under "456". + +The default name of a restore is `-`, where `` is formatted as *YYYYMMDDhhmmss*. You can also specify a custom name. A restored object also includes a label with key `velero.io/restore-name` and value ``. + +By default, backup storage locations are created in read-write mode. However, during a restore, you can configure a backup storage location to be in read-only mode, which disables backup creation and deletion for the storage location. This is useful to ensure that no backups are inadvertently created or deleted during a restore scenario. + +You can optionally specify restore hooks to be executed during a restore or after resources are restored. For example, you might need to perform a custom database restore operation before the database application containers start. [More about restore hooks][11]. + +## Backup workflow + +When you run `velero backup create test-backup`: + +1. The Velero client makes a call to the Kubernetes API server to create a `Backup` object. + +1. The `BackupController` notices the new `Backup` object and performs validation. + +1. The `BackupController` begins the backup process. It collects the data to back up by querying the API server for resources. + +1. The `BackupController` makes a call to the object storage service -- for example, AWS S3 -- to upload the backup file. + +By default, `velero backup create` makes disk snapshots of any persistent volumes. You can adjust the snapshots by specifying additional flags. Run `velero backup create --help` to see available flags. Snapshots can be disabled with the option `--snapshot-volumes=false`. + +![19] + +## Backed-up API versions + +Velero backs up resources using the Kubernetes API server's *preferred version* for each group/resource. When restoring a resource, this same API group/version must exist in the target cluster in order for the restore to be successful. + +For example, if the cluster being backed up has a `gizmos` resource in the `things` API group, with group/versions `things/v1alpha1`, `things/v1beta1`, and `things/v1`, and the server's preferred group/version is `things/v1`, then all `gizmos` will be backed up from the `things/v1` API endpoint. When backups from this cluster are restored, the target cluster **must** have the `things/v1` endpoint in order for `gizmos` to be restored. Note that `things/v1` **does not** need to be the preferred version in the target cluster; it just needs to exist. + +## Set a backup to expire + +When you create a backup, you can specify a TTL (time to live) by adding the flag `--ttl `. If Velero sees that an existing backup resource is expired, it removes: + +* The backup resource +* The backup file from cloud object storage +* All PersistentVolume snapshots +* All associated Restores + +The TTL flag allows the user to specify the backup retention period with the value specified in hours, minutes and seconds in the form `--ttl 24h0m0s`. If not specified, a default TTL value of 30 days will be applied. + +## Object storage sync + +Velero treats object storage as the source of truth. It continuously checks to see that the correct backup resources are always present. If there is a properly formatted backup file in the storage bucket, but no corresponding backup resource in the Kubernetes API, Velero synchronizes the information from object storage to Kubernetes. + +This allows restore functionality to work in a cluster migration scenario, where the original backup objects do not exist in the new cluster. + +Likewise, if a backup object exists in Kubernetes but not in object storage, it will be deleted from Kubernetes since the backup tarball no longer exists. + +[10]: backup-hooks.md +[11]: restore-hooks.md +[19]: /docs/main/img/backup-process.png +[20]: https://kubernetes.io/docs/concepts/api-extension/custom-resources/#customresourcedefinitions +[21]: https://kubernetes.io/docs/concepts/api-extension/custom-resources/#custom-controllers +[22]: https://github.com/coreos/etcd diff --git a/site/content/docs/v1.8/image-tagging.md b/site/content/docs/v1.8/image-tagging.md new file mode 100644 index 000000000..64e9310b5 --- /dev/null +++ b/site/content/docs/v1.8/image-tagging.md @@ -0,0 +1,24 @@ +--- +title: "Image tagging policy" +layout: docs +--- + +This document describes Velero's image tagging policy. + +## Released versions + +`velero/velero:` + +Velero follows the [Semantic Versioning](http://semver.org/) standard for releases. Each tag in the `github.com/vmware-tanzu/velero` repository has a matching image, `velero/velero:v1.0.0`. + +### Latest + +`velero/velero:latest` + +The `latest` tag follows the most recently released version of Velero. + +## Development + +`velero/velero:main` + +The `main` tag follows the latest commit to land on the `main` branch. diff --git a/site/content/docs/v1.8/img/README.md b/site/content/docs/v1.8/img/README.md new file mode 100644 index 000000000..85c071c63 --- /dev/null +++ b/site/content/docs/v1.8/img/README.md @@ -0,0 +1 @@ +Some of these diagrams (for instance backup-process.png), have been created on [draw.io](https://www.draw.io), using the "Include a copy of my diagram" option. If you want to make changes to these diagrams, try importing them into draw.io, and you should have access to the original shapes/text that went into the originals. diff --git a/site/content/docs/v1.8/img/backup-process.png b/site/content/docs/v1.8/img/backup-process.png new file mode 100644 index 000000000..7d4f10d56 Binary files /dev/null and b/site/content/docs/v1.8/img/backup-process.png differ diff --git a/site/content/docs/v1.8/img/gv_priority1-caseA.png b/site/content/docs/v1.8/img/gv_priority1-caseA.png new file mode 100644 index 000000000..a1940453f Binary files /dev/null and b/site/content/docs/v1.8/img/gv_priority1-caseA.png differ diff --git a/site/content/docs/v1.8/img/gv_priority1-caseB.png b/site/content/docs/v1.8/img/gv_priority1-caseB.png new file mode 100644 index 000000000..6623f7f81 Binary files /dev/null and b/site/content/docs/v1.8/img/gv_priority1-caseB.png differ diff --git a/site/content/docs/v1.8/img/gv_priority2-caseC.png b/site/content/docs/v1.8/img/gv_priority2-caseC.png new file mode 100644 index 000000000..980c0d5f2 Binary files /dev/null and b/site/content/docs/v1.8/img/gv_priority2-caseC.png differ diff --git a/site/content/docs/v1.8/img/gv_priority3-caseD.png b/site/content/docs/v1.8/img/gv_priority3-caseD.png new file mode 100644 index 000000000..11c9a11ce Binary files /dev/null and b/site/content/docs/v1.8/img/gv_priority3-caseD.png differ diff --git a/site/content/docs/v1.8/img/velero.png b/site/content/docs/v1.8/img/velero.png new file mode 100644 index 000000000..a1937a1dc Binary files /dev/null and b/site/content/docs/v1.8/img/velero.png differ diff --git a/site/content/docs/v1.8/locations.md b/site/content/docs/v1.8/locations.md new file mode 100644 index 000000000..2f54077dd --- /dev/null +++ b/site/content/docs/v1.8/locations.md @@ -0,0 +1,260 @@ +--- +title: "Backup Storage Locations and Volume Snapshot Locations" +layout: docs +--- + +## Overview + +Velero has two custom resources, `BackupStorageLocation` and `VolumeSnapshotLocation`, that are used to configure where Velero backups and their associated persistent volume snapshots are stored. + +A `BackupStorageLocation` is defined as a bucket or a prefix within a bucket under which all Velero data is stored and a set of additional provider-specific fields (AWS region, Azure storage account, etc.). Velero assumes it has control over the location you provide so you should use a dedicated bucket or prefix. If you provide a prefix, then the rest of the bucket is safe to use for multiple purposes. The [API documentation][1] captures the configurable parameters for each in-tree provider. + +A `VolumeSnapshotLocation` is defined entirely by provider-specific fields (AWS region, Azure resource group, Portworx snapshot type, etc.) The [API documentation][2] captures the configurable parameters for each in-tree provider. + +The user can pre-configure one or more possible `BackupStorageLocations` and one or more `VolumeSnapshotLocations`, and can select *at backup creation time* the location in which the backup and associated snapshots should be stored. + +This configuration design enables a number of different use cases, including: + +- Take snapshots of more than one kind of persistent volume in a single Velero backup. For example, in a cluster with both EBS volumes and Portworx volumes +- Have some Velero backups go to a bucket in an eastern USA region, and others go to a bucket in a western USA region, or to a different storage provider +- For volume providers that support it, like Portworx, you can have some snapshots stored locally on the cluster and have others stored in the cloud + +## Limitations / Caveats + +- Velero supports multiple credentials for `BackupStorageLocations`, allowing you to specify the credentials to use with any `BackupStorageLocation`. + However, use of this feature requires support within the plugin for the object storage provider you wish to use. + All [plugins maintained by the Velero team][5] support this feature. + If you are using a plugin from another provider, please check their documentation to determine if this feature is supported. + +- Velero only supports a single set of credentials for `VolumeSnapshotLocations`. + Velero will always use the credentials provided at install time (stored in the `cloud-credentials` secret) for volume snapshots. + +- Volume snapshots are still limited by where your provider allows you to create snapshots. For example, AWS and Azure do not allow you to create a volume snapshot in a different region than where the volume is. If you try to take a Velero backup using a volume snapshot location with a different region than where your cluster's volumes are, the backup will fail. + +- Each Velero backup has one `BackupStorageLocation`, and one `VolumeSnapshotLocation` per volume provider. It is not possible (yet) to send a single Velero backup to multiple backup storage locations simultaneously, or a single volume snapshot to multiple locations simultaneously. However, you can always set up multiple scheduled backups that differ only in the storage locations used if redundancy of backups across locations is important. + +- Cross-provider snapshots are not supported. If you have a cluster with more than one type of volume, like EBS and Portworx, but you only have a `VolumeSnapshotLocation` configured for EBS, then Velero will **only** snapshot the EBS volumes. + +- Restic data is stored under a prefix/subdirectory of the main Velero bucket, and will go into the bucket corresponding to the `BackupStorageLocation` selected by the user at backup creation time. + +- Velero's backups are split into 2 pieces - the metadata stored in object storage, and snapshots/backups of the persistent volume data. Right now, Velero *itself* does not encrypt either of them, instead it relies on the native mechanisms in the object and snapshot systems. A special case is restic, which backs up the persistent volume data at the filesystem level and send it to Velero's object storage. + +- Velero's compression for object metadata is limited, using Golang's tar implementation. In most instances, Kubernetes objects are limited to 1.5MB in size, but many don't approach that, meaning that compression may not be necessary. Note that restic has not yet implemented compression, but does have de-deduplication capabilities. + +## Examples + +Let's look at some examples of how you can use this configuration mechanism to address some common use cases: + +### Take snapshots of more than one kind of persistent volume in a single Velero backup + +During server configuration: + +```shell +velero snapshot-location create ebs-us-east-1 \ + --provider aws \ + --config region=us-east-1 + +velero snapshot-location create portworx-cloud \ + --provider portworx \ + --config type=cloud +``` + +During backup creation: + +```shell +velero backup create full-cluster-backup \ + --volume-snapshot-locations ebs-us-east-1,portworx-cloud +``` + +Alternately, since in this example there's only one possible volume snapshot location configured for each of our two providers (`ebs-us-east-1` for `aws`, and `portworx-cloud` for `portworx`), Velero doesn't require them to be explicitly specified when creating the backup: + +```shell +velero backup create full-cluster-backup +``` + +### Have some Velero backups go to a bucket in an eastern USA region (default), and others go to a bucket in a western USA region + +In this example, two `BackupStorageLocations` will be created within the same account but in different regions. +They will both use the credentials provided at install time and stored in the `cloud-credentials` secret. +If you need to configure unique credentials for each `BackupStorageLocation`, please refer to the [later example][8]. + +During server configuration: + +```shell +velero backup-location create backups-primary \ + --provider aws \ + --bucket velero-backups \ + --config region=us-east-1 \ + --default + +velero backup-location create backups-secondary \ + --provider aws \ + --bucket velero-backups \ + --config region=us-west-1 +``` + +A "default" backup storage location (BSL) is where backups get saved to when no BSL is specified at backup creation time. + +You can change the default backup storage location at any time by setting the `--default` flag using the +`velero backup-location set` command and configure a different location to be the default. + +Examples: + +```shell +velero backup-location set backups-secondary --default +``` + + + +During backup creation: + +```shell +velero backup create full-cluster-backup +``` + +Or: + +```shell +velero backup create full-cluster-alternate-location-backup \ + --storage-location backups-secondary +``` + +### For volume providers that support it (like Portworx), have some snapshots be stored locally on the cluster and have others be stored in the cloud + +During server configuration: + +```shell +velero snapshot-location create portworx-local \ + --provider portworx \ + --config type=local + +velero snapshot-location create portworx-cloud \ + --provider portworx \ + --config type=cloud +``` + +During backup creation: + +```shell +# Note that since in this example you have two possible volume snapshot locations for the Portworx +# provider, you need to explicitly specify which one to use when creating a backup. Alternately, +# you can set the --default-volume-snapshot-locations flag on the `velero server` command (run by +# the Velero deployment) to specify which location should be used for each provider by default, in +# which case you don't need to specify it when creating a backup. +velero backup create local-snapshot-backup \ + --volume-snapshot-locations portworx-local +``` + +Or: + +```shell +velero backup create cloud-snapshot-backup \ + --volume-snapshot-locations portworx-cloud +``` + +### Use a single location + +If you don't have a use case for more than one location, it's still easy to use Velero. Let's assume you're running on AWS, in the `us-west-1` region: + +During server configuration: + +```shell +velero backup-location create backups-primary \ + --provider aws \ + --bucket velero-backups \ + --config region=us-west-1 \ + --default + +velero snapshot-location create ebs-us-west-1 \ + --provider aws \ + --config region=us-west-1 +``` + +During backup creation: + +```shell +# Velero will automatically use your configured backup storage location and volume snapshot location. +# Nothing needs to be specified when creating a backup. +velero backup create full-cluster-backup +``` + +### Create a storage location that uses unique credentials + +It is possible to create additional `BackupStorageLocations` that use their own credentials. +This enables you to save backups to another storage provider or to another account with the storage provider you are already using. + +If you create additional `BackupStorageLocations` without specifying the credentials to use, Velero will use the credentials provided at install time and stored in the `cloud-credentials` secret. +Please see the [earlier example][9] for details on how to create multiple `BackupStorageLocations` that use the same credentials. + +#### Prerequisites +- This feature requires support from the [object storage provider plugin][5] you wish to use. + All plugins maintained by the Velero team support this feature. + If you are using a plugin from another provider, please check their documentation to determine if this is supported. +- The [plugin for the object storage provider][5] you wish to use must be [installed][6]. +- You must create a file with the object storage credentials. Follow the instructions provided by your object storage provider plugin to create this file. + +Once you have installed the necessary plugin and created the credentials file, create a [Kubernetes Secret][7] in the Velero namespace that contains these credentials: + +```shell +kubectl create secret generic -n velero credentials --from-file=bsl= +``` + +This will create a secret named `credentials` with a single key (`bsl`) which contains the contents of your credentials file. +Next, create a `BackupStorageLocation` that uses this Secret by passing the Secret name and key in the `--credential` flag. +When interacting with this `BackupStroageLocation` in the future, Velero will fetch the data from the key within the Secret you provide. + +For example, a new `BackupStorageLocation` with a Secret would be configured as follows: + +```bash +velero backup-location create \ + --provider \ + --bucket \ + --config region= \ + --credential== +``` + +The `BackupStorageLocation` is ready to use when it has the phase `Available`. +You can check the status with the following command: + +```bash +velero backup-location get +``` + +To use this new `BackupStorageLocation` when performing a backup, use the flag `--storage-location ` when running `velero backup create`. +You may also set this new `BackupStorageLocation` as the default with the command `velero backup-location set --default `. + +### Modify the credentials used by an existing storage location + +By default, `BackupStorageLocations` will use the credentials provided at install time and stored in the `cloud-credentials` secret in the Velero namespace. +You can modify these existing credentials by [editing the `cloud-credentials` secret][10], however, these changes will apply to all locations using this secret. +This may be the desired outcome, for example, in the case where you wish to rotate the credentials used for a particular account. + +You can also opt to modify an existing `BackupStorageLocation` such that it uses its own credentials by using the `backup-location set` command. + +If you have a credentials file that you wish to use for a `BackupStorageLocation`, follow the instructions above to create the Secret with that file in the Velero namespace. + +Once you have created the Secret, or have an existing Secret which contains the credentials you wish to use for your `BackupStorageLocation`, set the credential to use as follows: + +```bash +velero backup-location set \ + --credential== +``` + +## Additional Use Cases + +1. If you're using Azure's AKS, you may want to store your volume snapshots outside of the "infrastructure" resource group that is automatically created when you create your AKS cluster. This is possible using a `VolumeSnapshotLocation`, by specifying a `resourceGroup` under the `config` section of the snapshot location. See the [Azure volume snapshot location documentation][3] for details. + +1. If you're using Azure, you may want to store your Velero backups across multiple storage accounts and/or resource groups/subscriptions. This is possible using a `BackupStorageLocation`, by specifying a `storageAccount`, `resourceGroup` and/or `subscriptionId`, respectively, under the `config` section of the backup location. See the [Azure backup storage location documentation][4] for details. + + + +[1]: api-types/backupstoragelocation.md +[2]: api-types/volumesnapshotlocation.md +[3]: https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure/blob/main/volumesnapshotlocation.md +[4]: https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure/blob/main/backupstoragelocation.md +[5]: /plugins +[6]: overview-plugins.md +[7]: https://kubernetes.io/docs/concepts/configuration/secret/ +[8]: #create-a-storage-location-that-uses-unique-credentials +[9]: #have-some-velero-backups-go-to-a-bucket-in-an-eastern-usa-region-default-and-others-go-to-a-bucket-in-a-western-usa-region +[10]: https://kubernetes.io/docs/concepts/configuration/secret/#editing-a-secret diff --git a/site/content/docs/v1.8/maintainers.md b/site/content/docs/v1.8/maintainers.md new file mode 100644 index 000000000..01faab1af --- /dev/null +++ b/site/content/docs/v1.8/maintainers.md @@ -0,0 +1,37 @@ +--- +title: "Instructions for Maintainers" +layout: docs +toc: "true" +--- + +There are some guidelines maintainers need to follow. We list them here for quick reference, especially for new maintainers. These guidelines apply to all projects in the Velero org, including the main project, the Velero Helm chart, and all other [related repositories](https://github.com/vmware-tanzu/velero/blob/v1.8.0/GOVERNANCE.md#code-repositories). + +Please be sure to also go through the guidance under the entire [Contribute](start-contributing/) section. + +## Reviewing PRs +- PRs require 2 approvals before it is mergeable. +- The second reviewer usually merges the PR (if you notice a PR open for a while and with 2 approvals, go ahead and merge it!) +- As you review a PR that is not yet ready to merge, please check if the "request review" needs to be refreshed for any reviewer (this is better than @mention at them) +- Refrain from @mention other maintainers to review the PR unless it is an immediate need. All maintainers already get notified through the automated add to the "request review". If it is an urgent need, please add a helpful message as to why it is so people can properly prioritize work. +- There is no need to manually request reviewers: after the PR is created, all maintainers will be automatically added to the list (note: feel free to remove people if they are on PTO, etc). +- Be familiar with the [lazy consensus](https://github.com/vmware-tanzu/velero/blob/v1.8.0/GOVERNANCE.md#lazy-consensus) policy for the project. + +Some tips for doing reviews: +- There are some [code standards and general guidelines](https://velero.io/docs/v1.8.0/code-standards) we aim for +- We have [guidelines for writing and reviewing documentation](https://velero.io/docs/v1.8.0/style-guide/) +- When reviewing a design document, ensure it follows [our format and guidelines]( https://github.com/vmware-tanzu/velero/blob/v1.8.0/design/_template.md). Also, when reviewing a PR that implements a previously accepted design, ensure the associated design doc is moved to the [design/implemented](https://github.com/vmware-tanzu/velero/tree/main/design/implemented) folder. + + +## Creating a release +Maintainers are expected to create releases for the project. We have parts of the process automated, and full [instructions](release-instructions). +We are working towards automating more the Velero testing, but there is still a need for manual testing as part of the release process. +The manual test cases for release testing are documented [here](./manual-testing). + +## Community support +Maintainers are expected to participate in the community support rotation. We have guidelines for how we handle the [support](support-process). + +## Community engagement +Maintainers for the Velero project are highly involved with the open source community. All the online community meetings for the project are listed in our [community](community) page. + +## How do I become a maintainer? +The Velero project welcomes contributors of all kinds. We are also always on the look out for a high level of engagement from contributors and opportunities to bring in new maintainers. If this is of interest, take a look at how [adding a maintainer](https://github.com/vmware-tanzu/velero/blob/v1.8.0/GOVERNANCE.md#maintainers) is decided. diff --git a/site/content/docs/v1.8/manual-testing.md b/site/content/docs/v1.8/manual-testing.md new file mode 100644 index 000000000..c66366b8a --- /dev/null +++ b/site/content/docs/v1.8/manual-testing.md @@ -0,0 +1,92 @@ +--- +title: "Manual Testing Requirements for Velero" +layout: docs +--- + +Although we have automated unit and end-to-end tests, there is still a need for Velero to undergo manual tests during a release. +This document outlines the manual test operations that Velero needs to correctly perform in order to be considered ready for release. + +## Current test cases + +The following are test cases that are currently performed as part of a Velero release. + +### Install + +- Verify that Velero CRDs are compatible with the earliest and latest versions of Kubernetes that we support: + - Kubernetes v1.16 + - Kubernetes v1.22 + +### Upgrade + +- Verify that Velero upgrade instructions work + +### Basic functionality + +The "Backup and Restore" test cases below describe general backup and restore functionality that needs to run successfully on all the following providers that we maintain plugins for: +- AWS +- GCP +- Microsoft Azure +- VMware vSphere + +#### Backup and Restore + +- Verify that a backup and restore using Volume Snapshots can be performed +- Verify that a backup and restore using Restic can be performed +- Verify that a backup of a cluster workload can be restored in a new cluster +- Verify that an installation using the latest version can be used to restore from backups created with the last 3 versions. + - e.g. Install Velero 1.6 and use it to restore backups from Velero v1.3, v1.4, v1.5. + +### Working with Multiple Providers + +The following are test cases that exercise Velero behaviour when interacting with multiple providers: + +- Verify that a backup and restore to multiple BackupStorageLocations using the same provider with unique credentials can be performed +- Verify that a backup and restore to multiple BackupStorageLocations using different providers with unique credentials can be performed +- Verify that a backup and restore that includes volume snapshots using different providers for the snapshots and object storage can be performed + - e.g. perform a backup and restore using AWS for the VolumeSnapshotLocation and Azure Blob Storage as the BackupStorageLocation + +## Future test cases + +The following are test cases that are not currently performed as part of a Velero release but cases that we will want to cover with future releases. + +### Schedules + +- Verify that schedules create a backup upon creation and create Backup resources at the correct frequency + +### Resource management + +- Verify that deleted backups are successfully removed from object storage +- Verify that backups that have been removed from object storage can still be deleted with `velero delete backup` +- Verify that Volume Snapshots associated with a deleted backup are removed +- Verify that backups that exceed their TTL are deleted +- Verify that existing backups in object storage are synced to Velero + +### Restic repository test cases + +- Verify that restic repository maintenance is performed as the specified interval + +### Backup Hooks + +- Verify that a pre backup hook provided via pod annotation is performed during backup +- Verify that a pre backup hook provided via Backup spec is performed during backup +- Verify that a post backup hook provided via pod annotation is performed during backup +- Verify that a post backup hook provided via Backup spec is performed during backup + +### Restore Hooks + +- Verify that an InitContainer restore hook provided via pod annotation is performed during restore +- Verify that an InitContainer restore hook provided via Restore spec is performed during restore +- Verify that an InitContainer restore hook provided via Restore spec is performed during restore that includes restoring restic volumes +- Verify that an Exec restore hook provided via pod annotation is performed during restore +- Verify that an Exec restore hook provided via Restore spec is performed during restore + + +#### Resource filtering + +- Verify that backups and restores correctly apply the following resource filters: + - `--include-namespaces` + - `--include-resources` + - `--include-cluster-resources` + - `--exclude-namespaces` + - `--exclude-resources` + - `velero.io/exclude-from-backup=true` label diff --git a/site/content/docs/v1.8/migration-case.md b/site/content/docs/v1.8/migration-case.md new file mode 100644 index 000000000..c547ebbb4 --- /dev/null +++ b/site/content/docs/v1.8/migration-case.md @@ -0,0 +1,58 @@ +--- +title: "Cluster migration" +layout: docs +--- + +## Using Backups and Restores + +Velero can help you port your resources from one cluster to another, as long as you point each Velero instance to the same cloud object storage location. This scenario assumes that your clusters are hosted by the same cloud provider. **Note that Velero does not natively support the migration of persistent volumes snapshots across cloud providers.** If you would like to migrate volume data between cloud platforms, please enable [restic][2], which will backup volume contents at the filesystem level. + +1. *(Cluster 1)* Assuming you haven't already been checkpointing your data with the Velero `schedule` operation, you need to first back up your entire cluster (replacing `` as desired): + + ``` + velero backup create + ``` + + The default backup retention period, expressed as TTL (time to live), is 30 days (720 hours); you can use the `--ttl ` flag to change this as necessary. See [how velero works][1] for more information about backup expiry. + +1. *(Cluster 2)* Configure `BackupStorageLocations` and `VolumeSnapshotLocations`, pointing to the locations used by *Cluster 1*, using `velero backup-location create` and `velero snapshot-location create`. Make sure to configure the `BackupStorageLocations` as read-only + by using the `--access-mode=ReadOnly` flag for `velero backup-location create`. + +1. *(Cluster 2)* Make sure that the Velero Backup object is created. Velero resources are synchronized with the backup files in cloud storage. + + ``` + velero backup describe + ``` + + **Note:** The default sync interval is 1 minute, so make sure to wait before checking. You can configure this interval with the `--backup-sync-period` flag to the Velero server. + +1. *(Cluster 2)* Once you have confirmed that the right Backup (``) is now present, you can restore everything with: + + ``` + velero restore create --from-backup + ``` + +## Verify Both Clusters + +Check that the second cluster is behaving as expected: + +1. *(Cluster 2)* Run: + + ``` + velero restore get + ``` + +1. Then run: + + ``` + velero restore describe + ``` + +If you encounter issues, make sure that Velero is running in the same namespace in both clusters. + +## Migrating Workloads Across Different Kubernetes Versions + +Migration across clusters that are not running the same version of Kubernetes might be possible, but some factors need to be considered: compatibility of API groups between clusters for each custom resource, and if a Kubernetes version upgrade breaks the compatibility of core/native API groups. For more information about API group versions, please see [EnableAPIGroupVersions](enable-api-group-versions-feature.md). + +[1]: how-velero-works.md#set-a-backup-to-expire +[2]: restic.md diff --git a/site/content/docs/v1.8/namespace.md b/site/content/docs/v1.8/namespace.md new file mode 100644 index 000000000..68561e720 --- /dev/null +++ b/site/content/docs/v1.8/namespace.md @@ -0,0 +1,22 @@ +--- +title: "Run in a non-default namespace" +layout: docs +--- + +The Velero installation and backups by default are run in the `velero` namespace. However, it is possible to use a different namespace. + +## Customize the namespace during install + +Use the `--namespace` flag, in conjunction with the other flags in the `velero install` command (as shown in the [the Velero install instructions][0]). This will inform Velero where to install. + +## Customize the namespace for operational commands + +To have namespace consistency, specify the namespace for all Velero operational commands to be the same as the namespace used to install Velero: + +```bash +velero client config set namespace= +``` + +Alternatively, you may use the global `--namespace` flag with any operational command to tell Velero where to run. + +[0]: basic-install.md#install-the-cli diff --git a/site/content/docs/v1.8/on-premises.md b/site/content/docs/v1.8/on-premises.md new file mode 100644 index 000000000..30ddf98b3 --- /dev/null +++ b/site/content/docs/v1.8/on-premises.md @@ -0,0 +1,95 @@ +--- +title: "On-Premises Environments" +layout: docs +--- + +You can run Velero in an on-premises cluster in different ways depending on your requirements. + +### Selecting an object storage provider + +You must select an object storage backend that Velero can use to store backup data. [Supported providers][0] contains information on various +options that are supported or have been reported to work by users. + +If you do not already have an object storage system, [MinIO][2] is an open-source S3-compatible object storage system that can be installed on-premises and is compatible with Velero. The details of configuring it for production usage are out of scope for Velero's documentation, but an [evaluation install guide][3] using MinIO is provided for convenience. + +### (Optional) Selecting volume snapshot providers + +If you need to back up persistent volume data, you must select a volume backup solution. [Supported providers][0] contains information on the supported options. + +For example, if you use [Portworx][4] for persistent storage, you can install their Velero plugin to get native Portworx snapshots as part of your Velero backups. + +If there is no native snapshot plugin available for your storage platform, you can use Velero's [restic integration][1], which provides a platform-agnostic file-level backup solution for volume data. + +### Air-gapped deployments + +In an air-gapped deployment, there is no access to the public internet, and therefore no access to public container registries. + +In these scenarios, you will need to make sure that you have an internal registry, such as [Harbor][5], installed and the Velero core and plugin images loaded into your internal registry. + +Below you will find instructions to downloading the Velero images to your local machine, tagging them, then uploading them to your custom registry. + +#### Preparing the Velero image + +First, download the Velero image, tag it for the your private registry, then upload it into the registry so that it can be pulled by your cluster. + +```bash +PRIVATE_REG= +VELERO_VERSION= + +docker pull velero/velero:$VELERO_VERSION +docker tag velero/velero:$VELERO_VERSION $PRIVATE_REG/velero:$VELERO_VERSION +docker push $PRIVATE_REG/velero:$VELERO_VERSION +``` + +#### Preparing plugin images + +Next, repeat these steps for any plugins you may need. This example will use the AWS plugin, but the plugin name should be replaced with the plugins you will need. + +```bash +PRIVATE_REG= +PLUGIN_VERSION= + +docker pull velero/velero-plugin-for-aws:$PLUGIN_VERSION +docker tag velero/velero-plugin-for-aws:$PLUGIN_VERSION $PRIVATE_REG/velero-plugin-for-aws:$PLUGIN_VERSION +docker push $PRIVATE_REG/velero-plugin-for-aws:$PLUGIN_VERSION +``` + +#### Preparing the restic helper image (optional) + +If you are using restic, you will also need to upload the restic helper image. + +```bash +PRIVATE_REG= +VELERO_VERSION= + +docker pull velero/velero-restic-restore-helper:$VELERO_VERSION +docker tag velero/velero-restic-restore-helper:$VELERO_VERSION $PRIVATE_REG/velero-restic-restore-helper:$VELERO_VERSION +docker push $PRIVATE_REG/velero-restic-restore-helper:$VELERO_VERSION +``` + +#### Pulling specific architecture images (optional) + +Velero uses Docker manifests for its images, allowing Docker to pull the image needed based on your client machine's architecture. + +If you need to pull a specific image, you should replace the `velero/velero` image with the specific architecture image, such as `velero/velero-arm`. + +To see an up-to-date list of architectures, be sure to enable Docker experimental features and use `docker manifest inspect velero/velero` (or whichever image you're interested in), and join the architecture string to the end of the image name with `-`. + +#### Installing Velero + +By default, `velero install` will use the public `velero/velero` image. When using an air-gapped deployment, use your private registry's image for Velero and your private registry's images for any plugins. + +```bash +velero install \ + --image=$PRIVATE_REG/velero:$VELERO_VERSION \ + --plugins=$PRIVATE_REG/velero-plugin-for-aws:$PLUGIN_VERSION \ +<....> +``` + + +[0]: supported-providers.md +[1]: restic.md +[2]: https://min.io +[3]: contributions/minio.md +[4]: https://portworx.com +[5]: https://goharbor.io/ diff --git a/site/content/docs/v1.8/output-file-format.md b/site/content/docs/v1.8/output-file-format.md new file mode 100644 index 000000000..8d4823505 --- /dev/null +++ b/site/content/docs/v1.8/output-file-format.md @@ -0,0 +1,224 @@ +--- +title: "Output file format" +layout: docs +--- + +A backup is a gzip-compressed tar file whose name matches the Backup API resource's `metadata.name` (what is specified during `velero backup create `). + +In cloud object storage, each backup file is stored in its own subdirectory in the bucket specified in the Velero server configuration. This subdirectory includes an additional file called `velero-backup.json`. The JSON file lists all information about your associated Backup resource, including any default values. This gives you a complete historical record of the backup configuration. The JSON file also specifies `status.version`, which corresponds to the output file format. + +The directory structure in your cloud storage looks something like: + +``` +rootBucket/ + backup1234/ + velero-backup.json + backup1234.tar.gz +``` + +## Example backup JSON file + +```json +{ + "kind": "Backup", + "apiVersion": "velero.io/v1", + "metadata": { + "name": "test-backup", + "namespace": "velero", + "selfLink": "/apis/velero.io/v1/namespaces/velero/backups/test-backup", + "uid": "a12345cb-75f5-11e7-b4c2-abcdef123456", + "resourceVersion": "337075", + "creationTimestamp": "2017-07-31T13:39:15Z" + }, + "spec": { + "includedNamespaces": [ + "*" + ], + "excludedNamespaces": null, + "includedResources": [ + "*" + ], + "excludedResources": null, + "labelSelector": null, + "snapshotVolumes": true, + "ttl": "24h0m0s" + }, + "status": { + "version": 1, + "formatVersion": "1.1.0", + "expiration": "2017-08-01T13:39:15Z", + "phase": "Completed", + "volumeBackups": { + "pvc-e1e2d345-7583-11e7-b4c2-abcdef123456": { + "snapshotID": "snap-04b1a8e11dfb33ab0", + "type": "gp2", + "iops": 100 + } + }, + "validationErrors": null + } +} +``` +Note that this file includes detailed info about your volume snapshots in the `status.volumeBackups` field, which can be helpful if you want to manually check them in your cloud provider GUI. + +## Output File Format Versioning + +The Velero output file format is intended to be relatively stable, but may change over time to support new features. + +To accommodate this, Velero follows [Semantic Versioning](http://semver.org/) for the file format version. + +Minor and patch versions will indicate backwards-compatible changes that previous versions of Velero can restore, including new directories or files. + +A major version would indicate that a version of Velero older than the version that created the backup could not restore it, usually because of moved or renamed directories or files. + +Major versions of the file format will be incremented with major version releases of Velero. +However, a major version release of Velero does not necessarily mean that the backup format version changed - Velero 3.0 could still use backup file format 2.0, as an example. + +## Versions + +### File Format Version: 1.1 (Current) + +Version 1.1 added support of API groups versions as part of the backup. Previously, only the preferred version of each API groups was backed up. Each resource has one or more sub-directories: one sub-directory for each supported version of the API group. The preferred version API Group of each resource has the suffix "-preferredversion" as part of the sub-directory name. For backward compatibility, we kept the classic directory structure without the API group version, which sits on the same level as the API group sub-directory versions. + +By default, only the preferred API group of each resource is backed up. To take a backup of all API group versions, you need to run the Velero server with the `--features=EnableAPIGroupVersions` feature flag. This is an experimental flag and the restore logic to handle multiple API group versions is documented at [EnableAPIGroupVersions](enable-api-group-versions-feature.md). + +When unzipped, a typical backup directory (`backup1234.tar.gz`) taken with this file format version looks like the following (with the feature flag): + +``` +resources/ + persistentvolumes/ + cluster/ + pv01.json + ... + v1-preferredversion/ + cluster/ + pv01.json + ... + configmaps/ + namespaces/ + namespace1/ + myconfigmap.json + ... + namespace2/ + ... + v1-preferredversion/ + namespaces/ + namespace1/ + myconfigmap.json + ... + namespace2/ + ... + pods/ + namespaces/ + namespace1/ + mypod.json + ... + namespace2/ + ... + v1-preferredversion/ + namespaces/ + namespace1/ + mypod.json + ... + namespace2/ + ... + jobs.batch/ + namespaces/ + namespace1/ + awesome-job.json + ... + namespace2/ + ... + v1-preferredversion/ + namespaces/ + namespace1/ + awesome-job.json + ... + namespace2/ + ... + deployments/ + namespaces/ + namespace1/ + cool-deployment.json + ... + namespace2/ + ... + v1-preferredversion/ + namespaces/ + namespace1/ + cool-deployment.json + ... + namespace2/ + ... + horizontalpodautoscalers.autoscaling/ + namespaces/ + namespace1/ + hpa-to-the-rescue.json + ... + namespace2/ + ... + v1-preferredversion/ + namespaces/ + namespace1/ + hpa-to-the-rescue.json + ... + namespace2/ + ... + v2beta1/ + namespaces/ + namespace1/ + hpa-to-the-rescue.json + ... + namespace2/ + ... + v2beta2/ + namespaces/ + namespace1/ + hpa-to-the-rescue.json + ... + namespace2/ + ... + + ... +``` + +### File Format Version: 1 + +When unzipped, a typical backup directory (`backup1234.tar.gz`) looks like the following: + +``` +resources/ + persistentvolumes/ + cluster/ + pv01.json + ... + configmaps/ + namespaces/ + namespace1/ + myconfigmap.json + ... + namespace2/ + ... + pods/ + namespaces/ + namespace1/ + mypod.json + ... + namespace2/ + ... + jobs/ + namespaces/ + namespace1/ + awesome-job.json + ... + namespace2/ + ... + deployments/ + namespaces/ + namespace1/ + cool-deployment.json + ... + namespace2/ + ... + ... +``` diff --git a/site/content/docs/v1.8/overview-plugins.md b/site/content/docs/v1.8/overview-plugins.md new file mode 100644 index 000000000..037601d51 --- /dev/null +++ b/site/content/docs/v1.8/overview-plugins.md @@ -0,0 +1,29 @@ +--- +title: "Velero plugin system" +layout: docs +--- + +Velero uses storage provider plugins to integrate with a variety of storage systems to support backup and snapshot operations. + +For server installation, Velero requires that at least one plugin is added (with the `--plugins` flag). The plugin will be either of the type object store or volume snapshotter, or a plugin that contains both. An exception to this is that when the user is not configuring a backup storage location or a snapshot storage location at the time of install, this flag is optional. + +Any plugin can be added after Velero has been installed by using the command `velero plugin add `. + +Example with a dockerhub image: `velero plugin add velero/velero-plugin-for-aws:v1.0.0`. + +In the same way, any plugin can be removed by using the command `velero plugin remove `. + +## Creating a new plugin + +Anyone can add integrations for any platform to provide additional backup and volume storage without modifying the Velero codebase. To write a plugin for a new backup or volume storage platform, take a look at our [example repo][1] and at our documentation for [Custom plugins][2]. + +## Adding a new plugin + +After you publish your plugin on your own repository, open a PR that adds a link to it under the appropriate list of [supported providers][3] page in our documentation. + +You can also add the [`velero-plugin` GitHub Topic][4] to your repo, and it will be shown under the aggregated list of repositories automatically. + +[1]: https://github.com/vmware-tanzu/velero-plugin-example/ +[2]: custom-plugins.md +[3]: supported-providers.md +[4]: https://github.com/topics/velero-plugin diff --git a/site/content/docs/v1.8/plugin-release-instructions.md b/site/content/docs/v1.8/plugin-release-instructions.md new file mode 100644 index 000000000..4323d8c9a --- /dev/null +++ b/site/content/docs/v1.8/plugin-release-instructions.md @@ -0,0 +1,30 @@ +--- +title: Releasing Velero plugins +layout: docs +toc: "true" +--- + +Velero plugins maintained by the core maintainers do not have any shipped binaries, only container images, so there is no need to invoke a GoReleaser script. +Container images are built via a CI job on git push. + +Plugins the Velero core team is responsible include all those listed in [the Velero-supported providers list](supported-providers.md) _except_ the vSphere plugin. + + +## Steps +### Open a PR to prepare the repo +1. Update the README.md file to update the compatibility matrix and `velero install` instructions with the expected version number and open a PR. +1. Determining the version number is based on semantic versioning and whether the plugin uses any newly introduced, changed, or removed methods or variables from Velero. +2. Roll all unreleased changelogs into a new `CHANGELOG-v.md` file and delete the content of the `unreleased` folder. Edit the new changelog file as needed. +### Tag +1. Once the PR is merged, checkout the upstream `main` branch. Your local upstream might be named `upstream` or `origin`, so use this command: `git checkout /main`. +1. Tag the git version - `git tag v`. +1. Push the git tag - `git push --tags ` to trigger the image build. +2. Wait for the container images to build. You may check the progress of the GH action that triggers the image build at `https://github.com/vmware-tanzu//actions` +3. Verify that an image with the new tag is available at `https://hub.docker.com/repository/docker/velero//`. +4. Run the Velero [e2e tests][2] using the new image. Until it is made configurable, you will have to edit the [plugin version][1] in the test. +### Release +1. If all e2e tests pass, go to the GitHub release page of the plugin (`https://github.com/vmware-tanzu//releases`) and manually create a release for the new tag. +1. Copy and paste the content of the new changelog file into the release description field. + +[1]: https://github.com/vmware-tanzu/velero/blob/c8dfd648bbe85db0184ea53296de4220895497e6/test/e2e/velero_utils.go#L27 +[2]: https://github.com/vmware-tanzu/velero/tree/main/test/e2e diff --git a/site/content/docs/v1.8/rbac.md b/site/content/docs/v1.8/rbac.md new file mode 100644 index 000000000..a9c22b9f1 --- /dev/null +++ b/site/content/docs/v1.8/rbac.md @@ -0,0 +1,50 @@ +--- +title: "Run Velero more securely with restrictive RBAC settings" +layout: docs +--- + +By default Velero runs with an RBAC policy of ClusterRole `cluster-admin`. This is to make sure that Velero can back up or restore anything in your cluster. But `cluster-admin` access is wide open -- it gives Velero components access to everything in your cluster. Depending on your environment and your security needs, you should consider whether to configure additional RBAC policies with more restrictive access. + +**Note:** Roles and RoleBindings are associated with a single namespaces, not with an entire cluster. PersistentVolume backups are associated only with an entire cluster. This means that any backups or restores that use a restrictive Role and RoleBinding pair can manage only the resources that belong to the namespace. You do not need a wide open RBAC policy to manage PersistentVolumes, however. You can configure a ClusterRole and ClusterRoleBinding that allow backups and restores only of PersistentVolumes, not of all objects in the cluster. + +For more information about RBAC and access control generally in Kubernetes, see the Kubernetes documentation about [access control][1], [managing service accounts][2], and [RBAC authorization][3]. + +## Set up Roles and RoleBindings + +Here's a sample Role and RoleBinding pair. + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: YOUR_NAMESPACE_HERE + name: ROLE_NAME_HERE + labels: + component: velero +rules: + - apiGroups: + - velero.io + verbs: + - "*" + resources: + - "*" +``` + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ROLEBINDING_NAME_HERE +subjects: + - kind: ServiceAccount + name: YOUR_SERVICEACCOUNT_HERE +roleRef: + kind: Role + name: ROLE_NAME_HERE + apiGroup: rbac.authorization.k8s.io +``` + +[1]: https://kubernetes.io/docs/reference/access-authn-authz/controlling-access/ +[2]: https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/ +[3]: https://kubernetes.io/docs/reference/access-authn-authz/rbac/ +[4]: namespace.md diff --git a/site/content/docs/v1.8/release-instructions.md b/site/content/docs/v1.8/release-instructions.md new file mode 100644 index 000000000..43e2e632a --- /dev/null +++ b/site/content/docs/v1.8/release-instructions.md @@ -0,0 +1,167 @@ +--- +title: "Release Instructions" +layout: docs +toc: "true" +--- +This page covers the steps to perform when releasing a new version of Velero. + +## General notes +- Please read the documented variables in each script to understand what they are for and how to properly format their values. +- You will need to have an upstream remote configured to use to the [vmware-tanzu/velero](https://github.com/vmware-tanzu/velero) repository. + You can check this using `git remote -v`. + The release script ([`tag-release.sh`](https://github.com/vmware-tanzu/velero/blob/v1.8.0/hack/release-tools/tag-release.sh)) will use `upstream` as the default remote name if it is not specified using the environment variable `REMOTE`. +- GA release: major and minor releases only. Example: 1.0 (major), 1.5 (minor). +- Pre-releases: Any release leading up to a GA. Example: 1.4.0-beta.1, 1.5.0-rc.1 +- RC releases: Release Candidate, contains everything that is supposed to ship with the GA release. This is still a pre-release. + +## Velero Release Requirements + +Velero is on a "train leaves the station" model for releases. We will generate a release candidate (RC) +at the scheduled time. Multiple release candidates may be generated, depending on if bugs are found during testing. +When testing has passed a release build will be generated. + +### Release Candidate criteria +The release candidate commit must meet the following criteria: + +* No major bugs outstanding +* Unit tests pass +* E2E tests against latest Kubernetes on AWS, vSphere and kind pass + +Once the release has moved to RC, a code freeze is in effect. Only changes needed to release are allowable. + +### Release criteria +In order for a release candidate to be released, it must meet the following criteria: + +* Unit tests pass +* E2E tests against latest K8S and earliest supported K8S on Azure, vSphere, Kind, AWS, GCP +* Manual tests pass (manual tests will be converted to automated testing) + +When bugs are identified by any of these methods, we will determine whether the bug is a release blocker or not and +a fix generated if it is. When release blocker bugs identifies in an release candidate are fixed, another RC will +be generated and the test cycle will restart. + +## Preparing + +### Create release blog post (GA only) +For each major or minor release, create and publish a blog post to let folks know what's new. Please follow these [instructions](how-to-write-and-release-a-blog-post). + +### Changelog and Docs PR +#### Troubleshooting +- If you encounter the error `You don't have enough free space in /var/cache/apt/archives/` when running `make serve-docs`: run `docker system prune`. + +#### Steps +1. If it doesn't already exist: in a branch, create the file `changelogs/CHANGELOG-..md` by copying the most recent one. +1. Update the file `changelogs/CHANGELOG-..md` + - Run `make changelog` to generate a list of all unreleased changes. + - Copy/paste the output into `CHANGELOG-..md`, under the "All Changes" section for the release. + - You *may* choose to tweak formatting on the list of changes by adding code blocks, etc. + - Update links at the top of the file to point to the new release version +1. Update the main `CHANGELOG.md` file to properly reference the release-specific changelog file + - Under "Current release": + - Should contain only the current GA release. + - Under "Development release": + - Should contain only the latest pre-release + - Move any prior pre-release into "Older releases" +1. GA Only: Remove all changelog files from `changelogs/unreleased`. +1. Generate new docs + - Run `make gen-docs`, passing the appropriate variables. Examples: + a) `VELERO_VERSION=v1.5.0-rc.1 NEW_DOCS_VERSION=v1.5.0-rc.1 make gen-docs`. + b) `VELERO_VERSION=v1.5.0 NEW_DOCS_VERSION=v1.5 make gen-docs`). + - Note: `PREVIOUS_DOCS_VERSION=` is optional; when not set, it will default to the latest doc version. +1. Clean up when there is an existing set of pre-release versioned docs for the version you are releasing + - Example: `site/content/docs/v1.5.0-beta.1` exists, and you're releasing `v1.5.0-rc.1` or `v1.5` + - Remove the directory containing the pre-release docs, i.e. `site/content/docs/`. + - Delete the pre-release docs table of contents file, i.e. `site/data/docs/-toc.yml`. + - Remove the pre-release docs table of contents mapping entry from `site/data/toc-mapping.yml`. + - Remove all references to the pre-release docs from `site/config.yml`. +1. Create the "Upgrade to $major.minor" page if it does not already exist ([example](https://velero.io/docs/v1.5/upgrade-to-1.5/)). + If it already exists, update any usage of the previous version string within this file to use the new version string instead ([example](https://github.com/vmware-tanzu/velero/pull/2941/files#diff-d594f8fd0901fed79c39aab4b348193d)). + This needs to be done in both the versioned and the `main` folders. +1. Review and submit PR + - Follow the additional instructions at `site/README-HUGO.md` to complete the docs generation process. + - Do a review of the diffs, and/or run `make serve-docs` and review the site. + - Submit a PR containing the changelog and the version-tagged docs. + +### Pin the base image +The image of velero is built based on [Distroless docker image](https://github.com/GoogleContainerTools/distroless). +For the reproducibility of the release, before the release candidate is tagged, we need to make sure the in the Dockerfile +on the release branch, the base image is referenced by digest, such as +https://github.com/vmware-tanzu/velero/blob/release-1.7/Dockerfile#L53-L54 + +## Velero release +### Notes +- Pre-requisite: PR with the changelog and docs is merged, so that it's included in the release tag. +- This process is the same for both pre-release and GA. +- Refer to the [General notes](general-notes) above for instructions. + +#### Troubleshooting +- If the dry-run fails with random errors, try running it again. + +#### Steps +1. Create a tagged release in dry-run mode + - This won't push anything to GitHub. + - Run `VELERO_VERSION=v1.0.0-rc.1 REMOTE= GITHUB_TOKEN=REDACTED ./hack/release-tools/tag-release.sh`. + - Fix any issue. +1. Create a tagged release and push it to GitHub + - Run `VELERO_VERSION=v1.0.0-rc.1 REMOTE= GITHUB_TOKEN=REDACTED ./hack/release-tools/tag-release.sh publish`. +1. Publish the release + - Navigate to the draft GitHub release at https://github.com/vmware-tanzu/velero/releases and edit the release. + - If this is a patch release (e.g. `v1.4.1`), note that the full `CHANGELOG-1.4.md` contents will be included in the body of the GitHub release. You need to delete the previous releases' content (e.g. `v1.2.0`'s changelog) so that only the latest patch release's changelog shows. + - Do a quick review for formatting. + - **Note:** the `goreleaser` process should have detected if it's a pre-release version and, if so, checked the box at the bottom of the GitHub release page appropriately, but it's always worth double-checking. + - Verify that GitHub has built and pushed all the images (it takes a while): https://github.com/vmware-tanzu/velero/actions + - Verify that the images are on Docker Hub: https://hub.docker.com/r/velero/velero/tags + - Verify that the assets were published to the GitHub release + - Publish the release. +1. Test the release + - By now, the Docker images should have been published. + - Perform a smoke-test - for example: + - Download the CLI from the GitHub release + - Use it to install Velero into a cluster (or manually update an existing deployment to use the new images) + - Verify that `velero version` shows the expected output + - Run a backup/restore and ensure it works + +## Homebrew release (GA only) +These are the steps to update the Velero Homebrew version. + +### Steps +- If you don't already have one, create a [GitHub access token for Homebrew](https://github.com/settings/tokens/new?scopes=gist,public_repo&description=Homebrew) +- Run `export HOMEBREW_GITHUB_API_TOKEN=your_token_here` on your command line to make sure that `brew` can work on GitHub on your behalf. +- Run `hack/release-tools/brew-update.sh`. This script will download the necessary files, do the checks, and invoke the brew helper to submit the PR, which will open in your browser. +- Update Windows Chocolatey version. From a Windows computer, follow the step-by-step instructions to [create the Windows Chocolatey package for Velero CLI](https://github.com/adamrushuk/velero-choco/blob/main/README.md) +- +## Plugins + +To release plugins maintained by the Velero team, follow the [plugin release instructions](plugin-release-instructions.md). + +After the plugin images are built, be sure to update any [e2e tests][3] that use these plugins. + +## How to write and release a blog post +What to include in a release blog: +* Thank all contributors for their involvement in the release. + * Where possible shoutout folks by name or consider spotlighting new maintainers. +* Highlight the themes, or areas of focus, for the release. Some examples of themes are security, bug fixes, feature improvements. See past Velero [release blog posts][1] for more examples. +* Include summaries of new features or workflows introduced in a release. + * This can also include new project initiatives, like a code-of-conduct update. + * Consider creating additional blog posts that go through new features in more detail. Plan to publish additional blogs after the release blog (all blogs don’t have to be publish all at once). + +Release blog post PR: +* Prepare a PR containing the release blog post. Read the [website guidelines][2] for more information on creating a blog post. It's usually easiest to make a copy of the most recent existing post, then replace the content as appropriate. +* You also need to update `site/index.html` to have "Latest Release Information" contain a link to the new post. +* Plan to publish the blog post the same day as the release. + +## Announce a release +Once you are finished doing the release, let the rest of the world know it's available by posting messages in the following places. +1. GA Only: Merge the blog post PR. +1. Velero's Twitter account. Maintainers are encouraged to help spread the word by posting or reposting on social media. +1. Community Slack channel. +1. Google group message. + +What to include: +* Thank all contributors +* A brief list of highlights in the release +* Link to the release blog post, release notes, and/or github release page + +[1]: https://velero.io/blog +[2]: website-guidelines.md +[3]: https://github.com/vmware-tanzu/velero/tree/main/test/e2e diff --git a/site/content/docs/v1.8/release-schedule.md b/site/content/docs/v1.8/release-schedule.md new file mode 100644 index 000000000..ca88383d2 --- /dev/null +++ b/site/content/docs/v1.8/release-schedule.md @@ -0,0 +1,15 @@ +--- +title: "Release Schedule" +layout: docs +toc: "true" +--- + +Definitions borrowed from [the Kubernetes release process document](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-release/release.md#definitions) + +General phases for a Velero release +- Enhancement/Design freeze +- Implementation phase +- Feature freeze & pruning +- Code freeze & prerelease +- Release + diff --git a/site/content/docs/v1.8/resource-filtering.md b/site/content/docs/v1.8/resource-filtering.md new file mode 100644 index 000000000..ddcd97e4a --- /dev/null +++ b/site/content/docs/v1.8/resource-filtering.md @@ -0,0 +1,138 @@ +--- +title: "Resource filtering" +layout: docs +--- + +*Filter objects by namespace, type, or labels.* + +Velero includes all objects in a backup or restore when no filtering options are used. + +## Includes + +Only specific resources are included, excluding all others. + +Wildcard takes precedence when both a wildcard and specific resource are included. + +### --include-namespaces + +* Backup a namespace and it's objects. + + ```bash + velero backup create --include-namespaces + ``` + +* Restore two namespaces and their objects. + + ```bash + velero restore create --include-namespaces , + ``` + +### --include-resources + +* Backup all deployments in the cluster. + + ```bash + velero backup create --include-resources deployments + ``` + +* Restore all deployments and configmaps in the cluster. + + ```bash + velero restore create --include-resources deployments,configmaps + ``` + +* Backup the deployments in a namespace. + + ```bash + velero backup create --include-resources deployments --include-namespaces + ``` + +### --include-cluster-resources + + This option can have three possible values: + +* `true`: all cluster-scoped resources are included. + +* `false`: no cluster-scoped resources are included. + +* `nil` ("auto" or not supplied): + + - Cluster-scoped resources are included when backing up or restoring all namespaces. Default: `true`. + + - Cluster-scoped resources are not included when namespace filtering is used. Default: `false`. + + * Some related cluster-scoped resources may still be backed/restored up if triggered by a custom action (for example, PVC->PV) unless `--include-cluster-resources=false`. + +* Backup entire cluster including cluster-scoped resources. + + ```bash + velero backup create + ``` + +* Restore only namespaced resources in the cluster. + + ```bash + velero restore create --include-cluster-resources=false + ``` + +* Backup a namespace and include cluster-scoped resources. + + ```bash + velero backup create --include-namespaces --include-cluster-resources=true + ``` + +### --selector + +* Include resources matching the label selector. + + ```bash + velero backup create --selector = + ``` +* Include resources that are not matching the selector + ```bash + velero backup create --selector != + ``` + +For more information read the [Kubernetes label selector documentation](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors) + + +## Excludes + +Exclude specific resources from the backup. + +Wildcard excludes are ignored. + +### --exclude-namespaces + +* Exclude kube-system from the cluster backup. + + ```bash + velero backup create --exclude-namespaces kube-system + ``` + +* Exclude two namespaces during a restore. + + ```bash + velero restore create --exclude-namespaces , + ``` + +### --exclude-resources + +* Exclude secrets from the backup. + + ```bash + velero backup create --exclude-resources secrets + ``` + +* Exclude secrets and rolebindings. + + ```bash + velero backup create --exclude-resources secrets,rolebindings + ``` + +### velero.io/exclude-from-backup=true + +* Resources with the label `velero.io/exclude-from-backup=true` are not included in backup, even if it contains a matching selector label. + + + diff --git a/site/content/docs/v1.8/restic.md b/site/content/docs/v1.8/restic.md new file mode 100644 index 000000000..a15e08bf2 --- /dev/null +++ b/site/content/docs/v1.8/restic.md @@ -0,0 +1,528 @@ +--- +title: "Restic Integration" +layout: docs +--- + +Velero supports backing up and restoring Kubernetes volumes using a free open-source backup tool called [restic][1]. This support is considered beta quality. Please see the list of [limitations](#limitations) to understand if it fits your use case. + +Velero allows you to take snapshots of persistent volumes as part of your backups if you’re using one of +the supported cloud providers’ block storage offerings (Amazon EBS Volumes, Azure Managed Disks, Google Persistent Disks). +It also provides a plugin model that enables anyone to implement additional object and block storage backends, outside the +main Velero repository. + +Velero's Restic integration was added to give you an out-of-the-box solution for backing up and restoring almost any type of Kubernetes volume. This integration is an addition to Velero's capabilities, not a replacement for existing functionality. If you're running on AWS, and taking EBS snapshots as part of your regular Velero backups, there's no need to switch to using Restic. However, if you need a volume snapshot plugin for your storage platform, or if you're using EFS, AzureFile, NFS, emptyDir, +local, or any other volume type that doesn't have a native snapshot concept, Restic might be for you. + +Restic is not tied to a specific storage platform, which means that this integration also paves the way for future work to enable +cross-volume-type data migrations. + +**NOTE:** hostPath volumes are not supported, but the [local volume type][4] is supported. + +## Setup Restic + +### Prerequisites + +- Understand how Velero performs [backups with the Restic integration](#how-backup-and-restore-work-with-restic). +- [Download][3] the latest Velero release. +- Kubernetes v1.12.0 and later. Velero's Restic integration requires the Kubernetes [MountPropagation feature][6], which is enabled by default in Kubernetes v1.12.0 and later. + +### Install Restic + +To install Restic, use the `--use-restic` flag in the `velero install` command. See the [install overview][2] for more details on other flags for the install command. + +``` +velero install --use-restic +``` + +When using Restic on a storage provider that doesn't have Velero support for snapshots, the `--use-volume-snapshots=false` flag prevents an unused `VolumeSnapshotLocation` from being created on installation. + +### Configure Restic DaemonSet spec + +After installation, some PaaS/CaaS platforms based on Kubernetes also require modifications the Restic DaemonSet spec. The steps in this section are only needed if you are installing on RancherOS, OpenShift, VMware Tanzu Kubernetes Grid Integrated Edition (formerly VMware Enterprise PKS), or Microsoft Azure. + + +**RancherOS** + + +Update the host path for volumes in the Restic DaemonSet in the Velero namespace from `/var/lib/kubelet/pods` to `/opt/rke/var/lib/kubelet/pods`. + +```yaml +hostPath: + path: /var/lib/kubelet/pods +``` + +to + +```yaml +hostPath: + path: /opt/rke/var/lib/kubelet/pods +``` + + +**OpenShift** + + +To mount the correct hostpath to pods volumes, run the Restic pod in `privileged` mode. + +1. Add the `velero` ServiceAccount to the `privileged` SCC: + + ``` + $ oc adm policy add-scc-to-user privileged -z velero -n velero + ``` + +2. For OpenShift version >= `4.1`, modify the DaemonSet yaml to request a privileged mode: + + ```diff + @@ -67,3 +67,5 @@ spec: + value: /credentials/cloud + - name: VELERO_SCRATCH_DIR + value: /scratch + + securityContext: + + privileged: true + ``` + + or + + ```shell + oc patch ds/restic \ + --namespace velero \ + --type json \ + -p '[{"op":"add","path":"/spec/template/spec/containers/0/securityContext","value": { "privileged": true}}]' + ``` + +3. For OpenShift version < `4.1`, modify the DaemonSet yaml to request a privileged mode and mount the correct hostpath to pods volumes. + + ```diff + @@ -35,7 +35,7 @@ spec: + secretName: cloud-credentials + - name: host-pods + hostPath: + - path: /var/lib/kubelet/pods + + path: /var/lib/origin/openshift.local.volumes/pods + - name: scratch + emptyDir: {} + containers: + @@ -67,3 +67,5 @@ spec: + value: /credentials/cloud + - name: VELERO_SCRATCH_DIR + value: /scratch + + securityContext: + + privileged: true + ``` + + or + + ```shell + oc patch ds/restic \ + --namespace velero \ + --type json \ + -p '[{"op":"add","path":"/spec/template/spec/containers/0/securityContext","value": { "privileged": true}}]' + + oc patch ds/restic \ + --namespace velero \ + --type json \ + -p '[{"op":"replace","path":"/spec/template/spec/volumes/0/hostPath","value": { "path": "/var/lib/origin/openshift.local.volumes/pods"}}]' + ``` + + +If Restic is not running in a privileged mode, it will not be able to access pods volumes within the mounted hostpath directory because of the default enforced SELinux mode configured in the host system level. You can [create a custom SCC](https://docs.openshift.com/container-platform/3.11/admin_guide/manage_scc.html) to relax the security in your cluster so that Restic pods are allowed to use the hostPath volume plug-in without granting them access to the `privileged` SCC. + +By default a userland openshift namespace will not schedule pods on all nodes in the cluster. + +To schedule on all nodes the namespace needs an annotation: + +``` +oc annotate namespace openshift.io/node-selector="" +``` + +This should be done before velero installation. + +Or the ds needs to be deleted and recreated: + +``` +oc get ds restic -o yaml -n > ds.yaml +oc annotate namespace openshift.io/node-selector="" +oc create -n -f ds.yaml +``` + +**VMware Tanzu Kubernetes Grid Integrated Edition (formerly VMware Enterprise PKS)** + +You need to enable the `Allow Privileged` option in your plan configuration so that Restic is able to mount the hostpath. + +The hostPath should be changed from `/var/lib/kubelet/pods` to `/var/vcap/data/kubelet/pods` + +```yaml +hostPath: + path: /var/vcap/data/kubelet/pods +``` + + +**Microsoft Azure** + +If you are using [Azure Files][8], you need to add `nouser_xattr` to your storage class's `mountOptions`. See [this restic issue][9] for more details. + +You can use the following command to patch the storage class: + +```bash +kubectl patch storageclass/ \ + --type json \ + --patch '[{"op":"add","path":"/mountOptions/-","value":"nouser_xattr"}]' +``` + + +## To back up + +Velero supports two approaches of discovering pod volumes that need to be backed up using Restic: + +- Opt-in approach: Where every pod containing a volume to be backed up using Restic must be annotated with the volume's name. +- Opt-out approach: Where all pod volumes are backed up using Restic, with the ability to opt-out any volumes that should not be backed up. + +The following sections provide more details on the two approaches. + +### Using the opt-out approach + +In this approach, Velero will back up all pod volumes using Restic with the exception of: + +- Volumes mounting the default service account token, kubernetes secrets, and config maps +- Hostpath volumes + +It is possible to exclude volumes from being backed up using the `backup.velero.io/backup-volumes-excludes` annotation on the pod. + +Instructions to back up using this approach are as follows: + +1. Run the following command on each pod that contains volumes that should **not** be backed up using Restic + + ```bash + kubectl -n YOUR_POD_NAMESPACE annotate pod/YOUR_POD_NAME backup.velero.io/backup-volumes-excludes=YOUR_VOLUME_NAME_1,YOUR_VOLUME_NAME_2,... + ``` + where the volume names are the names of the volumes in the pod spec. + + For example, in the following pod: + + ```yaml + apiVersion: v1 + kind: Pod + metadata: + name: app1 + namespace: sample + spec: + containers: + - image: k8s.gcr.io/test-webserver + name: test-webserver + volumeMounts: + - name: pvc1-vm + mountPath: /volume-1 + - name: pvc2-vm + mountPath: /volume-2 + volumes: + - name: pvc1-vm + persistentVolumeClaim: + claimName: pvc1 + - name: pvc2-vm + claimName: pvc2 + ``` + to exclude Restic backup of volume `pvc1-vm`, you would run: + + ```bash + kubectl -n sample annotate pod/app1 backup.velero.io/backup-volumes-excludes=pvc1-vm + ``` + +2. Take a Velero backup: + + ```bash + velero backup create BACKUP_NAME --default-volumes-to-restic OTHER_OPTIONS + ``` + + The above steps uses the opt-out approach on a per backup basis. + + Alternatively, this behavior may be enabled on all velero backups running the `velero install` command with the `--default-volumes-to-restic` flag. Refer [install overview][11] for details. + +3. When the backup completes, view information about the backups: + + ```bash + velero backup describe YOUR_BACKUP_NAME + ``` + ```bash + kubectl -n velero get podvolumebackups -l velero.io/backup-name=YOUR_BACKUP_NAME -o yaml + ``` + +### Using opt-in pod volume backup + +Velero, by default, uses this approach to discover pod volumes that need to be backed up using Restic, where every pod containing a volume to be backed up using Restic must be annotated with the volume's name. + +Instructions to back up using this approach are as follows: + +1. Run the following for each pod that contains a volume to back up: + + ```bash + kubectl -n YOUR_POD_NAMESPACE annotate pod/YOUR_POD_NAME backup.velero.io/backup-volumes=YOUR_VOLUME_NAME_1,YOUR_VOLUME_NAME_2,... + ``` + + where the volume names are the names of the volumes in the pod spec. + + For example, for the following pod: + + ```yaml + apiVersion: v1 + kind: Pod + metadata: + name: sample + namespace: foo + spec: + containers: + - image: k8s.gcr.io/test-webserver + name: test-webserver + volumeMounts: + - name: pvc-volume + mountPath: /volume-1 + - name: emptydir-volume + mountPath: /volume-2 + volumes: + - name: pvc-volume + persistentVolumeClaim: + claimName: test-volume-claim + - name: emptydir-volume + emptyDir: {} + ``` + + You'd run: + + ```bash + kubectl -n foo annotate pod/sample backup.velero.io/backup-volumes=pvc-volume,emptydir-volume + ``` + + This annotation can also be provided in a pod template spec if you use a controller to manage your pods. + +1. Take a Velero backup: + + ```bash + velero backup create NAME OPTIONS... + ``` + +1. When the backup completes, view information about the backups: + + ```bash + velero backup describe YOUR_BACKUP_NAME + ``` + ```bash + kubectl -n velero get podvolumebackups -l velero.io/backup-name=YOUR_BACKUP_NAME -o yaml + ``` + +## To restore + +Regardless of how volumes are discovered for backup using Restic, the process of restoring remains the same. + +1. Restore from your Velero backup: + + ```bash + velero restore create --from-backup BACKUP_NAME OPTIONS... + ``` + +1. When the restore completes, view information about your pod volume restores: + + ```bash + velero restore describe YOUR_RESTORE_NAME + ``` + ```bash + kubectl -n velero get podvolumerestores -l velero.io/restore-name=YOUR_RESTORE_NAME -o yaml + ``` + +## Limitations + +- `hostPath` volumes are not supported. [Local persistent volumes][4] are supported. +- Those of you familiar with [restic][1] may know that it encrypts all of its data. Velero uses a static, +common encryption key for all Restic repositories it creates. **This means that anyone who has access to your +bucket can decrypt your Restic backup data**. Make sure that you limit access to the Restic bucket +appropriately. +- An incremental backup chain will be maintained across pod reschedules for PVCs. However, for pod volumes that are *not* +PVCs, such as `emptyDir` volumes, when a pod is deleted/recreated (for example, by a ReplicaSet/Deployment), the next backup of those +volumes will be full rather than incremental, because the pod volume's lifecycle is assumed to be defined by its pod. +- Restic scans each file in a single thread. This means that large files (such as ones storing a database) will take a long time to scan for data deduplication, even if the actual +difference is small. +- If you plan to use Velero's Restic integration to backup 100GB of data or more, you may need to [customize the resource limits](/docs/main/customize-installation/#customize-resource-requests-and-limits) to make sure backups complete successfully. +- Velero's Restic integration backs up data from volumes by accessing the node's filesystem, on which the pod is running. For this reason, Velero's Restic integration can only backup volumes that are mounted by a pod and not directly from the PVC. For orphan PVC/PV pairs (without running pods), some Velero users overcame this limitation running a staging pod (i.e. a busybox or alpine container with an infinite sleep) to mount these PVC/PV pairs prior taking a Velero backup. + +## Customize Restore Helper Container + +Velero uses a helper init container when performing a Restic restore. By default, the image for this container is `velero/velero-restic-restore-helper:`, +where `VERSION` matches the version/tag of the main Velero image. You can customize the image that is used for this helper by creating a ConfigMap in the Velero namespace with +the alternate image. + +In addition, you can customize the resource requirements for the init container, should you need. + +The ConfigMap must look like the following: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + # any name can be used; Velero uses the labels (below) + # to identify it rather than the name + name: restic-restore-action-config + # must be in the velero namespace + namespace: velero + # the below labels should be used verbatim in your + # ConfigMap. + labels: + # this value-less label identifies the ConfigMap as + # config for a plugin (i.e. the built-in restic restore + # item action plugin) + velero.io/plugin-config: "" + # this label identifies the name and kind of plugin + # that this ConfigMap is for. + velero.io/restic: RestoreItemAction +data: + # The value for "image" can either include a tag or not; + # if the tag is *not* included, the tag from the main Velero + # image will automatically be used. + image: myregistry.io/my-custom-helper-image[:OPTIONAL_TAG] + + # "cpuRequest" sets the request.cpu value on the restic init containers during restore. + # If not set, it will default to "100m". A value of "0" is treated as unbounded. + cpuRequest: 200m + + # "memRequest" sets the request.memory value on the restic init containers during restore. + # If not set, it will default to "128Mi". A value of "0" is treated as unbounded. + memRequest: 128Mi + + # "cpuLimit" sets the request.cpu value on the restic init containers during restore. + # If not set, it will default to "100m". A value of "0" is treated as unbounded. + cpuLimit: 200m + + # "memLimit" sets the request.memory value on the restic init containers during restore. + # If not set, it will default to "128Mi". A value of "0" is treated as unbounded. + memLimit: 128Mi + + # "secCtxRunAsUser sets the securityContext.runAsUser value on the restic init containers during restore." + secCtxRunAsUser: 1001 + + # "secCtxRunAsGroup sets the securityContext.runAsGroup value on the restic init containers during restore." + secCtxRunAsGroup: 999 +``` + +## Troubleshooting + +Run the following checks: + +Are your Velero server and daemonset pods running? + +```bash +kubectl get pods -n velero +``` + +Does your Restic repository exist, and is it ready? + +```bash +velero restic repo get + +velero restic repo get REPO_NAME -o yaml +``` + +Are there any errors in your Velero backup/restore? + +```bash +velero backup describe BACKUP_NAME +velero backup logs BACKUP_NAME + +velero restore describe RESTORE_NAME +velero restore logs RESTORE_NAME +``` + +What is the status of your pod volume backups/restores? + +```bash +kubectl -n velero get podvolumebackups -l velero.io/backup-name=BACKUP_NAME -o yaml + +kubectl -n velero get podvolumerestores -l velero.io/restore-name=RESTORE_NAME -o yaml +``` + +Is there any useful information in the Velero server or daemon pod logs? + +```bash +kubectl -n velero logs deploy/velero +kubectl -n velero logs DAEMON_POD_NAME +``` + +**NOTE**: You can increase the verbosity of the pod logs by adding `--log-level=debug` as an argument +to the container command in the deployment/daemonset pod template spec. + +## How backup and restore work with Restic + +Velero has three custom resource definitions and associated controllers: + +- `ResticRepository` - represents/manages the lifecycle of Velero's [restic repositories][5]. Velero creates +a Restic repository per namespace when the first Restic backup for a namespace is requested. The controller +for this custom resource executes Restic repository lifecycle commands -- `restic init`, `restic check`, +and `restic prune`. + + You can see information about your Velero's Restic repositories by running `velero restic repo get`. + +- `PodVolumeBackup` - represents a Restic backup of a volume in a pod. The main Velero backup process creates +one or more of these when it finds an annotated pod. Each node in the cluster runs a controller for this +resource (in a daemonset) that handles the `PodVolumeBackups` for pods on that node. The controller executes +`restic backup` commands to backup pod volume data. + +- `PodVolumeRestore` - represents a Restic restore of a pod volume. The main Velero restore process creates one +or more of these when it encounters a pod that has associated Restic backups. Each node in the cluster runs a +controller for this resource (in the same daemonset as above) that handles the `PodVolumeRestores` for pods +on that node. The controller executes `restic restore` commands to restore pod volume data. + +### Backup + +1. Based on configuration, the main Velero backup process uses the opt-in or opt-out approach to check each pod that it's backing up for the volumes to be backed up using Restic. +1. When found, Velero first ensures a Restic repository exists for the pod's namespace, by: + - checking if a `ResticRepository` custom resource already exists + - if not, creating a new one, and waiting for the `ResticRepository` controller to init/check it +1. Velero then creates a `PodVolumeBackup` custom resource per volume listed in the pod annotation +1. The main Velero process now waits for the `PodVolumeBackup` resources to complete or fail +1. Meanwhile, each `PodVolumeBackup` is handled by the controller on the appropriate node, which: + - has a hostPath volume mount of `/var/lib/kubelet/pods` to access the pod volume data + - finds the pod volume's subdirectory within the above volume + - runs `restic backup` + - updates the status of the custom resource to `Completed` or `Failed` +1. As each `PodVolumeBackup` finishes, the main Velero process adds it to the Velero backup in a file named `-podvolumebackups.json.gz`. This file gets uploaded to object storage alongside the backup tarball. It will be used for restores, as seen in the next section. + +### Restore + +1. The main Velero restore process checks each existing `PodVolumeBackup` custom resource in the cluster to backup from. +1. For each `PodVolumeBackup` found, Velero first ensures a Restic repository exists for the pod's namespace, by: + - checking if a `ResticRepository` custom resource already exists + - if not, creating a new one, and waiting for the `ResticRepository` controller to init/check it (note that + in this case, the actual repository should already exist in object storage, so the Velero controller will simply + check it for integrity) +1. Velero adds an init container to the pod, whose job is to wait for all Restic restores for the pod to complete (more +on this shortly) +1. Velero creates the pod, with the added init container, by submitting it to the Kubernetes API. Then, the Kubernetes scheduler schedules this pod to a worker node, and the pod must be in a running state. If the pod fails to start for some reason (i.e. lack of cluster resources), the Restic restore will not be done. +1. Velero creates a `PodVolumeRestore` custom resource for each volume to be restored in the pod +1. The main Velero process now waits for each `PodVolumeRestore` resource to complete or fail +1. Meanwhile, each `PodVolumeRestore` is handled by the controller on the appropriate node, which: + - has a hostPath volume mount of `/var/lib/kubelet/pods` to access the pod volume data + - waits for the pod to be running the init container + - finds the pod volume's subdirectory within the above volume + - runs `restic restore` + - on success, writes a file into the pod volume, in a `.velero` subdirectory, whose name is the UID of the Velero restore + that this pod volume restore is for + - updates the status of the custom resource to `Completed` or `Failed` +1. The init container that was added to the pod is running a process that waits until it finds a file +within each restored volume, under `.velero`, whose name is the UID of the Velero restore being run +1. Once all such files are found, the init container's process terminates successfully and the pod moves +on to running other init containers/the main containers. + +## 3rd party controllers + +### Monitor backup annotation + +Velero does not provide a mechanism to detect persistent volume claims that are missing the Restic backup annotation. + +To solve this, a controller was written by Thomann Bits&Beats: [velero-pvc-watcher][7] + +[1]: https://github.com/restic/restic +[2]: customize-installation.md#enable-restic-integration +[3]: https://github.com/vmware-tanzu/velero/releases/ +[4]: https://kubernetes.io/docs/concepts/storage/volumes/#local +[5]: http://restic.readthedocs.io/en/latest/100_references.html#terminology +[6]: https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation +[7]: https://github.com/bitsbeats/velero-pvc-watcher +[8]: https://docs.microsoft.com/en-us/azure/aks/azure-files-dynamic-pv +[9]: https://github.com/restic/restic/issues/1800 +[11]: customize-installation.md#default-pod-volume-backup-to-restic diff --git a/site/content/docs/v1.8/restore-hooks.md b/site/content/docs/v1.8/restore-hooks.md new file mode 100644 index 000000000..0ec36c5d7 --- /dev/null +++ b/site/content/docs/v1.8/restore-hooks.md @@ -0,0 +1,261 @@ +--- +title: "Restore Hooks" +layout: docs +--- + +Velero supports Restore Hooks, custom actions that can be executed during or after the restore process. There are two kinds of Restore Hooks: + +1. InitContainer Restore Hooks: These will add init containers into restored pods to perform any necessary setup before the application containers of the restored pod can start. +1. Exec Restore Hooks: These can be used to execute custom commands or scripts in containers of a restored Kubernetes pod. + +## InitContainer Restore Hooks + +Use an `InitContainer` hook to add init containers into a pod before it's restored. You can use these init containers to run any setup needed for the pod to resume running from its backed-up state. +The InitContainer added by the restore hook will be the first init container in the `podSpec` of the restored pod. +In the case where the pod had volumes backed up using restic, then, the restore hook InitContainer will be added after the `restic-wait` InitContainer. + +NOTE: This ordering can be altered by any mutating webhooks that may be installed in the cluster. + +There are two ways to specify `InitContainer` restore hooks: +1. Specifying restore hooks in annotations +1. Specifying restore hooks in the restore spec + +### Specifying Restore Hooks As Pod Annotations + +Below are the annotations that can be added to a pod to specify restore hooks: +* `init.hook.restore.velero.io/container-image` + * The container image for the init container to be added. +* `init.hook.restore.velero.io/container-name` + * The name for the init container that is being added. +* `init.hook.restore.velero.io/command` + * This is the `ENTRYPOINT` for the init container being added. This command is not executed within a shell and the container image's `ENTRYPOINT` is used if this is not provided. + +#### Example + +Use the below commands to add annotations to the pods before taking a backup. + +```bash +$ kubectl annotate pod -n \ + init.hook.restore.velero.io/container-name=restore-hook \ + init.hook.restore.velero.io/container-image=alpine:latest \ + init.hook.restore.velero.io/command='["/bin/ash", "-c", "date"]' +``` + +With the annotation above, Velero will add the following init container to the pod when it's restored. + +```json +{ + "command": [ + "/bin/ash", + "-c", + "date" + ], + "image": "alpine:latest", + "imagePullPolicy": "Always", + "name": "restore-hook" + ... +} +``` + +### Specifying Restore Hooks In Restore Spec + +Init container restore hooks can also be specified using the `RestoreSpec`. +Please refer to the documentation on the [Restore API Type][1] for how to specify hooks in the Restore spec. + +#### Example + +Below is an example of specifying restore hooks in `RestoreSpec` + +```yaml +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: r2 + namespace: velero +spec: + backupName: b2 + excludedResources: + ... + includedNamespaces: + - '*' + hooks: + resources: + - name: restore-hook-1 + includedNamespaces: + - app + postHooks: + - init: + initContainers: + - name: restore-hook-init1 + image: alpine:latest + volumeMounts: + - mountPath: /restores/pvc1-vm + name: pvc1-vm + command: + - /bin/ash + - -c + - echo -n "FOOBARBAZ" >> /restores/pvc1-vm/foobarbaz + - name: restore-hook-init2 + image: alpine:latest + volumeMounts: + - mountPath: /restores/pvc2-vm + name: pvc2-vm + command: + - /bin/ash + - -c + - echo -n "DEADFEED" >> /restores/pvc2-vm/deadfeed +``` + +The `hooks` in the above `RestoreSpec`, when restored, will add two init containers to every pod in the `app` namespace + +```json +{ + "command": [ + "/bin/ash", + "-c", + "echo -n \"FOOBARBAZ\" >> /restores/pvc1-vm/foobarbaz" + ], + "image": "alpine:latest", + "imagePullPolicy": "Always", + "name": "restore-hook-init1", + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/restores/pvc1-vm", + "name": "pvc1-vm" + } + ] + ... +} +``` + +and + +```json +{ + "command": [ + "/bin/ash", + "-c", + "echo -n \"DEADFEED\" >> /restores/pvc2-vm/deadfeed" + ], + "image": "alpine:latest", + "imagePullPolicy": "Always", + "name": "restore-hook-init2", + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/restores/pvc2-vm", + "name": "pvc2-vm" + } + ] + ... +} +``` + +## Exec Restore Hooks + +Use an Exec Restore hook to execute commands in a restored pod's containers after they start. + +There are two ways to specify `Exec` restore hooks: +1. Specifying exec restore hooks in annotations +1. Specifying exec restore hooks in the restore spec + +If a pod has the annotation `post.hook.restore.velero.io/command` then that is the only hook that will be executed in the pod. +No hooks from the restore spec will be executed in that pod. + +### Specifying Exec Restore Hooks As Pod Annotations + +Below are the annotations that can be added to a pod to specify exec restore hooks: +* `post.hook.restore.velero.io/container` + * The container name where the hook will be executed. Defaults to the first container. Optional. +* `post.hook.restore.velero.io/command` + * The command that will be executed in the container. Required. +* `post.hook.restore.velero.io/on-error` + * How to handle execution failures. Valid values are `Fail` and `Continue`. Defaults to `Continue`. With `Continue` mode, execution failures are logged only. With `Fail` mode, no more restore hooks will be executed in any container in any pod and the status of the Restore will be `PartiallyFailed`. Optional. +* `post.hook.restore.velero.io/exec-timeout` + * How long to wait once execution begins. Defaults to 30 seconds. Optional. +* `post.hook.restore.velero.io/wait-timeout` + * How long to wait for a container to become ready. This should be long enough for the container to start plus any preceding hooks in the same container to complete. The wait timeout begins when the container is restored and may require time for the image to pull and volumes to mount. If not set the restore will wait indefinitely. Optional. + +#### Example + +Use the below commands to add annotations to the pods before taking a backup. + +```bash +$ kubectl annotate pod -n \ + post.hook.restore.velero.io/container=postgres \ + post.hook.restore.velero.io/command='["/bin/bash", "-c", "psql < /backup/backup.sql"]' \ + post.hook.restore.velero.io/wait-timeout=5m \ + post.hook.restore.velero.io/exec-timeout=45s \ + post.hook.restore.velero.io/on-error=Continue +``` + +### Specifying Exec Restore Hooks in Restore Spec + +Exec restore hooks can also be specified using the `RestoreSpec`. +Please refer to the documentation on the [Restore API Type][1] for how to specify hooks in the Restore spec. + +#### Multiple Exec Restore Hooks Example + +Below is an example of specifying restore hooks in a `RestoreSpec`. +When using the restore spec it is possible to specify multiple hooks for a single pod, as this example demonstrates. + +All hooks applicable to a single container will be executed sequentially in that container once it starts. +The ordering of hooks executed in a single container follows the order of the restore spec. +In this example, the `pg_isready` hook is guaranteed to run before the `psql` hook because they both apply to the same container and the `pg_isready` hook is defined first. + +If a pod has multiple containers with applicable hooks, all hooks for a single container will be executed before executing hooks in another container. +In this example, if the postgres container starts before the sidecar container, both postgres hooks will run before the hook in the sidecar. +This means the sidecar container may be running for several minutes before its hook is executed. + +Velero guarantees that no two hooks for a single pod are executed in parallel, but hooks executing in different pods may run in parallel. + + +```yaml +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: r2 + namespace: velero +spec: + backupName: b2 + excludedResources: + ... + includedNamespaces: + - '*' + hooks: + resources: + - name: restore-hook-1 + includedNamespaces: + - app + postHooks: + - exec: + execTimeout: 1m + waitTimeout: 5m + onError: Fail + container: postgres + command: + - /bin/bash + - '-c' + - 'while ! pg_isready; do sleep 1; done' + - exec: + container: postgres + waitTimeout: 6m + execTimeout: 1m + command: + - /bin/bash + - '-c' + - 'psql < /backup/backup.sql' + - exec: + container: sidecar + command: + - /bin/bash + - '-c' + - 'date > /start' +``` + +[1]: api-types/restore.md diff --git a/site/content/docs/v1.8/restore-reference.md b/site/content/docs/v1.8/restore-reference.md new file mode 100644 index 000000000..bb14977bc --- /dev/null +++ b/site/content/docs/v1.8/restore-reference.md @@ -0,0 +1,144 @@ +--- +title: "Restore Reference" +layout: docs +--- + +## Restoring Into a Different Namespace + +Velero can restore resources into a different namespace than the one they were backed up from. To do this, use the `--namespace-mappings` flag: + +```bash +velero restore create RESTORE_NAME \ + --from-backup BACKUP_NAME \ + --namespace-mappings old-ns-1:new-ns-1,old-ns-2:new-ns-2 +``` +## What happens when user removes restore objects +A **restore** object represents the restore operation. There are two types of deletion for restore objects: +1. Deleting with **`velero restore delete`**. +This command will delete the custom resource representing it, along with its individual log and results files. But, it will not delete any objects that were created by it from your cluster. +2. Deleting with **`kubectl -n velero delete restore`**. +This command will delete the custom resource representing the restore, but will not delete log/results files from object storage, or any objects that were created during the restore in your cluster. + +## Restore command-line options +To see all commands for restores, run : `velero restore --help` +To see all options associated with a specific command, provide the --help flag to that command. For example, **`velero restore create --help`** shows all options associated with the **create** command. + +To list all options of restore, use **`velero restore --help`** + +```Usage: + velero restore [command] + +Available Commands: + create Create a restore + delete Delete restores + describe Describe restores + get Get restores + logs Get restore logs +``` + +## What happens to NodePorts when restoring Services + +**Auto assigned** NodePorts **deleted** by default and Services get new **auto assigned** nodePorts after restore. + +**Explicitly specified** NodePorts auto detected using **`last-applied-config`** annotation and **preserved** after restore. NodePorts can be explicitly specified as .spec.ports[*].nodePort field on Service definition. + +#### Always Preserve NodePorts + +It is not always possible to set nodePorts explicitly on some big clusters because of operation complexity. Official Kubernetes documents states that preventing port collisions is responsibility of the user when explicitly specifying nodePorts: + +``` +If you want a specific port number, you can specify a value in the `nodePort` field. The control plane will either allocate you that port or report that the API transaction failed. This means that you need to take care of possible port collisions yourself. You also have to use a valid port number, one that's inside the range configured for NodePort use. + +https://kubernetes.io/docs/concepts/services-networking/service/#nodeport +``` + +The clusters which are not explicitly specifying nodePorts still may need to restore original NodePorts in case of disaster. Auto assigned nodePorts most probably defined on Load Balancers which located front side of cluster. Changing all these nodePorts on Load Balancers is another operation complexity after disaster if nodePorts are changed. + +Velero has a flag to let user deciding the preservation of nodePorts. **`velero restore create`** sub command has **`--preserve-nodeports`** flag to **preserve** Service nodePorts **always** regardless of nodePorts **explicitly specified** or **not**. This flag used for preserving the original nodePorts from backup and can be used as **`--preserve-nodeports`** or **`--preserve-nodeports=true`** + +If this flag given and/or set to true, Velero does not remove the nodePorts when restoring Service and tries to use the nodePorts which written on backup. + +Trying to preserve nodePorts may cause **port conflicts** when restoring on situations below: + +- If the nodePort from the backup already allocated on the target cluster then Velero prints error log as shown below and continue to restore operation. + + ``` + time="2020-11-23T12:58:31+03:00" level=info msg="Executing item action for services" logSource="pkg/restore/restore.go:1002" restore=velero/test-with-3-svc-20201123125825 + + time="2020-11-23T12:58:31+03:00" level=info msg="Restoring Services with original NodePort(s)" cmd=_output/bin/linux/amd64/velero logSource="pkg/restore/service_action.go:61" pluginName=velero restore=velero/test-with-3-svc-20201123125825 + + time="2020-11-23T12:58:31+03:00" level=info msg="Attempting to restore Service: hello-service" logSource="pkg/restore/restore.go:1107" restore=velero/test-with-3-svc-20201123125825 + + time="2020-11-23T12:58:31+03:00" level=error msg="error restoring hello-service: Service \"hello-service\" is invalid: spec.ports[0].nodePort: Invalid value: 31536: provided port is already allocated" logSource="pkg/restore/restore.go:1170" restore=velero/test-with-3-svc-20201123125825 + ``` + + + +- If the nodePort from the backup is not in the nodePort range of target cluster then Velero prints error log as below and continue to restore operation. Kubernetes default nodePort range is 30000-32767 but on the example cluster nodePort range is 20000-22767 and tried to restore Service with nodePort 31536 + + ``` + time="2020-11-23T13:09:17+03:00" level=info msg="Executing item action for services" logSource="pkg/restore/restore.go:1002" restore=velero/test-with-3-svc-20201123130915 + + time="2020-11-23T13:09:17+03:00" level=info msg="Restoring Services with original NodePort(s)" cmd=_output/bin/linux/amd64/velero logSource="pkg/restore/service_action.go:61" pluginName=velero restore=velero/test-with-3-svc-20201123130915 + + time="2020-11-23T13:09:17+03:00" level=info msg="Attempting to restore Service: hello-service" logSource="pkg/restore/restore.go:1107" restore=velero/test-with-3-svc-20201123130915 + + time="2020-11-23T13:09:17+03:00" level=error msg="error restoring hello-service: Service \"hello-service\" is invalid: spec.ports[0].nodePort: Invalid value: 31536: provided port is not in the valid range. The range of valid ports is 20000-22767" logSource="pkg/restore/restore.go:1170" restore=velero/test-with-3-svc-20201123130915 + ``` + +## Changing PV/PVC Storage Classes + +Velero can change the storage class of persistent volumes and persistent volume claims during restores. To configure a storage class mapping, create a config map in the Velero namespace like the following: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + # any name can be used; Velero uses the labels (below) + # to identify it rather than the name + name: change-storage-class-config + # must be in the velero namespace + namespace: velero + # the below labels should be used verbatim in your + # ConfigMap. + labels: + # this value-less label identifies the ConfigMap as + # config for a plugin (i.e. the built-in restore item action plugin) + velero.io/plugin-config: "" + # this label identifies the name and kind of plugin + # that this ConfigMap is for. + velero.io/change-storage-class: RestoreItemAction +data: + # add 1+ key-value pairs here, where the key is the old + # storage class name and the value is the new storage + # class name. + : +``` + +## Changing PVC selected-node + +Velero can update the selected-node annotation of persistent volume claim during restores, if selected-node doesn't exist in the cluster then it will remove the selected-node annotation from PersistentVolumeClaim. To configure a node mapping, create a config map in the Velero namespace like the following: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + # any name can be used; Velero uses the labels (below) + # to identify it rather than the name + name: change-pvc-node-selector-config + # must be in the velero namespace + namespace: velero + # the below labels should be used verbatim in your + # ConfigMap. + labels: + # this value-less label identifies the ConfigMap as + # config for a plugin (i.e. the built-in restore item action plugin) + velero.io/plugin-config: "" + # this label identifies the name and kind of plugin + # that this ConfigMap is for. + velero.io/change-pvc-node-selector: RestoreItemAction +data: + # add 1+ key-value pairs here, where the key is the old + # node name and the value is the new node name. + : +``` diff --git a/site/content/docs/v1.8/run-locally.md b/site/content/docs/v1.8/run-locally.md new file mode 100644 index 000000000..897ee1893 --- /dev/null +++ b/site/content/docs/v1.8/run-locally.md @@ -0,0 +1,53 @@ +--- +title: "Run Velero locally in development" +layout: docs +--- + +Running the Velero server locally can speed up iterative development. This eliminates the need to rebuild the Velero server +image and redeploy it to the cluster with each change. + +## Run Velero locally with a remote cluster + +Velero runs against the Kubernetes API server as the endpoint (as per the `kubeconfig` configuration), so both the Velero server and client use the same `client-go` to communicate with Kubernetes. This means the Velero server can be run locally just as functionally as if it was running in the remote cluster. + +### Prerequisites + +When running Velero, you will need to ensure that you set up all of the following: + +* Appropriate RBAC permissions in the cluster + * Read access for all data from the source cluster and namespaces + * Write access to the target cluster and namespaces +* Cloud provider credentials + * Read/write access to volumes + * Read/write access to object storage for backup data +* A [BackupStorageLocation][20] object definition for the Velero server +* (Optional) A [VolumeSnapshotLocation][21] object definition for the Velero server, to take PV snapshots + +### 1. Install Velero + +See documentation on how to install Velero in some specific providers: [Install overview][22] + +### 2. Scale deployment down to zero + +After you use the `velero install` command to install Velero into your cluster, you scale the Velero deployment down to 0 so it is not simultaneously being run on the remote cluster and potentially causing things to get out of sync: + +`kubectl scale --replicas=0 deployment velero -n velero` + +#### 3. Start the Velero server locally + +* To run the server locally, use the full path according to the binary you need. Example, if you are on a Mac, and using `AWS` as a provider, this is how to run the binary you built from source using the full path: `AWS_SHARED_CREDENTIALS_FILE= ./_output/bin/darwin/amd64/velero`. Alternatively, you may add the `velero` binary to your `PATH`. + +* Start the server: `velero server [CLI flags]`. The following CLI flags may be useful to customize, but see `velero server --help` for full details: + * `--log-level`: set the Velero server's log level (default `info`, use `debug` for the most logging) + * `--kubeconfig`: set the path to the kubeconfig file the Velero server uses to talk to the Kubernetes apiserver (default `$KUBECONFIG`) + * `--namespace`: the set namespace where the Velero server should look for backups, schedules, restores (default `velero`) + * `--plugin-dir`: set the directory where the Velero server looks for plugins (default `/plugins`) + * The `--plugin-dir` flag requires the plugin binary to be present locally, and should be set to the directory containing this built binary. + * `--metrics-address`: set the bind address and port where Prometheus metrics are exposed (default `:8085`) + +[15]: https://docs.aws.amazon.com/cli/latest/topic/config-vars.html#the-shared-credentials-file +[16]: https://cloud.google.com/docs/authentication/getting-started#setting_the_environment_variable +[18]: https://eksctl.io/ +[20]: api-types/backupstoragelocation.md +[21]: api-types/volumesnapshotlocation.md +[22]: basic-install.md diff --git a/site/content/docs/v1.8/self-signed-certificates.md b/site/content/docs/v1.8/self-signed-certificates.md new file mode 100644 index 000000000..0bb55be8c --- /dev/null +++ b/site/content/docs/v1.8/self-signed-certificates.md @@ -0,0 +1,48 @@ +--- +title: "Use Velero with a storage provider secured by a self-signed certificate" +layout: docs +--- + +If you are using an S3-Compatible storage provider that is secured with a self-signed certificate, connections to the object store may fail with a `certificate signed by unknown authority` message. +To proceed, provide a certificate bundle when adding the storage provider. + +## Trusting a self-signed certificate during installation + +When using the `velero install` command, you can use the `--cacert` flag to provide a path +to a PEM-encoded certificate bundle to trust. + +```bash +velero install \ + --plugins + --provider \ + --bucket \ + --secret-file \ + --cacert +``` + +Velero will then automatically use the provided CA bundle to verify TLS connections to +that storage provider when backing up and restoring. + +## Trusting a self-signed certificate with the Velero client + +To use the describe, download, or logs commands to access a backup or restore contained +in storage secured by a self-signed certificate as in the above example, you must use +the `--cacert` flag to provide a path to the certificate to be trusted. + +```bash +velero backup describe my-backup --cacert +``` + +## Error with client certificate with custom S3 server + +In case you are using a custom S3-compatible server, you may encounter that the backup fails with an error similar to one below. + +``` +rpc error: code = Unknown desc = RequestError: send request failed caused by: +Get https://minio.com:3000/k8s-backup-bucket?delimiter=%2F&list-type=2&prefix=: remote error: tls: alert(116) +``` + +Error 116 represents certificate required as seen here in [error codes](https://datatracker.ietf.org/doc/html/rfc8446#appendix-B.2). +Velero as a client does not include its certificate while performing SSL handshake with the server. +From [TLS 1.3 spec](https://tools.ietf.org/html/rfc8446), verifying client certificate is optional on the server. +You will need to change this setting on the server to make it work. diff --git a/site/content/docs/v1.8/start-contributing.md b/site/content/docs/v1.8/start-contributing.md new file mode 100644 index 000000000..c7a72b1f5 --- /dev/null +++ b/site/content/docs/v1.8/start-contributing.md @@ -0,0 +1,34 @@ +--- +title: "Start contributing" +layout: docs +--- + +## Before you start + +* Please familiarize yourself with the [Code of Conduct][1] before contributing. +* Also, see [CONTRIBUTING.md][2] for instructions on the developer certificate of origin that we require. + +## Creating a design doc + +Having a high level design document with the proposed change and the impacts helps the maintainers evaluate if a major change should be incorporated. + +To make a design pull request, you can copy the template found in the `design/_template.md` file into a new Markdown file. + +## Finding your way around + +You may join the Velero community and contribute in many different ways, including helping us design or test new features. For any significant feature we consider adding, we start with a design document. You may find a list of in progress new designs here: https://github.com/vmware-tanzu/velero/pulls?q=is%3Aopen+is%3Apr+label%3ADesign. Feel free to review and help us with your input. + +You can also vote on issues using :+1: and :-1:, as explained in our [Feature enhancement request][3] and [Bug issue][4] templates. This will help us quantify importance and prioritize issues. + +For information on how to connect with our maintainers and community, join our online meetings, or find good first issues, start on our [Velero community](https://velero.io/community/) page. + +Please browse our list of resources, including a playlist of past online community meetings, blog posts, and other resources to help you get familiar with our project: [Velero resources](https://velero.io/resources/). + +## Contributing + +If you are ready to jump in and test, add code, or help with documentation, please use the navigation on the left under `Contribute`. + +[1]: https://github.com/vmware-tanzu/velero/blob/v1.8.0/CODE_OF_CONDUCT.md +[2]: https://github.com/vmware-tanzu/velero/blob/v1.8.0/CONTRIBUTING.md +[3]: https://github.com/vmware-tanzu/velero/blob/v1.8.0/.github/ISSUE_TEMPLATE/feature-enhancement-request.md +[4]: https://github.com/vmware-tanzu/velero/blob/v1.8.0/.github/ISSUE_TEMPLATE/bug_report.md diff --git a/site/content/docs/v1.8/style-guide.md b/site/content/docs/v1.8/style-guide.md new file mode 100644 index 000000000..1b98e59dd --- /dev/null +++ b/site/content/docs/v1.8/style-guide.md @@ -0,0 +1,344 @@ +--- +title: "Documentation Style Guide" +layout: docs +--- + +_This style guide is adapted from the [Kubernetes style guide](https://kubernetes.io/docs/contribute/style/style-guide/)._ + +This page outlines writing style guidelines for the Velero documentation and you should use this page as a reference you write or edit content. Note that these are guidelines, not rules. Use your best judgment as you write documentation, and feel free to propose changes to these guidelines. Changes to the style guide are made by the Velero maintainers as a group. To propose a change or addition create an issue/PR, or add a suggestion to the [community meeting agenda](https://hackmd.io/Jq6F5zqZR7S80CeDWUklkA) and attend the meeting to participate in the discussion. + +The Velero documentation uses the [kramdown](https://kramdown.gettalong.org/) Markdown renderer. + +## Content best practices +### Use present tense + +{{< table caption="Do and Don't - Use present tense" >}} +|Do|Don't| +|--- |--- | +|This `command` starts a proxy.|This command will start a proxy.| +{{< /table >}} + +Exception: Use future or past tense if it is required to convey the correct meaning. + +### Use active voice + +{{< table caption="Do and Don't - Use active voice" >}} +|Do|Don't| +|--- |--- | +|You can explore the API using a browser.|The API can be explored using a browser.| +|The YAML file specifies the replica count.|The replica count is specified in the YAML file.| +{{< /table >}} + +Exception: Use passive voice if active voice leads to an awkward sentence construction. + +### Use simple and direct language + +Use simple and direct language. Avoid using unnecessary phrases, such as saying "please." + +{{< table caption="Do and Don't - Use simple and direct language" >}} +|Do|Don't| +|--- |--- | +|To create a ReplicaSet, ...|In order to create a ReplicaSet, ...| +|See the configuration file.|Please see the configuration file.| +|View the Pods.|With this next command, we'll view the Pods.| +{{< /table >}} + +### Address the reader as "you" + +{{< table caption="Do and Don't - Addressing the reader" >}} +|Do|Don't| +|--- |--- | +|You can create a Deployment by ...|We'll create a Deployment by ...| +|In the preceding output, you can see...|In the preceding output, we can see ...| +{{< /table >}} + +### Avoid Latin phrases + +Prefer English terms over Latin abbreviations. + +{{< table caption="Do and Don't - Avoid Latin phrases" >}} +|Do|Don't| +|--- |--- | +|For example, ...|e.g., ...| +|That is, ...|i.e., ...| +{{< /table >}} + +Exception: Use "etc." for et cetera. + +## Patterns to avoid + + +### Avoid using "we" + +Using "we" in a sentence can be confusing, because the reader might not know +whether they're part of the "we" you're describing. + +{{< table caption="Do and Don't - Avoid using we" >}} +|Do|Don't| +|--- |--- | +|Version 1.4 includes ...|In version 1.4, we have added ...| +|Kubernetes provides a new feature for ...|We provide a new feature ...| +|This page teaches you how to use Pods.|In this page, we are going to learn about Pods.| +{{< /table >}} + +### Avoid jargon and idioms + +Many readers speak English as a second language. Avoid jargon and idioms to help them understand better. + +{{< table caption="Do and Don't - Avoid jargon and idioms" >}} +|Do|Don't| +|--- |--- | +|Internally, ...|Under the hood, ...| +|Create a new cluster.|Turn up a new cluster.| +{{< /table >}} + +### Avoid statements about the future or that will soon be out of date + +Avoid making promises or giving hints about the future. If you need to talk about +a beta feature, put the text under a heading that identifies it as beta +information. + +Also avoid words like “recently”, "currently" and "new." A feature that is new today might not be +considered new in a few months. + +{{< table caption="Do and Don't - Avoid statements that will soon be out of date" >}} +|Do|Don't| +|--- |--- | +|In version 1.4, ...|In the current version, ...| +|The Federation feature provides ...|The new Federation feature provides ...| +{{< /table >}} + +### Language + +This documentation uses U.S. English spelling and grammar. + +## Documentation formatting standards + +### Use camel case for API objects + +When you refer to an API object, use the same uppercase and lowercase letters +that are used in the actual object name. Typically, the names of API +objects use +[camel case](https://en.wikipedia.org/wiki/Camel_case). + +Don't split the API object name into separate words. For example, use +PodTemplateList, not Pod Template List. + +Refer to API objects without saying "object," unless omitting "object" +leads to an awkward sentence construction. + +{{< table caption="Do and Don't - Do and Don't - API objects" >}} +|Do|Don't| +|--- |--- | +|The Pod has two containers.|The pod has two containers.| +|The Deployment is responsible for ...|The Deployment object is responsible for ...| +|A PodList is a list of Pods.|A Pod List is a list of pods.| +|The two ContainerPorts ...|The two ContainerPort objects ...| +|The two ContainerStateTerminated objects ...|The two ContainerStateTerminateds ...| +{{< /table >}} + +### Use angle brackets for placeholders + +Use angle brackets for placeholders. Tell the reader what a placeholder represents. + +1. Display information about a Pod: + + kubectl describe pod -n + + If the pod is in the default namespace, you can omit the '-n' parameter. + +### Use bold for user interface elements + +{{< table caption="Do and Don't - Bold interface elements" >}} +|Do|Don't| +|--- |--- | +|Click **Fork**.|Click "Fork".| +|Select **Other**.|Select "Other".| +{{< /table >}} + +### Use italics to define or introduce new terms + +{{< table caption="Do and Don't - Use italics for new terms" >}} +|Do|Don't| +|--- |--- | +|A _cluster_ is a set of nodes ...|A "cluster" is a set of nodes ...| +|These components form the _control plane_.|These components form the **control plane**.| +{{< /table >}} + +### Use code style for filenames, directories, paths, object field names and namespaces +{{< table caption="Do and Don't - Use code style for filenames, directories, paths, object field names and namespaces" >}} +|Do|Don't| +|--- |--- | +|Open the `envars.yaml` file.|Open the envars.yaml file.| +|Go to the `/docs/tutorials` directory.|Go to the /docs/tutorials directory.| +|Open the `/_data/concepts.yaml` file.|Open the /\_data/concepts.yaml file.| +{{< /table >}} + + +### Use punctuation inside quotes +{{< table caption="Do and Don't - Use code style for filenames, directories, paths, object field names and namespaces" >}} +|Do|Don't| +|--- |--- | +|events are recorded with an associated "stage."|events are recorded with an associated "stage".| +|The copy is called a "fork."|The copy is called a "fork".| +{{< /table >}} + +Exception: When the quoted word is a user input. + +Example: +* My user ID is “IM47g”. +* Did you try the password “mycatisawesome”? + +## Inline code formatting + + +### Use code style for inline code and commands + +For inline code in an HTML document, use the `` tag. In a Markdown +document, use the backtick (`` ` ``). + +{{< table caption="Do and Don't - Use code style for filenames, directories, paths, object field names and namespaces" >}} +|Do|Don't| +|--- |--- | +|The `kubectl run` command creates a Deployment.|The "kubectl run" command creates a Deployment.| +|For declarative management, use `kubectl apply`.|For declarative management, use "kubectl apply".| +|Use single backticks to enclose inline code. For example, `var example = true`.|Use two asterisks (`**`) or an underscore (`_`) to enclose inline code. For example, **var example = true**.| +|Use triple backticks (\`\`\`) before and after a multi-line block of code for fenced code blocks.|Use multi-line blocks of code to create diagrams, flowcharts, or other illustrations.| +|Use meaningful variable names that have a context.|Use variable names such as 'foo','bar', and 'baz' that are not meaningful and lack context.| +|Remove trailing spaces in the code.|Add trailing spaces in the code, where these are important, because a screen reader will read out the spaces as well.| +{{< /table >}} + +### Starting a sentence with a component tool or component name + +{{< table caption="Do and Don't - Starting a sentence with a component tool or component name" >}} +|Do|Don't| +|--- |--- | +|The `kubeadm` tool bootstraps and provisions machines in a cluster.|`kubeadm` tool bootstraps and provisions machines in a cluster.| +|The kube-scheduler is the default scheduler for Kubernetes.|kube-scheduler is the default scheduler for Kubernetes.| +{{< /table >}} + +### Use normal style for string and integer field values + +For field values of type string or integer, use normal style without quotation marks. + +{{< table caption="Do and Don't - Use normal style for string and integer field values" >}} +|Do|Don't| +|--- |--- | +|Set the value of `imagePullPolicy` to `Always`.|Set the value of `imagePullPolicy` to "Always".| +|Set the value of `image` to `nginx:1.16`.|Set the value of `image` to nginx:1.16.| +|Set the value of the `replicas` field to `2`.|Set the value of the `replicas` field to 2.| +{{< /table >}} + +## Code snippet formatting + + +### Don't include the command prompt + +{{< table caption="Do and Don't - Don't include the command prompt" >}} +|Do|Don't| +|--- |--- | +|kubectl get pods|$ kubectl get pods| +{{< /table >}} + +### Separate commands from output + +Verify that the Pod is running on your chosen node: + +``` +kubectl get pods --output=wide +``` + +The output is similar to this: + +``` +NAME READY STATUS RESTARTS AGE IP NODE +nginx 1/1 Running 0 13s 10.200.0.4 worker0 +``` + +## Velero.io word list + + +A list of Velero-specific terms and words to be used consistently across the site. + +{{< table caption="Velero.io word list" >}} +|Trem|Usage| +|--- |--- | +|Kubernetes|Kubernetes should always be capitalized.| +|Docker|Docker should always be capitalized.| +|Velero|Velero should always be capitalized.| +|VMware|VMware should always be correctly capitalized.| +|On-premises|On-premises or on-prem rather than on-premise or other variations.| +|Backup|Backup rather than back up, back-up or other variations.| +|Plugin|Plugin rather than plug-in or other variations.| +|Allowlist|Use allowlist instead of whitelist.| +|Denylist|Use denylist instead of blacklist.| +{{< /table >}} + +## Markdown elements + +### Headings +People accessing this documentation may use a screen reader or other assistive technology (AT). [Screen readers](https://en.wikipedia.org/wiki/Screen_reader) are linear output devices, they output items on a page one at a time. If there is a lot of content on a page, you can use headings to give the page an internal structure. A good page structure helps all readers to easily navigate the page or filter topics of interest. + +{{< table caption="Do and Don't - Headings" >}} +|Do|Don't| +|--- |--- | +|Include a title on each page or blog post.|Include more than one title headings (#) in a page.| +|Use ordered headings to provide a meaningful high-level outline of your content.|Use headings level 4 through 6, unless it is absolutely necessary. If your content is that detailed, it may need to be broken into separate articles.| +|Use sentence case for headings. For example, **Extend kubectl with plugins**|Use title case for headings. For example, **Extend Kubectl With Plugins**| +{{< /table >}} + +### Paragraphs + +{{< table caption="Do and Don't - Paragraphs" >}} + +|Do|Don't| +|--- |--- | +|Try to keep paragraphs under 6 sentences.|Write long-winded paragraphs.| +|Use three hyphens (`---`) to create a horizontal rule for breaks in paragraph content.|Use horizontal rules for decoration.| +{{< /table >}} + +### Links + +{{< table caption="Do and Don't - Links" >}} +|Do|Don't| +|--- |--- | +|Write hyperlinks that give you context for the content they link to. For example: Certain ports are open on your machines. See [check required ports](#check-required-ports) for more details.|Use ambiguous terms such as “click here”. For example: Certain ports are open on your machines. See [here](#check-required-ports) for more details.| +|Write Markdown-style links: `[link text](URL)`. For example: `[community meeting agenda](https://hackmd.io/Jq6F5zqZR7S80CeDWUklkA)` and the output is [community meeting agenda](https://hackmd.io/Jq6F5zqZR7S80CeDWUklkA).|Write HTML-style links: `Visit our tutorial!`| +{{< /table >}} + + +### Lists + +Group items in a list that are related to each other and need to appear in a specific order or to indicate a correlation between multiple items. When a screen reader comes across a list—whether it is an ordered or unordered list—it will be announced to the user that there is a group of list items. The user can then use the arrow keys to move up and down between the various items in the list. +Website navigation links can also be marked up as list items; after all they are nothing but a group of related links. + + - End each item in a list with a period if one or more items in the list are complete sentences. For the sake of consistency, normally either all items or none should be complete sentences. + + - Ordered lists that are part of an incomplete introductory sentence can be in lowercase and punctuated as if each item was a part of the introductory sentence. + + - Use the number one (`1.`) for ordered lists. + + - Use (`+`), (`*`), or (`-`) for unordered lists - be consistent within the same document. + + - Leave a blank line after each list. + + - Indent nested lists with four spaces (for example, ⋅⋅⋅⋅). + + - List items may consist of multiple paragraphs. Each subsequent paragraph in a list item must be indented by either four spaces or one tab. + +### Tables + +The semantic purpose of a data table is to present tabular data. Sighted users can quickly scan the table but a screen reader goes through line by line. A table [caption](https://www.w3schools.com/tags/tag_caption.asp) is used to create a descriptive title for a data table. Assistive technologies (AT) use the HTML table caption element to identify the table contents to the user within the page structure. + +If you need to create a table, create the table in markdown and use the table [Hugo shortcode](https://gohugo.io/content-management/shortcodes/) to include a caption. + +``` +{{}} +Parameter | Description | Default +:---------|:------------|:------- +`timeout` | The timeout for requests | `30s` +`logLevel` | The log level for log output | `INFO` +{{< /table */>}} + +``` +**Note:** This shortcode does not support markdown reference-style links. Use inline-style links in tables. See more information about [markdown link styles](https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet#links). diff --git a/site/content/docs/v1.8/support-process.md b/site/content/docs/v1.8/support-process.md new file mode 100644 index 000000000..d07268c52 --- /dev/null +++ b/site/content/docs/v1.8/support-process.md @@ -0,0 +1,44 @@ +--- +title: "Support Process" +layout: docs +--- + +## Weekly Rotation + +The Velero maintainers use a weekly rotation to manage community support. Each week, a different maintainer is the point person for responding to incoming support issues via Slack, GitHub, and the Google group. The point person is *not* expected to be on-call 24x7. Instead, they choose one or more hour(s) per day to be available/responding to incoming issues. They will communicate to the community what that time slot will be each week. + +## Start of Week + +We will update the public Slack channel's topic to indicate that you are the point person for the week, and what hours you'll be available. + +## During the Week + +### Where we will monitor +- `#velero` public Slack channel in Kubernetes org +- [all Velero-related repos][0] in GitHub (`velero`, `velero-plugin-for-[aws|gcp|microsoft-azure|csi]`, `helm-charts`) +- [Project Velero Google Group][1] + +### GitHub issue flow + +Generally speaking, new GitHub issues will fall into one of several categories. We use the following process for each: + +1. **Feature request** + - Label the issue with `Enhancement/User` or `Enhancement/Dev` + - Leave the issue in the `New Issues` swimlane for triage by product mgmt +1. **Bug** + - Label the issue with `Bug` + - Leave the issue in the `New Issues` swimlane for triage by product mgmt +1. **User question/problem** that does not clearly fall into one of the previous categories + - When you start investigating/responding, label the issue with `Investigating` + - Add comments as you go, so both the user and future support people have as much context as possible + - Use the `Needs Info` label to indicate an issue is waiting for information from the user. Remove/re-add the label as needed. + - If you resolve the issue with the user, close it out + - If the issue ends up being a feature request or a bug, update the title and follow the appropriate process for it + - If the reporter becomes unresponsive after multiple pings, close out the issue due to inactivity and comment that the user can always reach out again as needed + +## End of Week + +We ensure all GitHub issues worked on during the week on are labeled with `Investigating` and `Needs Info` (if appropriate), and have updated comments so the next person can pick them up. + +[0]: https://github.com/vmware-tanzu?q=velero&type=&language= +[1]: https://groups.google.com/forum/#!forum/projectvelero diff --git a/site/content/docs/v1.8/supported-providers.md b/site/content/docs/v1.8/supported-providers.md new file mode 100644 index 000000000..5e75dc1e2 --- /dev/null +++ b/site/content/docs/v1.8/supported-providers.md @@ -0,0 +1,70 @@ +--- +title: "Providers" +layout: docs +--- + +Velero supports a variety of storage providers for different backup and snapshot operations. Velero has a plugin system which allows anyone to add compatibility for additional backup and volume storage platforms without modifying the Velero codebase. + +## Velero supported providers + +{{< table caption="Velero supported providers" >}} + +| Provider | Object Store | Volume Snapshotter | Plugin Provider Repo | Setup Instructions | +|-----------------------------------|---------------------|------------------------------|-----------------------------------------|-------------------------------| +| [Amazon Web Services (AWS)](https://aws.amazon.com) | AWS S3 | AWS EBS | [Velero plugin for AWS](https://github.com/vmware-tanzu/velero-plugin-for-aws) | [AWS Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-aws#setup) | +| [Google Cloud Platform (GCP)](https://cloud.google.com) | Google Cloud Storage| Google Compute Engine Disks | [Velero plugin for GCP](https://github.com/vmware-tanzu/velero-plugin-for-gcp) | [GCP Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-gcp#setup) | +| [Microsoft Azure](https://azure.com) | Azure Blob Storage | Azure Managed Disks | [Velero plugin for Microsoft Azure](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure) | [Azure Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#setup) | +| [VMware vSphere](https://www.vmware.com/ca/products/vsphere.html) | 🚫 | vSphere Volumes | [VMware vSphere](https://github.com/vmware-tanzu/velero-plugin-for-vsphere) | [vSphere Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-vsphere#velero-plugin-for-vsphere-installation-and-configuration-details) | +| [Container Storage Interface (CSI)](https://kubernetes.io/blog/2019/01/15/container-storage-interface-ga/)| 🚫 | CSI Volumes | [Velero plugin for CSI](https://github.com/vmware-tanzu/velero-plugin-for-csi/) | [CSI Plugin Setup](https://github.com/vmware-tanzu/velero-plugin-for-csi#kinds-of-plugins-included) | +{{< /table >}} + +Contact: [#Velero Slack](https://kubernetes.slack.com/messages/velero), [GitHub Issues](https://github.com/vmware-tanzu/velero/issues) + +## Community supported providers +{{< table caption="Community supported providers" >}} + +| Provider | Object Store | Volume Snapshotter | Plugin Documentation | Contact | +|---------------------------|------------------------------|------------------------------------|------------------------|---------------------------------| +| [AlibabaCloud](https://www.alibabacloud.com/) | Alibaba Cloud OSS | Alibaba Cloud | [AlibabaCloud](https://github.com/AliyunContainerService/velero-plugin) | [GitHub Issue](https://github.com/AliyunContainerService/velero-plugin/issues) | +| [DigitalOcean](https://www.digitalocean.com/) | DigitalOcean Object Storage | DigitalOcean Volumes Block Storage | [StackPointCloud](https://github.com/StackPointCloud/ark-plugin-digitalocean) | | +| [Hewlett Packard](https://www.hpe.com/us/en/storage.html) | 🚫 | HPE Storage | [Hewlett Packard](https://github.com/hpe-storage/velero-plugin) | [Slack](https://slack.hpedev.io/), [GitHub Issue](https://github.com/hpe-storage/velero-plugin/issues) | +| [OpenEBS](https://openebs.io/) | 🚫 | OpenEBS CStor Volume | [OpenEBS](https://github.com/openebs/velero-plugin) | [Slack](https://openebs-community.slack.com/), [GitHub Issue](https://github.com/openebs/velero-plugin/issues) | +| [OpenStack](https://www.openstack.org/) | Swift | Cinder | [OpenStack](https://github.com/Lirt/velero-plugin-for-openstack) | [GitHub Issue](https://github.com/Lirt/velero-plugin-for-openstack/issues) | +| [Portworx](https://portworx.com/) | 🚫 | Portworx Volume | [Portworx](https://docs.portworx.com/scheduler/kubernetes/ark.html) | [Slack](https://portworx.slack.com/messages/px-k8s), [GitHub Issue](https://github.com/portworx/ark-plugin/issues) | +| [Storj](https://storj.io) | Storj Object Storage | 🚫 | [Storj](https://github.com/storj-thirdparty/velero-plugin) | [GitHub Issue](https://github.com/storj-thirdparty/velero-plugin/issues) | +{{< /table >}} + +## S3-Compatible object store providers + +Velero's AWS Object Store plugin uses [Amazon's Go SDK][0] to connect to the AWS S3 API. Some third-party storage providers also support the S3 API, and users have reported the following providers work with Velero: + +_Note that these storage providers are not regularly tested by the Velero team._ + + * [IBM Cloud][1] + * [Oracle Cloud][2] + * [Minio][3] + * [DigitalOcean][4] + * [NooBaa][5] + * [Tencent Cloud][7] + * Ceph RADOS v12.2.7 + * Quobyte + * [Cloudian HyperStore][38] + +_Some storage providers, like Quobyte, may need a different [signature algorithm version][6]._ + +## Non-supported volume snapshots + +In the case you want to take volume snapshots but didn't find a plugin for your provider, Velero has support for snapshotting using restic. Please see the [restic integration][30] documentation. + +[0]: https://github.com/aws/aws-sdk-go/aws +[1]: contributions/ibm-config.md +[2]: contributions/oracle-config.md +[3]: contributions/minio.md +[4]: https://github.com/StackPointCloud/ark-plugin-digitalocean +[5]: http://www.noobaa.com/ +[6]: https://github.com/vmware-tanzu/velero-plugin-for-aws/blob/main/backupstoragelocation.md +[7]: contributions/tencent-config.md +[25]: https://github.com/hpe-storage/velero-plugin +[30]: restic.md +[36]: https://github.com/vmware-tanzu/velero-plugin-for-gcp#setup +[38]: https://www.cloudian.com/ diff --git a/site/content/docs/v1.8/tilt.md b/site/content/docs/v1.8/tilt.md new file mode 100644 index 000000000..42f68cfe4 --- /dev/null +++ b/site/content/docs/v1.8/tilt.md @@ -0,0 +1,210 @@ +--- +title: "Rapid iterative Velero development with Tilt " +layout: docs +--- + +## Overview +This document describes how to use [Tilt](https://tilt.dev) with any cluster for a simplified +workflow that offers easy deployments and rapid iterative builds. + +This setup allows for continuing deployment of the Velero server and, if specified, any provider plugin or the restic daemonset. +It does this work by: + +1. Deploying the necessary Kubernetes resources, such as the Velero CRDs and Velero deployment +1. Building a local binary for Velero and (if specified) provider plugins as a `local_resource` +1. Invoking `docker_build` to live update any binary into the container/init container and trigger a re-start + +Tilt will look for configuration files under `velero/tilt-resources`. Most of the +files in this directory are gitignored so you may configure your setup according to your needs. + +## Prerequisites +1. [Docker](https://docs.docker.com/install/) v19.03 or newer +1. A Kubernetes cluster v1.12 or greater (does not have to be Kind) +1. [Tilt](https://docs.tilt.dev/install.html) v0.12.0 or newer +1. Clone the [Velero project](https://github.com/vmware-tanzu/velero) repository + locally +1. Access to an S3 object storage +1. Clone any [provider plugin(s)](https://velero.io/plugins/) you want to make changes to and deploy (optional, must be configured to be deployed by the Velero Tilt's setup, [more info below](#provider-plugins)) + +Note: To properly configure any plugin you use, please follow the plugin's documentation. + +## Getting started + +### tl;dr +- Copy all sample files under `velero/tilt-resources/examples` into `velero/tilt-resources`. +- Configure the `velero_v1_backupstoragelocation.yaml` file, and the `cloud` file for the storage credentials/secret. + +- Run `tilt up`. + +### Create a Tilt settings file +Create a configuration file named `tilt-settings.json` and place it in your local copy of `velero/tilt-resources`. Alternatively, +you may copy and paste the sample file found in `velero/tilt-resources/examples`. + +Here is an example: + +```json +{ + "default_registry": "", + "enable_providers": [ + "aws", + "gcp", + "azure", + "csi" + ], + "providers": { + "aws": "../velero-plugin-for-aws", + "gcp": "../velero-plugin-for-gcp", + "azure": "../velero-plugin-for-microsoft-azure", + "csi": "../velero-plugin-for-csi" + }, + "allowed_contexts": [ + "development" + ], + "enable_restic": false, + "create_backup_locations": true, + "setup-minio": true, + "enable_debug": false, + "debug_continue_on_start": true +} +``` + +#### tilt-settings.json fields +**default_registry** (String, default=""): The image registry to use if you need to push images. See the [Tilt +*documentation](https://docs.tilt.dev/api.html#api.default_registry) for more details. + +**provider_repos** (Array[]String, default=[]): A list of paths to all the provider plugins you want to make changes to. Each provider must have a +`tilt-provider.json` file describing how to build the provider. + +**enable_providers** (Array[]String, default=[]): A list of the provider plugins to enable. See [provider plugins](provider-plugins) +for more details. Note: when not making changes to a plugin, it is not necessary to load them into +Tilt: an existing image and version might be specified in the Velero deployment instead, and Tilt will load that. + +**allowed_contexts** (Array, default=[]): A list of kubeconfig contexts Tilt is allowed to use. See the Tilt documentation on +*[allow_k8s_contexts](https://docs.tilt.dev/api.html#api.allow_k8s_contexts) for more details. Note: Kind is automatically allowed. + +**enable_restic** (Bool, default=false): Indicate whether to deploy the restic Daemonset. If set to `true`, Tilt will look for a `velero/tilt-resources/restic.yaml` file +containing the configuration of the Velero restic DaemonSet. + +**create_backup_locations** (Bool, default=false): Indicate whether to create one or more backup storage locations. If set to `true`, Tilt will look for a `velero/tilt-resources/velero_v1_backupstoragelocation.yaml` file +containing at least one configuration for a Velero backup storage location. + +**setup-minio** (Bool, default=false): Configure this to `true` if you want to configure backup storage locations in a Minio instance running inside your cluster. + +**enable_debug** (Bool, default=false): Configure this to `true` if you want to debug the velero process using [Delve](https://github.com/go-delve/delve). + +**debug_continue_on_start** (Bool, default=true): Configure this to `true` if you want the velero process to continue on start when in debug mode. See [Delve CLI documentation](https://github.com/go-delve/delve/blob/master/Documentation/usage/dlv.md). + +### Create Kubernetes resource files to deploy +All needed Kubernetes resource files are provided as ready to use samples in the `velero/tilt-resources/examples` directory. You only have to move them to the `velero/tilt-resources` level. + +Because the Velero Kubernetes deployment as well as the restic DaemonSet contain the configuration +for any plugin to be used, files for these resources are expected to be provided by the user so you may choose +which provider plugin to load as a init container. Currently, the sample files provided are configured with all the +plugins supported by Velero, feel free to remove any of them as needed. + +For Velero to operate fully, it also needs at least one backup +storage location. A sample file is provided that needs to be modified with the specific +configuration for your object storage. See the next sub-section for more details on this. + +### Configure a backup storage location +You will have to configure the `velero/tilt-resources/velero_v1_backupstoragelocation.yaml` with the proper values according to your storage provider. Read the [plugin documentation](https://velero.io/plugins/) +to learn what field/value pairs are required for your particular provider's backup storage location configuration. + +Below are some ways to configure a backup storage location for Velero. +#### As a storage with a service provider +Follow the provider documentation to provision the storage. We have a [list of all known object storage providers](supported-providers/) with corresponding plugins for Velero. + +#### Using MinIO as an object storage +Note: to use MinIO as an object storage, you will need to use the [`AWS` plugin](https://github.com/vmware-tanzu/velero-plugin-for-aws), and configure the storage location with the `spec.provider` set to `aws` and the `spec.config.region` set to `minio`. Example: +``` +spec: + config: + region: minio + s3ForcePathStyle: "true" + s3Url: http://minio.velero.svc:9000 + objectStorage: + bucket: velero + provider: aws +``` + +Here are two ways to use MinIO as the storage: + +1) As a MinIO instance running inside your cluster (don't do this for production!) + + In the `tilt-settings.json` file, set `"setup-minio": true`. This will configure a Kubernetes deployment containing a running +instance of MinIO inside your cluster. There are [extra steps](contributions/minio/#expose-minio-outside-your-cluster-with-a-service) +necessary to expose MinIO outside the cluster. + + To access this storage, you will need to expose MinIO outside the cluster by forwarding the MinIO port to the local machine using kubectl port-forward -n svc/minio 9000. Update the BSL configuration to use that as its "public URL" by adding `publicUrl: http://localhost:9000` to the BSL config. This is necessary to do things like download a backup file. + + Note: with this setup, when your cluster is terminated so is the storage and any backup/restore in it. + +1) As a standalone MinIO instance running locally in a Docker container + + See [these instructions](https://github.com/vmware-tanzu/velero/discussions/3381) to run MinIO locally on your computer, as a standalone as opposed to running it on a Pod. + +Please see our [locations documentation](locations/) to learn more how backup locations work. + +### Configure the provider credentials (secret) +Whatever object storage provider you use, configure the credentials for in the `velero/tilt-resources/cloud` file. Read the [plugin documentation](https://velero.io/plugins/) +to learn what field/value pairs are required for your provider's credentials. The Tilt file will invoke Kustomize to create the secret under the hard-coded key `secret.cloud-credentials.data.cloud` in the Velero namespace. + +There is a sample credentials file properly formatted for a MinIO storage credentials in `velero/tilt-resources/examples/cloud`. + +### Configure debugging with Delve +If you would like to debug the Velero process, you can enable debug mode by setting the field `enable_debug` to `true` in your `tilt-resources/tile-settings.json` file. +This will enable you to debug the process using [Delve](https://github.com/go-delve/delve). +By enabling debug mode, the Velero executable will be built in debug mode (using the flags `-gcflags="-N -l"` which disables optimizations and inlining), and the process will be started in the Velero deployment using [`dlv exec`](https://github.com/go-delve/delve/blob/master/Documentation/usage/dlv_exec.md). + +The debug server will accept connections on port 2345 and Tilt is configured to forward this port to the local machine. +Once Tilt is [running](#run-tilt) and the Velero resource is ready, you can connect to the debug server to begin debugging. +To connect to the session, you can use the Delve CLI locally by running `dlv connect 127.0.0.1:2345`. See the [Delve CLI documentation](https://github.com/go-delve/delve/tree/master/Documentation/cli) for more guidance on how to use Delve. +Delve can also be used within a number of [editors and IDEs](https://github.com/go-delve/delve/blob/master/Documentation/EditorIntegration.md). + +By default, the Velero process will continue on start when in debug mode. +This means that the process will run until a breakpoint is set. +You can disable this by setting the field `debug_continue_on_start` to `false` in your `tilt-resources/tile-settings.json` file. +When this setting is disabled, the Velero process will not continue to run until a `continue` instruction is issued through your Delve session. + +When exiting your debug session, the CLI and editor integrations will typically ask if the remote process should be stopped. +It is important to leave the remote process running and just disconnect from the debugging session. +By stopping the remote process, that will cause the Velero container to stop and the pod to restart. +If backups are in progress, these will be left in a stale state as they are not resumed when the Velero pod restarts. + +### Run Tilt! +To launch your development environment, run: + +``` bash +tilt up +``` + +This will output the address to a web browser interface where you can monitor Tilt's status and the logs for each Tilt resource. After a brief amount of time, you should have a running development environment, and you should now be able to +create backups/restores and fully operate Velero. + +Note: Running `tilt down` after exiting out of Tilt [will delete all resources](https://docs.tilt.dev/cli/tilt_down.html) specified in the Tiltfile. + +Tip: Create an alias to `velero/_tuiltbuild/local/velero` and you won't have to run `make local` to get a refreshed version of the Velero CLI, just use the alias. + +Please see the documentation for [how Velero works](how-velero-works/). + +## Provider plugins +A provider must supply a `tilt-provider.json` file describing how to build it. Here is an example: + +```json +{ + "plugin_name": "velero-plugin-for-aws", + "context": ".", + "image": "velero/velero-plugin-for-aws", + "live_reload_deps": [ + "velero-plugin-for-aws" + ], + "go_main": "./velero-plugin-for-aws" +} +``` + +## Live updates +Each provider plugin configured to be deployed by Velero's Tilt setup has a `live_reload_deps` list. This defines the files and/or directories that Tilt +should monitor for changes. When a dependency is modified, Tilt rebuilds the provider's binary **on your local +machine**, copies the binary to the init container, and triggers a restart of the Velero container. This is significantly faster +than rebuilding the container image for each change. It also helps keep the size of each development image as small as +possible (the container images do not need the entire go toolchain, source code, module dependencies, etc.). diff --git a/site/content/docs/v1.8/troubleshooting.md b/site/content/docs/v1.8/troubleshooting.md new file mode 100644 index 000000000..cdfd25305 --- /dev/null +++ b/site/content/docs/v1.8/troubleshooting.md @@ -0,0 +1,220 @@ +--- +title: "Troubleshooting" +layout: docs +--- + +These tips can help you troubleshoot known issues. If they don't help, you can [file an issue][4], or talk to us on the [#velero channel][25] on the Kubernetes Slack server. + +## Debug installation/ setup issues + +- [Debug installation/setup issues][2] + +## Debug restores + +- [Debug restores][1] + +## General troubleshooting information + +You can use the `velero bug` command to open a [Github issue][4] by launching a browser window with some prepopulated values. Values included are OS, CPU architecture, `kubectl` client and server versions (if available) and the `velero` client version. This information isn't submitted to Github until you click the `Submit new issue` button in the Github UI, so feel free to add, remove or update whatever information you like. + +You can use the `velero debug` command to generate a debug bundle, which is a tarball +that contains: +* Version information +* Logs of velero server and plugins +* Resources managed by velero server such as backup, restore, podvolumebackup, podvolumerestore, etc. +* Logs of the backup and restore, if specified in the parameters + +Please use command `velero debug --help` to see more usage details. + +### Getting velero debug logs + +You can increase the verbosity of the Velero server by editing your Velero deployment to look like this: + + +``` +kubectl edit deployment/velero -n velero +... + containers: + - name: velero + image: velero/velero:latest + command: + - /velero + args: + - server + - --log-level # Add this line + - debug # Add this line +... +``` + +## Known issue with restoring LoadBalancer Service + +Because of how Kubernetes handles Service objects of `type=LoadBalancer`, when you restore these objects you might encounter an issue with changed values for Service UIDs. Kubernetes automatically generates the name of the cloud resource based on the Service UID, which is different when restored, resulting in a different name for the cloud load balancer. If the DNS CNAME for your application points to the DNS name of your cloud load balancer, you'll need to update the CNAME pointer when you perform a Velero restore. + +Alternatively, you might be able to use the Service's `spec.loadBalancerIP` field to keep connections valid, if your cloud provider supports this value. See [the Kubernetes documentation about Services of Type LoadBalancer](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). + +## Miscellaneous issues + +### Velero reports `custom resource not found` errors when starting up. + +Velero's server will not start if the required Custom Resource Definitions are not found in Kubernetes. Run `velero install` again to install any missing custom resource definitions. + +### `velero backup logs` returns a `SignatureDoesNotMatch` error + +Downloading artifacts from object storage utilizes temporary, signed URLs. In the case of S3-compatible +providers, such as Ceph, there may be differences between their implementation and the official S3 +API that cause errors. + +Here are some things to verify if you receive `SignatureDoesNotMatch` errors: + + * Make sure your S3-compatible layer is using [signature version 4][5] (such as Ceph RADOS v12.2.7) + * For Ceph, try using a native Ceph account for credentials instead of external providers such as OpenStack Keystone + +## Velero (or a pod it was backing up) restarted during a backup and the backup is stuck InProgress + +Velero cannot resume backups that were interrupted. Backups stuck in the `InProgress` phase can be deleted with `kubectl delete backup -n `. +Backups in the `InProgress` phase have not uploaded any files to object storage. + +## Velero is not publishing prometheus metrics + +Steps to troubleshoot: + +- Confirm that your velero deployment has metrics publishing enabled. The [latest Velero helm charts][6] have been setup with [metrics enabled by default][7]. +- Confirm that the Velero server pod exposes the port on which the metrics server listens on. By default, this value is 8085. + +```yaml + ports: + - containerPort: 8085 + name: metrics + protocol: TCP +``` + +- Confirm that the metric server is listening for and responding to connections on this port. This can be done using [port-forwarding][9] as shown below + +```bash +$ kubectl -n port-forward 8085:8085 +Forwarding from 127.0.0.1:8085 -> 8085 +Forwarding from [::1]:8085 -> 8085 +. +. +. +``` + +Now, visiting http://localhost:8085/metrics on a browser should show the metrics that are being scraped from Velero. + +- Confirm that the Velero server pod has the necessary [annotations][8] for prometheus to scrape metrics. +- Confirm, from the Prometheus UI, that the Velero pod is one of the targets being scraped from Prometheus. + + +## Is Velero using the correct cloud credentials? + +Cloud provider credentials are given to Velero to store and retrieve backups from the object store and to perform volume snapshotting operations. + +These credentials are either passed to Velero at install time using: +1. `--secret-file` flag to the `velero install` command. OR +1. `--set-file credentials.secretContents.cloud` flag to the `helm install` command. + +Or, they are specified when creating a `BackupStorageLocation` using the `--credential` flag. + +### Troubleshooting credentials provided during install + +If using the credentials provided at install time, they are stored in the cluster as a Kubernetes secret named `cloud-credentials` in the same namespace in which Velero is installed. + +Follow the below troubleshooting steps to confirm that Velero is using the correct credentials: +1. Confirm that the `cloud-credentials` secret exists and has the correct content. + ```bash + $ kubectl -n velero get secrets cloud-credentials + NAME TYPE DATA AGE + cloud-credentials Opaque 1 11h + $ kubectl -n velero get secrets cloud-credentials -ojsonpath={.data.cloud} | base64 --decode + + ``` + +1. Confirm that velero deployment is mounting the `cloud-credentials` secret. + ```bash + $ kubectl -n velero get deploy velero -ojson | jq .spec.template.spec.containers[0].volumeMounts + [ + { + "mountPath": "/plugins", + "name": "plugins" + }, + { + "mountPath": "/scratch", + "name": "scratch" + }, + { + "mountPath": "/credentials", + "name": "cloud-credentials" + } + ] + ``` + + If [restic-integration][3] is enabled, then, confirm that the restic daemonset is also mounting the `cloud-credentials` secret. + ```bash + $ kubectl -n velero get ds restic -ojson |jq .spec.template.spec.containers[0].volumeMounts + [ + { + "mountPath": "/host_pods", + "mountPropagation": "HostToContainer", + "name": "host-pods" + }, + { + "mountPath": "/scratch", + "name": "scratch" + }, + { + "mountPath": "/credentials", + "name": "cloud-credentials" + } + ] + ``` + +1. Confirm if the correct credentials are mounted into the Velero pod. + ```bash + $ kubectl -n velero exec -ti deploy/velero -- bash + nobody@velero-69f9c874c-l8mqp:/$ cat /credentials/cloud + + ``` + +### Troubleshooting `BackupStorageLocation` credentials + +Follow the below troubleshooting steps to confirm that Velero is using the correct credentials if using credentials specific to a [`BackupStorageLocation`][10]: +1. Confirm that the object storage provider plugin being used supports multiple credentials. + + If the logs from the Velero deployment contain the error message `"config has invalid keys credentialsFile"`, the version of your object storage plugin does not yet support multiple credentials. + + The object storage plugins [maintained by the Velero team][11] support this feature, so please update your plugin to the latest version if you see the above error message. + + If you are using a plugin from a different provider, please contact them for further advice. + +1. Confirm that the secret and key referenced by the `BackupStorageLocation` exists in the Velero namespace and has the correct content: + ```bash + # Determine which secret and key the BackupStorageLocation is using + BSL_SECRET=$(kubectl get backupstoragelocations.velero.io -n velero -o yaml -o jsonpath={.spec.credential.name}) + BSL_SECRET_KEY=$(kubectl get backupstoragelocations.velero.io -n velero -o yaml -o jsonpath={.spec.credential.key}) + + # Confirm that the secret exists + kubectl -n velero get secret $BSL_SECRET + + # Print the content of the secret and ensure it is correct + kubectl -n velero get secret $BSL_SECRET -ojsonpath={.data.$BSL_SECRET_KEY} | base64 --decode + ``` + If the secret can't be found, the secret does not exist within the Velero namespace and must be created. + + If no output is produced when printing the contents of the secret, the key within the secret may not exist or may have no content. + Ensure that the key exists within the secret's data by checking the output from `kubectl -n velero describe secret $BSL_SECRET`. + If it does not exist, follow the instructions for [editing a Kubernetes secret][12] to add the base64 encoded credentials data. + + +[1]: debugging-restores.md +[2]: debugging-install.md +[3]: restic.md +[4]: https://github.com/vmware-tanzu/velero/issues +[5]: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html +[6]: https://github.com/vmware-tanzu/helm-charts/blob/main/charts/velero +[7]: https://github.com/vmware-tanzu/helm-charts/blob/main/charts/velero/values.yaml#L44 +[8]: https://github.com/vmware-tanzu/helm-charts/blob/main/charts/velero/values.yaml#L49-L52 +[9]: https://kubectl.docs.kubernetes.io/pages/container_debugging/port_forward_to_pods.html +[10]: locations.md +[11]: /plugins +[12]: https://kubernetes.io/docs/concepts/configuration/secret/#editing-a-secret +[25]: https://kubernetes.slack.com/messages/velero diff --git a/site/content/docs/v1.8/uninstalling.md b/site/content/docs/v1.8/uninstalling.md new file mode 100644 index 000000000..f0b7b8d17 --- /dev/null +++ b/site/content/docs/v1.8/uninstalling.md @@ -0,0 +1,11 @@ +--- +title: "Uninstalling Velero" +layout: docs +--- + +If you would like to completely uninstall Velero from your cluster, the following commands will remove all resources created by `velero install`: + +```bash +kubectl delete namespace/velero clusterrolebinding/velero +kubectl delete crds -l component=velero +``` diff --git a/site/content/docs/v1.8/upgrade-to-1.8.md b/site/content/docs/v1.8/upgrade-to-1.8.md new file mode 100644 index 000000000..3a1e80cf5 --- /dev/null +++ b/site/content/docs/v1.8/upgrade-to-1.8.md @@ -0,0 +1,92 @@ +--- +title: "Upgrading to Velero 1.8" +layout: docs +--- + +## Prerequisites + +- Velero [v1.7.x][7] installed. + +If you're not yet running at least Velero v1.6, see the following: + +- [Upgrading to v1.1][1] +- [Upgrading to v1.2][2] +- [Upgrading to v1.3][3] +- [Upgrading to v1.4][4] +- [Upgrading to v1.5][5] +- [Upgrading to v1.6][6] +- [Upgrading to v1.7][7] + +Before upgrading, check the [Velero compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatabilty-matrix) to make sure your version of Kubernetes is supported by the new version of Velero. + +## Instructions + +1. Install the Velero v1.8 command-line interface (CLI) by following the [instructions here][0]. + + Verify that you've properly installed it by running: + + ```bash + velero version --client-only + ``` + + You should see the following output: + + ```bash + Client: + Version: v1.8.0 + Git commit: + ``` + +1. Update the Velero custom resource definitions (CRDs) to include schema changes across all CRDs that are at the core of the new features in this release: + + ```bash + velero install --crds-only --dry-run -o yaml | kubectl apply -f - + ``` + + **NOTE:** Since velero v1.8.0 only v1 CRD will be supported during installation, therefore, the v1.8.0 will only work on kubernetes version >= v1.16 + +1. Update the container image used by the Velero deployment and, optionally, the restic daemon set: + + ```bash + kubectl set image deployment/velero \ + velero=velero/velero:v1.8.0 \ + --namespace velero + + # optional, if using the restic daemon set + kubectl set image daemonset/restic \ + restic=velero/velero:v1.8.0 \ + --namespace velero + ``` + +1. Confirm that the deployment is up and running with the correct version by running: + + ```bash + velero version + ``` + + You should see the following output: + + ```bash + Client: + Version: v1.8.0 + Git commit: + + Server: + Version: v1.8.0 + ``` + +## Notes +### Default backup storage location +We have deprecated the way to indicate the default backup storage location. Previously, that was indicated according to the backup storage location name set on the velero server-side via the flag `velero server --default-backup-storage-location`. Now we configure the default backup storage location on the velero client-side. Please refer to the [About locations][9] on how to indicate which backup storage location is the default one. + +After upgrading, if there is a previously created backup storage location with the name that matches what was defined on the server side as the default, it will be automatically set as the `default`. + +[0]: basic-install.md#install-the-cli +[1]: https://velero.io/docs/v1.1.0/upgrade-to-1.1/ +[2]: https://velero.io/docs/v1.2.0/upgrade-to-1.2/ +[3]: https://velero.io/docs/v1.3.2/upgrade-to-1.3/ +[4]: https://velero.io/docs/v1.4/upgrade-to-1.4/ +[5]: https://velero.io/docs/v1.5/upgrade-to-1.5 +[6]: https://velero.io/docs/v1.6/upgrade-to-1.6 +[7]: https://velero.io/docs/v1.7/upgrade-to-1.7 +[9]: https://velero.io/docs/v1.8/locations diff --git a/site/content/docs/v1.8/velero-install.md b/site/content/docs/v1.8/velero-install.md new file mode 100644 index 000000000..8cc6d4d2e --- /dev/null +++ b/site/content/docs/v1.8/velero-install.md @@ -0,0 +1,49 @@ +--- +title: "Velero Install CLI" +layout: docs +--- + +This document serves as a guide to using the `velero install` CLI command to install `velero` server components into your kubernetes cluster. + +_NOTE_: `velero install` will, by default, use the CLI's version information to determine the version of the server components to deploy. This behavior may be overridden by using the `--image` flag. Refer to [Building Server Component Container Images][1]. + +## Usage + +This section explains some of the basic flags supported by the `velero install` CLI command. For a complete explanation of the flags, please run `velero install --help` + +```bash +velero install \ + --plugins + --provider \ + --bucket \ + --secret-file \ + --velero-pod-cpu-request \ + --velero-pod-mem-request \ + --velero-pod-cpu-limit \ + --velero-pod-mem-limit \ + [--use-restic] \ + [--default-volumes-to-restic] \ + [--restic-pod-cpu-request ] \ + [--restic-pod-mem-request ] \ + [--restic-pod-cpu-limit ] \ + [--restic-pod-mem-limit ] +``` + +The values for the resource requests and limits flags follow the same format as [Kubernetes resource requirements][3] +For plugin container images, please refer to our [supported providers][2] page. + +## Examples + +This section provides examples that serve as a starting point for more customized installations. + +```bash +velero install --provider gcp --plugins velero/velero-plugin-for-gcp:v1.0.0 --bucket mybucket --secret-file ./gcp-service-account.json + +velero install --provider aws --plugins velero/velero-plugin-for-aws:v1.0.0 --bucket backups --provider aws --secret-file ./aws-iam-creds --backup-location-config region=us-east-2 --snapshot-location-config region=us-east-2 --use-restic + +velero install --provider azure --plugins velero/velero-plugin-for-microsoft-azure:v1.0.0 --bucket $BLOB_CONTAINER --secret-file ./credentials-velero --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] +``` + +[1]: build-from-source.md#making-images-and-updating-velero +[2]: supported-providers.md +[3]: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ diff --git a/site/content/docs/v1.8/vendoring-dependencies.md b/site/content/docs/v1.8/vendoring-dependencies.md new file mode 100644 index 000000000..9fc1bcac1 --- /dev/null +++ b/site/content/docs/v1.8/vendoring-dependencies.md @@ -0,0 +1,21 @@ +--- +title: "Vendoring dependencies" +layout: docs +--- + +## Overview + +We are using [dep][0] to manage dependencies. You can install it by following [these +instructions][1]. + +## Adding a new dependency + +Run `dep ensure`. If you want to see verbose output, you can append `-v` as in +`dep ensure -v`. + +## Updating an existing dependency + +Run `dep ensure -update [ ...]` to update one or more dependencies. + +[0]: https://github.com/golang/dep +[1]: https://golang.github.io/dep/docs/installation.html diff --git a/site/content/docs/v1.8/website-guidelines.md b/site/content/docs/v1.8/website-guidelines.md new file mode 100644 index 000000000..c5373f4d8 --- /dev/null +++ b/site/content/docs/v1.8/website-guidelines.md @@ -0,0 +1,45 @@ +--- +title: "Website Guidelines" +layout: docs +--- + +## Running the website locally + +When making changes to the website, please run the site locally before submitting a PR and manually verify your changes. + +At the root of the project, run: + +```bash +make serve-docs +``` + +This runs all the Hugo dependencies in a container. + +Alternatively, for quickly loading the website, under the `velero/site/` directory run: + +```bash +hugo serve +``` + +For more information on how to run the website locally, please see our [Hugo documentation](https://gohugo.io/getting-started/). + +## Adding a blog post + +To add a blog post, create a new markdown (.MD) file in the `/site/content/posts/` folder. A blog post requires the following front matter. + +```yaml +title: "Title of the blog" +excerpt: Brief summary of thee blog post that appears as a preview on velero.io/blogs +author_name: Jane Smith +slug: URL-For-Blog +# Use different categories that apply to your blog. This is used to connect related blogs on the site +categories: ['velero','release'] +# Image to use for blog. The path is relative to the site/static/ folder +image: /img/posts/example-image.jpg +# Tag should match author to drive author pages. Tags can have multiple values. +tags: ['Velero Team', 'Nolan Brubaker'] +``` + +Include the `author_name` value in tags field so the page that lists the author's posts will work properly, for example https://velero.io/tags/carlisia-thompson/. + +Ideally each blog will have a unique image to use on the blog home page, but if you do not include an image, the default Velero logo will be used instead. Use an image that is less than 70KB and add it to the `/site/static/img/posts` folder. diff --git a/site/data/docs/main-toc.yml b/site/data/docs/main-toc.yml index faffa10f7..7d21ed6cf 100644 --- a/site/data/docs/main-toc.yml +++ b/site/data/docs/main-toc.yml @@ -13,8 +13,8 @@ toc: url: /basic-install - page: Customize Installation url: /customize-installation - - page: Upgrade to 1.6 - url: /upgrade-to-1.6 + - page: Upgrade to 1.8 + url: /upgrade-to-1.8 - page: Supported providers url: /supported-providers - page: Evaluation install diff --git a/site/data/docs/toc-mapping.yml b/site/data/docs/toc-mapping.yml index c787dd575..f520c0f8b 100644 --- a/site/data/docs/toc-mapping.yml +++ b/site/data/docs/toc-mapping.yml @@ -3,6 +3,8 @@ # that the navigation for older versions still work. main: main-toc +v1.8: v1-8-toc +v1.7: v1-7-toc v1.6: v1-6-toc v1.5: v1-5-toc v1.4: v1-4-toc diff --git a/site/data/docs/v1-7-toc.yml b/site/data/docs/v1-7-toc.yml new file mode 100644 index 000000000..ec0585229 --- /dev/null +++ b/site/data/docs/v1-7-toc.yml @@ -0,0 +1,97 @@ +toc: + - title: Introduction + subfolderitems: + - page: About Velero + url: /index.html + - page: How Velero works + url: /how-velero-works + - page: About locations + url: /locations + - title: Install + subfolderitems: + - page: Basic Install + url: /basic-install + - page: Customize Installation + url: /customize-installation + - page: Upgrade to 1.7 + url: /upgrade-to-1.7 + - page: Supported providers + url: /supported-providers + - page: Evaluation install + url: /contributions/minio + - page: Restic integration + url: /restic + - page: Examples + url: /examples + - page: Uninstalling + url: /uninstalling + - title: Use + subfolderitems: + - page: Disaster recovery + url: /disaster-case + - page: Cluster migration + url: /migration-case + - page: Enable API group versions + url: /enable-api-group-versions-feature + - page: Resource filtering + url: /resource-filtering + - page: Backup reference + url: /backup-reference + - page: Backup hooks + url: /backup-hooks + - page: Restore reference + url: /restore-reference + - page: Restore hooks + url: /restore-hooks + - page: Run in any namespace + url: /namespace + - page: CSI Support (beta) + url: /csi + - page: Verifying Self-signed Certificates + url: /self-signed-certificates + - page: Changing RBAC permissions + url: /rbac + - title: Plugins + subfolderitems: + - page: Overview + url: /overview-plugins + - page: Custom plugins + url: /custom-plugins + - title: Troubleshoot + subfolderitems: + - page: Troubleshooting + url: /troubleshooting + - page: Troubleshoot an install or setup + url: /debugging-install + - page: Troubleshoot a restore + url: /debugging-restores + - page: Troubleshoot Restic + url: /restic#troubleshooting + - title: Contribute + subfolderitems: + - page: Start Contributing + url: /start-contributing + - page: Development + url: /development + - page: Rapid development with Tilt + url: /tilt + - page: Build from source + url: /build-from-source + - page: Run locally + url: /run-locally + - page: Code standards + url: /code-standards + - page: Website guidelines + url: /website-guidelines + - page: Documentation style guide + url: /style-guide + - title: More information + subfolderitems: + - page: Backup file format + url: /output-file-format + - page: API types + url: /api-types + - page: Support process + url: /support-process + - page: For maintainers + url: /maintainers diff --git a/site/data/docs/v1-8-toc.yml b/site/data/docs/v1-8-toc.yml new file mode 100644 index 000000000..7d21ed6cf --- /dev/null +++ b/site/data/docs/v1-8-toc.yml @@ -0,0 +1,97 @@ +toc: + - title: Introduction + subfolderitems: + - page: About Velero + url: /index.html + - page: How Velero works + url: /how-velero-works + - page: About locations + url: /locations + - title: Install + subfolderitems: + - page: Basic Install + url: /basic-install + - page: Customize Installation + url: /customize-installation + - page: Upgrade to 1.8 + url: /upgrade-to-1.8 + - page: Supported providers + url: /supported-providers + - page: Evaluation install + url: /contributions/minio + - page: Restic integration + url: /restic + - page: Examples + url: /examples + - page: Uninstalling + url: /uninstalling + - title: Use + subfolderitems: + - page: Disaster recovery + url: /disaster-case + - page: Cluster migration + url: /migration-case + - page: Enable API group versions + url: /enable-api-group-versions-feature + - page: Resource filtering + url: /resource-filtering + - page: Backup reference + url: /backup-reference + - page: Backup hooks + url: /backup-hooks + - page: Restore reference + url: /restore-reference + - page: Restore hooks + url: /restore-hooks + - page: Run in any namespace + url: /namespace + - page: CSI Support (beta) + url: /csi + - page: Verifying Self-signed Certificates + url: /self-signed-certificates + - page: Changing RBAC permissions + url: /rbac + - title: Plugins + subfolderitems: + - page: Overview + url: /overview-plugins + - page: Custom plugins + url: /custom-plugins + - title: Troubleshoot + subfolderitems: + - page: Troubleshooting + url: /troubleshooting + - page: Troubleshoot an install or setup + url: /debugging-install + - page: Troubleshoot a restore + url: /debugging-restores + - page: Troubleshoot Restic + url: /restic#troubleshooting + - title: Contribute + subfolderitems: + - page: Start Contributing + url: /start-contributing + - page: Development + url: /development + - page: Rapid development with Tilt + url: /tilt + - page: Build from source + url: /build-from-source + - page: Run locally + url: /run-locally + - page: Code standards + url: /code-standards + - page: Website guidelines + url: /website-guidelines + - page: Documentation style guide + url: /style-guide + - title: More information + subfolderitems: + - page: Backup file format + url: /output-file-format + - page: API types + url: /api-types + - page: Support process + url: /support-process + - page: For maintainers + url: /maintainers diff --git a/site/static/img/adopters/cloudcasa.svg b/site/static/img/adopters/cloudcasa.svg new file mode 100644 index 000000000..e93dc2416 --- /dev/null +++ b/site/static/img/adopters/cloudcasa.svg @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/site/static/img/contributors/bridget-mcerlean.png b/site/static/img/contributors/bridget-mcerlean.png deleted file mode 100644 index 3dbf1f47e..000000000 Binary files a/site/static/img/contributors/bridget-mcerlean.png and /dev/null differ diff --git a/site/static/img/contributors/daniel-jiang.png b/site/static/img/contributors/daniel-jiang.png new file mode 100644 index 000000000..7975a0094 Binary files /dev/null and b/site/static/img/contributors/daniel-jiang.png differ diff --git a/site/static/img/contributors/wenkai-yin.png b/site/static/img/contributors/wenkai-yin.png new file mode 100644 index 000000000..80be05092 Binary files /dev/null and b/site/static/img/contributors/wenkai-yin.png differ diff --git a/site/static/img/contributors/xun-jiang.png b/site/static/img/contributors/xun-jiang.png new file mode 100644 index 000000000..24e6f05c9 Binary files /dev/null and b/site/static/img/contributors/xun-jiang.png differ diff --git a/test/e2e/Makefile b/test/e2e/Makefile index ed47614b2..b49829ab6 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -46,9 +46,15 @@ GINKGO := $(GOPATH)/bin/ginkgo KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin GINKGO_FOCUS ?= +GINKGO_SKIP ?= VELERO_CLI ?=$$(pwd)/../../_output/bin/$(GOOS)/$(GOARCH)/velero VELERO_IMAGE ?= velero/velero:main -CRDS_VERSION ?= v1 +VELERO_VERSION ?= $(VERSION) +PLUGINS ?= +RESTIC_HELPER_IMAGE ?= +#Released version only +UPGRADE_FROM_VELERO_CLI ?= +UPGRADE_FROM_VELERO_VERSION ?= v1.7.1 VELERO_NAMESPACE ?= velero CREDS_FILE ?= BSL_BUCKET ?= @@ -61,6 +67,7 @@ INSTALL_VELERO ?= true REGISTRY_CREDENTIAL_FILE ?= # Flags to create an additional BSL for multiple credentials tests +ADDITIONAL_BSL_PLUGINS ?= ADDITIONAL_OBJECT_STORE_PROVIDER ?= ADDITIONAL_CREDS_FILE ?= ADDITIONAL_BSL_BUCKET ?= @@ -78,11 +85,15 @@ run: ginkgo @[ "${BSL_BUCKET}" ] && echo "Using bucket ${BSL_BUCKET} to store backups from E2E tests" || \ (echo "Bucket to store the backups from E2E tests is required, please re-run with BSL_BUCKET="; exit 1 ) @[ "${CLOUD_PROVIDER}" ] && echo "Using cloud provider ${CLOUD_PROVIDER}" || \ - (echo "Cloud provider for target cloud/plug-in provider is required, please rerun with CLOUD_PROVIDER="; exit 1) - @$(GINKGO) -v -focus="$(GINKGO_FOCUS)" . -- -velerocli=$(VELERO_CLI) \ + (echo "Cloud provider for target cloud/plug-in provider is required, please rerun with CLOUD_PROVIDER="; exit 1) + @$(GINKGO) -v -focus="$(GINKGO_FOCUS)" -skip="$(GINKGO_SKIP)" . -- -velerocli=$(VELERO_CLI) \ -velero-image=$(VELERO_IMAGE) \ + -plugins=$(PLUGINS) \ + -velero-version=$(VELERO_VERSION) \ + -restic-helper-image=$(RESTIC_HELPER_IMAGE) \ + -upgrade-from-velero-cli=$(UPGRADE_FROM_VELERO_CLI) \ + -upgrade-from-velero-version=$(UPGRADE_FROM_VELERO_VERSION) \ -velero-namespace=$(VELERO_NAMESPACE) \ - -crds-version=$(CRDS_VERSION) \ -credentials-file=$(CREDS_FILE) \ -bucket=$(BSL_BUCKET) \ -prefix=$(BSL_PREFIX) \ @@ -90,6 +101,7 @@ run: ginkgo -vsl-config=$(VSL_CONFIG) \ -cloud-provider=$(CLOUD_PROVIDER) \ -object-store-provider="$(OBJECT_STORE_PROVIDER)" \ + -additional-bsl-plugins=$(ADDITIONAL_BSL_PLUGINS) \ -additional-bsl-object-store-provider="$(ADDITIONAL_OBJECT_STORE_PROVIDER)" \ -additional-bsl-credentials-file=$(ADDITIONAL_CREDS_FILE) \ -additional-bsl-bucket=$(ADDITIONAL_BSL_BUCKET) \ diff --git a/test/e2e/backup_test.go b/test/e2e/backup/backup.go similarity index 52% rename from test/e2e/backup_test.go rename to test/e2e/backup/backup.go index b906759b3..21ac1c509 100644 --- a/test/e2e/backup_test.go +++ b/test/e2e/backup/backup.go @@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package backup import ( "context" @@ -23,98 +23,93 @@ import ( "github.com/google/uuid" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/kibishii" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) -var ( - uuidgen uuid.UUID -) - -// Test backup and restore of Kibishi using restic -var _ = Describe("[Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", backup_restore_with_restic) - -var _ = Describe("[Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", backup_restore_with_snapshots) - -func backup_restore_with_snapshots() { - backup_restore_test(true) +func BackupRestoreWithSnapshots() { + BackupRestoreTest(true) } -func backup_restore_with_restic() { - backup_restore_test(false) +func BackupRestoreWithRestic() { + BackupRestoreTest(false) } -func backup_restore_test(useVolumeSnapshots bool) { +func BackupRestoreTest(useVolumeSnapshots bool) { var ( backupName, restoreName string ) - client, err := newTestClient() + client, err := NewTestClient() Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") BeforeEach(func() { - if useVolumeSnapshots && cloudProvider == "kind" { + if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" { Skip("Volume snapshots not supported on kind") } var err error flag.Parse() - uuidgen, err = uuid.NewRandom() + UUIDgen, err = uuid.NewRandom() Expect(err).To(Succeed()) - if installVelero { - Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, - cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed()) + if VeleroCfg.InstallVelero { + Expect(VeleroInstall(context.Background(), &VeleroCfg, "", useVolumeSnapshots)).To(Succeed()) } }) AfterEach(func() { - if installVelero { - err = veleroUninstall(context.Background(), veleroCLI, veleroNamespace) + if VeleroCfg.InstallVelero { + err = VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) Expect(err).To(Succeed()) } }) When("kibishii is the sample workload", func() { It("should be successfully backed up and restored to the default BackupStorageLocation", func() { - backupName = "backup-" + uuidgen.String() - restoreName = "restore-" + uuidgen.String() + backupName = "backup-" + UUIDgen.String() + restoreName = "restore-" + UUIDgen.String() // Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on // KinD. So use the kind installation for Kibishii. - Expect(runKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, "", useVolumeSnapshots, registryCredentialFile)).To(Succeed(), + Expect(RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, "", useVolumeSnapshots, VeleroCfg.RegistryCredentialFile)).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace") }) It("should successfully back up and restore to an additional BackupStorageLocation with unique credentials", func() { - if additionalBSLProvider == "" { + if VeleroCfg.AdditionalBSLProvider == "" { Skip("no additional BSL provider given, not running multiple BackupStorageLocation with unique credentials tests") } - if additionalBSLBucket == "" { + if VeleroCfg.AdditionalBSLBucket == "" { Skip("no additional BSL bucket given, not running multiple BackupStorageLocation with unique credentials tests") } - if additionalBSLCredentials == "" { + if VeleroCfg.AdditionalBSLCredentials == "" { Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests") } - Expect(veleroAddPluginsForProvider(context.TODO(), veleroCLI, veleroNamespace, additionalBSLProvider)).To(Succeed()) + Expect(VeleroAddPluginsForProvider(context.TODO(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, VeleroCfg.AdditionalBSLProvider, VeleroCfg.AddBSLPlugins)).To(Succeed()) // Create Secret for additional BSL - secretName := fmt.Sprintf("bsl-credentials-%s", uuidgen) - secretKey := fmt.Sprintf("creds-%s", additionalBSLProvider) + secretName := fmt.Sprintf("bsl-credentials-%s", UUIDgen) + secretKey := fmt.Sprintf("creds-%s", VeleroCfg.AdditionalBSLProvider) files := map[string]string{ - secretKey: additionalBSLCredentials, + secretKey: VeleroCfg.AdditionalBSLCredentials, } - Expect(createSecretFromFiles(context.TODO(), client, veleroNamespace, secretName, files)).To(Succeed()) + Expect(CreateSecretFromFiles(context.TODO(), client, VeleroCfg.VeleroNamespace, secretName, files)).To(Succeed()) // Create additional BSL using credential - additionalBsl := fmt.Sprintf("bsl-%s", uuidgen) - Expect(veleroCreateBackupLocation(context.TODO(), - veleroCLI, - veleroNamespace, + additionalBsl := fmt.Sprintf("bsl-%s", UUIDgen) + Expect(VeleroCreateBackupLocation(context.TODO(), + VeleroCfg.VeleroCLI, + VeleroCfg.VeleroNamespace, additionalBsl, - additionalBSLProvider, - additionalBSLBucket, - additionalBSLPrefix, - additionalBSLConfig, + VeleroCfg.AdditionalBSLProvider, + VeleroCfg.AdditionalBSLBucket, + VeleroCfg.AdditionalBSLPrefix, + VeleroCfg.AdditionalBSLConfig, secretName, secretKey, )).To(Succeed()) @@ -127,11 +122,10 @@ func backup_restore_test(useVolumeSnapshots bool) { // We limit the length of backup name here to avoid the issue of vsphere plugin https://github.com/vmware-tanzu/velero-plugin-for-vsphere/issues/370 // We can remove the logic once the issue is fixed if bsl == "default" { - backupName = fmt.Sprintf("%s-%s", backupName, uuidgen) - restoreName = fmt.Sprintf("%s-%s", restoreName, uuidgen) + backupName = fmt.Sprintf("%s-%s", backupName, UUIDgen) + restoreName = fmt.Sprintf("%s-%s", restoreName, UUIDgen) } - - Expect(runKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots, registryCredentialFile)).To(Succeed(), + Expect(RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots, VeleroCfg.RegistryCredentialFile)).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace using BSL %s", bsl) } }) diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go new file mode 100644 index 000000000..477bf227d --- /dev/null +++ b/test/e2e/backups/deletion.go @@ -0,0 +1,159 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package backups + +import ( + "context" + "flag" + "fmt" + "time" + + "github.com/google/uuid" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/pkg/errors" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/kibishii" + . "github.com/vmware-tanzu/velero/test/e2e/util/providers" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" +) + +const ( + deletionTest = "deletion-workload" +) + +// Test backup and restore of Kibishi using restic + +func Backup_deletion_with_snapshots() { + backup_deletion_test(true) +} + +func Backup_deletion_with_restic() { + backup_deletion_test(false) +} +func backup_deletion_test(useVolumeSnapshots bool) { + var ( + backupName string + ) + + client, err := NewTestClient() + Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup deletion tests") + + BeforeEach(func() { + if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" { + Skip("Volume snapshots not supported on kind") + } + var err error + flag.Parse() + UUIDgen, err = uuid.NewRandom() + Expect(err).To(Succeed()) + if VeleroCfg.InstallVelero { + Expect(VeleroInstall(context.Background(), &VeleroCfg, "", useVolumeSnapshots)).To(Succeed()) + } + }) + + AfterEach(func() { + if VeleroCfg.InstallVelero { + err = VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) + Expect(err).To(Succeed()) + } + }) + + When("kibishii is the sample workload", func() { + It("Deleted backups are deleted from object storage and backups deleted from object storage can be deleted locally", func() { + backupName = "backup-" + UUIDgen.String() + Expect(runBackupDeletionTests(client, VeleroCfg.VeleroCLI, VeleroCfg.CloudProvider, VeleroCfg.VeleroNamespace, backupName, "", useVolumeSnapshots, VeleroCfg.RegistryCredentialFile, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig)).To(Succeed(), + "Failed to run backup deletion test") + }) + }) +} + +// runUpgradeTests runs upgrade test on the provider by kibishii. +func runBackupDeletionTests(client TestClient, veleroCLI, providerName, veleroNamespace, backupName, backupLocation string, + useVolumeSnapshots bool, registryCredentialFile, bslPrefix, bslConfig string) error { + + oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) + + if err := CreateNamespace(oneHourTimeout, client, deletionTest); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", deletionTest) + } + defer func() { + if err := DeleteNamespace(context.Background(), client, deletionTest, true); err != nil { + fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", deletionTest)) + } + }() + + if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, deletionTest, registryCredentialFile); err != nil { + return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", deletionTest) + } + err := ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1) + if err != nil { + return err + } + if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, deletionTest, backupLocation, useVolumeSnapshots); err != nil { + // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command + // TODO move to "runDebug" after we bump up to 1.7 in the upgrade case + VeleroBackupLogs(context.Background(), VeleroCfg.UpgradeFromVeleroCLI, veleroNamespace, backupName) + return errors.Wrapf(err, "Failed to backup kibishii namespace %s", deletionTest) + } + + if providerName == "vsphere" && useVolumeSnapshots { + // Wait for uploads started by the Velero Plug-in for vSphere to complete + // TODO - remove after upload progress monitoring is implemented + fmt.Println("Waiting for vSphere uploads to complete") + if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, deletionTest); err != nil { + return errors.Wrapf(err, "Error waiting for uploads to complete") + } + } + err = ObjectsShouldBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) + if err != nil { + return err + } + err = DeleteBackupResource(context.Background(), veleroCLI, backupName) + if err != nil { + return err + } + err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5) + if err != nil { + fmt.Println(errors.Wrapf(err, "Failed to get object from bucket %q", backupName)) + return err + } + backupName = "backup-1-" + UUIDgen.String() + if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, deletionTest, backupLocation, useVolumeSnapshots); err != nil { + // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command + // TODO move to "runDebug" after we bump up to 1.7 in the upgrade case + VeleroBackupLogs(context.Background(), VeleroCfg.UpgradeFromVeleroCLI, veleroNamespace, backupName) + return errors.Wrapf(err, "Failed to backup kibishii namespace %s", deletionTest) + } + err = DeleteObjectsInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) + if err != nil { + fmt.Println(errors.Wrapf(err, "Failed to delete object in bucket %q", backupName)) + return err + } + err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1) + if err != nil { + return err + } + err = DeleteBackupResource(context.Background(), veleroCLI, backupName) + if err != nil { + fmt.Println(errors.Wrapf(err, "|| UNEXPECTED || - Failed to delete backup %q", backupName)) + return err + } + fmt.Printf("|| EXPECTED || - Backup deletion test completed successfully\n") + return nil +} diff --git a/test/e2e/backups/sync_backups.go b/test/e2e/backups/sync_backups.go new file mode 100644 index 000000000..bd81014d0 --- /dev/null +++ b/test/e2e/backups/sync_backups.go @@ -0,0 +1,155 @@ +/* + * + * Copyright the Velero contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +//Refer to https://github.com/vmware-tanzu/velero/issues/4253 +package backups + +import ( + "context" + "flag" + "fmt" + "math/rand" + "time" + + "github.com/google/uuid" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/util/wait" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/providers" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" +) + +type SyncBackups struct { + testNS string + backupName string + ctx context.Context +} + +func (b *SyncBackups) Init() { + rand.Seed(time.Now().UnixNano()) + UUIDgen, _ = uuid.NewRandom() + b.testNS = "sync-bsl-test-" + UUIDgen.String() + b.backupName = "sync-bsl-test-" + UUIDgen.String() + b.ctx, _ = context.WithTimeout(context.Background(), time.Duration(time.Minute*10)) +} + +func BackupsSyncTest() { + test := new(SyncBackups) + client, err := NewTestClient() + if err != nil { + println(err.Error()) + } + Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + + BeforeEach(func() { + flag.Parse() + if VeleroCfg.InstallVelero { + Expect(VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed()) + } + }) + + AfterEach(func() { + if VeleroCfg.InstallVelero { + Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) + } + }) + + It("Backups in object storage should be synced to a new Velero successfully", func() { + test.Init() + By(fmt.Sprintf("Prepare workload as target to backup by creating namespace %s namespace", test.testNS)) + Expect(CreateNamespace(test.ctx, client, test.testNS)).To(Succeed(), + fmt.Sprintf("Failed to create %s namespace", test.testNS)) + + defer func() { + Expect(DeleteNamespace(test.ctx, client, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS)) + }() + + By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() { + if err = VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, test.testNS, "", false); err != nil { + RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, "") + } + Expect(err).To(Succeed(), fmt.Sprintf("Failed to backup %s namespace", test.testNS)) + }) + + By("Uninstall velero", func() { + Expect(VeleroUninstall(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) + }) + + By("Install velero", func() { + VeleroCfg.ObjectStoreProvider = "" + Expect(VeleroInstall(test.ctx, &VeleroCfg, "", false)).To(Succeed()) + }) + + By("Check all backups in object storage are synced to Velero", func() { + Expect(test.IsBackupsSynced()).To(Succeed(), fmt.Sprintf("Failed to sync backup %s from object storage", test.backupName)) + }) + }) + + It("Deleted backups in object storage are synced to be deleted in Velero", func() { + test.Init() + By(fmt.Sprintf("Prepare workload as target to backup by creating namespace in %s namespace", test.testNS), func() { + Expect(CreateNamespace(test.ctx, client, test.testNS)).To(Succeed(), + fmt.Sprintf("Failed to create %s namespace", test.testNS)) + }) + + defer func() { + Expect(DeleteNamespace(test.ctx, client, test.testNS, false)).To(Succeed(), + fmt.Sprintf("Failed to delete the namespace %s", test.testNS)) + }() + + By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() { + if err = VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, test.testNS, "", false); err != nil { + RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, "") + } + Expect(err).To(Succeed(), fmt.Sprintf("Failed to backup %s namespace", test.testNS)) + }) + + By(fmt.Sprintf("Delete %s backup files in object store", test.backupName), func() { + err = DeleteObjectsInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, + VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.backupName, BackupObjectsPrefix) + Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete object in bucket %s with err %v", test.backupName, err)) + }) + + By(fmt.Sprintf("Check %s backup files in object store is deleted", test.backupName), func() { + err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, + VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.backupName, BackupObjectsPrefix, 1) + Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete object in bucket %s with err %v", test.backupName, err)) + }) + + By("Check if backups are deleted as a result of sync from BSL", func() { + Expect(WaitBackupDeleted(test.ctx, VeleroCfg.VeleroCLI, test.backupName, time.Minute*10)).To(Succeed(), fmt.Sprintf("Failed to check backup %s deleted", test.backupName)) + }) + }) +} + +func (b *SyncBackups) IsBackupsSynced() error { + return wait.PollImmediate(10*time.Second, 10*time.Minute, func() (bool, error) { + if exist, err := IsBackupExist(b.ctx, VeleroCfg.VeleroCLI, b.backupName); err != nil { + return false, err + } else { + if exist { + return true, nil + } else { + return false, nil + } + } + }) +} diff --git a/test/e2e/enable_api_group_versions_test.go b/test/e2e/basic/enable_api_group_versions.go similarity index 87% rename from test/e2e/enable_api_group_versions_test.go rename to test/e2e/basic/enable_api_group_versions.go index 42f9bf23f..215fdf773 100644 --- a/test/e2e/enable_api_group_versions_test.go +++ b/test/e2e/basic/enable_api_group_versions.go @@ -14,11 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package basic import ( "context" "encoding/json" + "flag" "fmt" "os/exec" "strconv" @@ -34,44 +35,31 @@ import ( "github.com/vmware-tanzu/velero/pkg/builder" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) -var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", func() { +func APIGropuVersionsTest() { var ( resource, group string err error ctx = context.Background() ) - client, err := newTestClient() + client, err := NewTestClient() Expect(err).To(Succeed(), "Failed to instantiate cluster client for group version tests") BeforeEach(func() { resource = "rockbands" group = "music.example.io" - - uuidgen, err = uuid.NewRandom() + UUIDgen, err = uuid.NewRandom() Expect(err).NotTo(HaveOccurred()) - + flag.Parse() // TODO: install Velero once for the test suite once feature flag is // removed and velero installation becomes the same as other e2e tests. - if installVelero { - err = veleroInstall( - context.Background(), - veleroCLI, - veleroImage, - veleroNamespace, - cloudProvider, - objectStoreProvider, - false, - cloudCredentialsFile, - bslBucket, - bslPrefix, - bslConfig, - vslConfig, - crdsVersion, - "EnableAPIGroupVersions", // TODO: remove when feature flag is removed - registryCredentialFile) + if VeleroCfg.InstallVelero { + err = VeleroInstall(context.Background(), &VeleroCfg, "EnableAPIGroupVersions", false) Expect(err).NotTo(HaveOccurred()) } }) @@ -86,8 +74,8 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", } Expect(err).NotTo(HaveOccurred()) - if installVelero { - err = veleroUninstall(ctx, veleroCLI, veleroNamespace) + if VeleroCfg.InstallVelero { + err = VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) Expect(err).NotTo(HaveOccurred()) } @@ -103,9 +91,9 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", )).To(Succeed(), "Failed to successfully backup and restore multiple API Groups") }) }) -}) +} -func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, resource, group string) error { +func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, resource, group string) error { tests := []struct { name string namespaces []string @@ -178,7 +166,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso }, tgtCrdYaml: "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml", tgtVer: "v2beta1", - cm: builder.ForConfigMap(veleroNamespace, "enableapigroupversions").Data( + cm: builder.ForConfigMap(VeleroCfg.VeleroNamespace, "enableapigroupversions").Data( "restoreResourcesVersionPriority", `rockbands.music.example.io=v2beta1,v2beta2,v2`, ).Result(), @@ -222,11 +210,11 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso for version, cr := range tc.srcCRs { ns := resource + "-src-" + version - if err := createNamespace(ctx, client, ns); err != nil { + if err := CreateNamespace(ctx, client, ns); err != nil { return errors.Wrapf(err, "create %s namespace", ns) } defer func(namespace string) { - if err = deleteNamespace(ctx, client, namespace, true); err != nil { + if err = DeleteNamespace(ctx, client, namespace, true); err != nil { fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", ns)) } }(ns) @@ -241,16 +229,16 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso // Restart Velero pods in order to recognize music-system CRD right away // instead of waiting for discovery helper to refresh. See // https://github.com/vmware-tanzu/velero/issues/3471. - if err := restartPods(ctx, veleroNamespace); err != nil { + if err := restartPods(ctx, VeleroCfg.VeleroNamespace); err != nil { return errors.Wrapf(err, "restart Velero pods") } - backup := "backup-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i) + backup := "backup-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i) namespacesStr := strings.Join(tc.namespaces, ",") - err = veleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr, "", false) + err = VeleroBackupNamespace(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, namespacesStr, "", false) if err != nil { - veleroBackupLogs(ctx, veleroCLI, veleroNamespace, backup) + RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, "") return errors.Wrapf(err, "back up %s namespaces on source cluster", namespacesStr) } @@ -259,7 +247,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso } for _, ns := range tc.namespaces { - if err := deleteNamespace(ctx, client, ns, true); err != nil { + if err := DeleteNamespace(ctx, client, ns, true); err != nil { return errors.Wrapf(err, "delete %s namespace from source cluster", ns) } } @@ -273,23 +261,23 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso // Apply config map if there is one. if tc.cm != nil { - _, err := client.clientGo.CoreV1().ConfigMaps(veleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{}) + _, err := client.ClientGo.CoreV1().ConfigMaps(VeleroCfg.VeleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{}) if err != nil { return errors.Wrap(err, "create config map with user version priorities") } } // Reset Velero to recognize music-system CRD. - if err := restartPods(ctx, veleroNamespace); err != nil { + if err := restartPods(ctx, VeleroCfg.VeleroNamespace); err != nil { return errors.Wrapf(err, "restart Velero pods") } // Restore rockbands namespaces. - restore := "restore-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i) + restore := "restore-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i) if tc.want != nil { - if err := veleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup); err != nil { - veleroRestoreLogs(ctx, veleroCLI, veleroNamespace, restore) + if err := VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restore, backup); err != nil { + RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", restore) return errors.Wrapf(err, "restore %s namespaces on target cluster", namespacesStr) } @@ -327,7 +315,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso } else { // No custom resource should have been restored. Expect "no resource found" // error during restore. - err := veleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup) + err := VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restore, backup) if err.Error() != "Unexpected restore phase got PartiallyFailed, expecting Completed" { return errors.New("expected error but not none") diff --git a/test/e2e/basic/resources-check/namespaces.go b/test/e2e/basic/resources-check/namespaces.go new file mode 100644 index 000000000..366e3adfb --- /dev/null +++ b/test/e2e/basic/resources-check/namespaces.go @@ -0,0 +1,129 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the Licensm. +You may obtain a copy of the License at + + http://www.apachm.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the Licensm. +*/ + +package basic + +import ( + "context" + "fmt" + "math/rand" + "strings" + "time" + + "github.com/google/uuid" + "github.com/pkg/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" +) + +type MultiNSBackup struct { + TestCase + IsScalTest bool + NSExcluded *[]string + TimeoutDuration time.Duration +} + +func (m *MultiNSBackup) Init() error { + rand.Seed(time.Now().UnixNano()) + UUIDgen, _ = uuid.NewRandom() + m.BackupName = "backup-" + UUIDgen.String() + m.RestoreName = "restore-" + UUIDgen.String() + m.NSBaseName = "nstest-" + UUIDgen.String() + m.Client = TestClientInstance + m.NSExcluded = &[]string{} + + if m.IsScalTest { + m.NamespacesTotal = 2500 + m.TimeoutDuration = time.Hour * 2 + m.TestMsg = &TestMSG{ + Text: "When I create 2500 namespaces should be successfully backed up and restored", + FailedMSG: "Failed to successfully backup and restore multiple namespaces", + } + } else { + m.NamespacesTotal = 2 + m.TimeoutDuration = time.Minute * 5 + m.TestMsg = &TestMSG{ + Text: "When I create 2 namespaces should be successfully backed up and restored", + FailedMSG: "Failed to successfully backup and restore multiple namespaces", + } + } + return nil +} + +func (m *MultiNSBackup) StartRun() error { + // Currently it's hard to build a large list of namespaces to include and wildcards do not work so instead + // we will exclude all of the namespaces that existed prior to the test from the backup + namespaces, err := m.Client.ClientGo.CoreV1().Namespaces().List(context.Background(), v1.ListOptions{}) + if err != nil { + return errors.Wrap(err, "Could not retrieve namespaces") + } + + for _, excludeNamespace := range namespaces.Items { + *m.NSExcluded = append(*m.NSExcluded, excludeNamespace.Name) + } + + m.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", m.BackupName, + "--exclude-namespaces", strings.Join(*m.NSExcluded, ","), + "--default-volumes-to-restic", "--wait", + } + + m.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", m.RestoreName, + "--from-backup", m.BackupName, "--wait", + } + return nil +} + +func (m *MultiNSBackup) CreateResources() error { + m.Ctx, _ = context.WithTimeout(context.Background(), m.TimeoutDuration) + fmt.Printf("Creating namespaces ...\n") + labels := map[string]string{ + "ns-test": "true", + } + for nsNum := 0; nsNum < m.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", m.NSBaseName, nsNum) + if err := CreateNamespaceWithLabel(m.Ctx, m.Client, createNSName, labels); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s", createNSName) + } + } + return nil +} + +func (m *MultiNSBackup) Verify() error { + // Verify that we got back all of the namespaces we created + for nsNum := 0; nsNum < m.NamespacesTotal; nsNum++ { + checkNSName := fmt.Sprintf("%s-%00000d", m.NSBaseName, nsNum) + checkNS, err := GetNamespace(m.Ctx, m.Client, checkNSName) + if err != nil { + return errors.Wrapf(err, "Could not retrieve test namespace %s", checkNSName) + } else if checkNS.Name != checkNSName { + return errors.Errorf("Retrieved namespace for %s has name %s instead", checkNSName, checkNS.Name) + } + } + return nil +} + +func (m *MultiNSBackup) Destroy() error { + err := CleanupNamespaces(m.Ctx, m.Client, m.NSBaseName) + if err != nil { + return errors.Wrap(err, "Could cleanup retrieve namespaces") + } + return WaitAllSelectedNSDeleted(m.Ctx, m.Client, "ns-test=true") +} diff --git a/test/e2e/basic/resources-check/namespaces_annotation.go b/test/e2e/basic/resources-check/namespaces_annotation.go new file mode 100644 index 000000000..f3f602147 --- /dev/null +++ b/test/e2e/basic/resources-check/namespaces_annotation.go @@ -0,0 +1,101 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package basic + +import ( + "context" + "fmt" + "math/rand" + "strings" + "time" + + "github.com/google/uuid" + "github.com/pkg/errors" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" +) + +type NSAnnotationCase struct { + TestCase +} + +func (n *NSAnnotationCase) Init() error { + rand.Seed(time.Now().UnixNano()) + UUIDgen, _ = uuid.NewRandom() + n.BackupName = "backup-namespace-annotations" + UUIDgen.String() + n.RestoreName = "restore-namespace-annotations" + UUIDgen.String() + n.NSBaseName = "namespace-annotations-" + UUIDgen.String() + n.NamespacesTotal = 1 + n.NSIncluded = &[]string{} + n.Client = TestClientInstance + for nsNum := 0; nsNum < n.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", n.NSBaseName, nsNum) + *n.NSIncluded = append(*n.NSIncluded, createNSName) + } + n.TestMsg = &TestMSG{ + Desc: "Backup/restore namespace annotation test", + Text: "Should be successfully backed up and restored including annotations", + FailedMSG: "Failed to successfully backup and restore multiple namespaces", + } + n.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", n.BackupName, + "--include-namespaces", strings.Join(*n.NSIncluded, ","), + "--default-volumes-to-restic", "--wait", + } + + n.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", n.RestoreName, + "--from-backup", n.BackupName, "--wait", + } + return nil +} + +func (n *NSAnnotationCase) CreateResources() error { + n.Ctx, _ = context.WithTimeout(context.Background(), 10*time.Minute) + for nsNum := 0; nsNum < n.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", n.NSBaseName, nsNum) + createAnnotationName := fmt.Sprintf("annotation-%s-%00000d", n.NSBaseName, nsNum) + if err := CreateNamespaceWithAnnotation(n.Ctx, n.Client, createNSName, map[string]string{"testAnnotation": createAnnotationName}); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s", createNSName) + } + } + return nil +} + +func (n *NSAnnotationCase) Verify() error { + for nsNum := 0; nsNum < n.NamespacesTotal; nsNum++ { + checkNSName := fmt.Sprintf("%s-%00000d", n.NSBaseName, nsNum) + checkAnnoName := fmt.Sprintf("annotation-%s-%00000d", n.NSBaseName, nsNum) + checkNS, err := GetNamespace(n.Ctx, n.Client, checkNSName) + + if err != nil { + return errors.Wrapf(err, "Could not retrieve test namespace %s", checkNSName) + } + if checkNS.Name != checkNSName { + return errors.Errorf("Retrieved namespace for %s has name %s instead", checkNSName, checkNS.Name) + } + + c := checkNS.ObjectMeta.Annotations["testAnnotation"] + + if c != checkAnnoName { + return errors.Errorf("Retrieved annotation for %s has name %s instead", checkAnnoName, c) + } + } + return nil +} diff --git a/test/e2e/basic/resources-check/rbac.go b/test/e2e/basic/resources-check/rbac.go new file mode 100644 index 000000000..c07329d81 --- /dev/null +++ b/test/e2e/basic/resources-check/rbac.go @@ -0,0 +1,188 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the Licensm. +You may obtain a copy of the License at + + http://www.apachm.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the Licensm. +*/ + +/* +Copyright 2021 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package basic + +import ( + "context" + "fmt" + "math/rand" + "strings" + "time" + + "github.com/google/uuid" + "github.com/pkg/errors" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" +) + +type RBACCase struct { + TestCase +} + +func (r *RBACCase) Init() error { + rand.Seed(time.Now().UnixNano()) + UUIDgen, _ = uuid.NewRandom() + r.BackupName = "backup-rbac" + UUIDgen.String() + r.RestoreName = "restore-rbac" + UUIDgen.String() + r.NSBaseName = "rabc-" + UUIDgen.String() + r.NamespacesTotal = 1 + r.NSIncluded = &[]string{} + for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", r.NSBaseName, nsNum) + *r.NSIncluded = append(*r.NSIncluded, createNSName) + } + r.TestMsg = &TestMSG{ + Desc: "Backup/restore of Namespaced Scoped and Cluster Scoped RBAC", + Text: "should be successfully backed up and restored", + FailedMSG: "Failed to successfully backup and restore RBAC", + } + r.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", r.BackupName, + "--include-namespaces", strings.Join(*r.NSIncluded, ","), + "--default-volumes-to-restic", "--wait", + } + + r.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", r.RestoreName, + "--from-backup", r.BackupName, "--wait", + } + r.Client = TestClientInstance + return nil +} + +func (r *RBACCase) CreateResources() error { + r.Ctx, _ = context.WithTimeout(context.Background(), 10*time.Minute) + for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", r.NSBaseName, nsNum) + fmt.Printf("Creating namespaces ...%s\n", createNSName) + if err := CreateNamespace(r.Ctx, r.Client, createNSName); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s", createNSName) + } + serviceAccountName := fmt.Sprintf("service-account-%s-%00000d", r.NSBaseName, nsNum) + fmt.Printf("Creating service account ...%s\n", createNSName) + if err := CreateServiceAccount(r.Ctx, r.Client, createNSName, serviceAccountName); err != nil { + return errors.Wrapf(err, "Failed to create service account %s", serviceAccountName) + } + clusterRoleName := fmt.Sprintf("clusterrole-%s-%00000d", r.NSBaseName, nsNum) + clusterRoleBindingName := fmt.Sprintf("clusterrolebinding-%s-%00000d", r.NSBaseName, nsNum) + if err := CreateRBACWithBindingSA(r.Ctx, r.Client, createNSName, serviceAccountName, clusterRoleName, clusterRoleBindingName); err != nil { + return errors.Wrapf(err, "Failed to create cluster role %s with role binding %s", clusterRoleName, clusterRoleBindingName) + } + } + return nil +} + +func (r *RBACCase) Verify() error { + for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ { + checkNSName := fmt.Sprintf("%s-%00000d", r.NSBaseName, nsNum) + checkServiceAccountName := fmt.Sprintf("service-account-%s-%00000d", r.NSBaseName, nsNum) + checkClusterRoleName := fmt.Sprintf("clusterrole-%s-%00000d", r.NSBaseName, nsNum) + checkClusterRoleBindingName := fmt.Sprintf("clusterrolebinding-%s-%00000d", r.NSBaseName, nsNum) + + checkNS, err := GetNamespace(r.Ctx, r.Client, checkNSName) + if err != nil { + return errors.Wrapf(err, "Could not retrieve test namespace %s", checkNSName) + } + if checkNS.Name != checkNSName { + return errors.Errorf("Retrieved namespace for %s has name %s instead", checkNSName, checkNS.Name) + } + + //getting service account from the restore + checkSA, err := GetServiceAccount(r.Ctx, r.Client, checkNSName, checkServiceAccountName) + + if err != nil { + return errors.Wrapf(err, "Could not retrieve test service account %s", checkSA) + } + + if checkSA.Name != checkServiceAccountName { + return errors.Errorf("Retrieved service account for %s has name %s instead", checkServiceAccountName, checkSA.Name) + } + + //getting cluster role from the restore + checkClusterRole, err := GetClusterRole(r.Ctx, r.Client, checkClusterRoleName) + + if err != nil { + return errors.Wrapf(err, "Could not retrieve test cluster role %s", checkClusterRole) + } + + if checkSA.Name != checkServiceAccountName { + return errors.Errorf("Retrieved cluster role for %s has name %s instead", checkClusterRoleName, checkClusterRole.Name) + } + + //getting cluster role binding from the restore + checkClusterRoleBinding, err := GetClusterRoleBinding(r.Ctx, r.Client, checkClusterRoleBindingName) + + if err != nil { + return errors.Wrapf(err, "Could not retrieve test cluster role binding %s", checkClusterRoleBinding) + } + + if checkClusterRoleBinding.Name != checkClusterRoleBindingName { + return errors.Errorf("Retrieved cluster role binding for %s has name %s instead", checkClusterRoleBindingName, checkClusterRoleBinding.Name) + } + + //check if the role binding maps to service account + checkSubjects := checkClusterRoleBinding.Subjects[0].Name + + if checkSubjects != checkServiceAccountName { + return errors.Errorf("Retrieved cluster role binding for %s has name %s instead", checkServiceAccountName, checkSubjects) + } + } + return nil +} + +func (r *RBACCase) Destroy() error { + //cleanup clusterrole + err := CleanupClusterRole(r.Ctx, r.Client, r.NSBaseName) + if err != nil { + return errors.Wrap(err, "Could not cleanup clusterroles") + } + + //cleanup cluster rolebinding + err = CleanupClusterRoleBinding(r.Ctx, r.Client, r.NSBaseName) + if err != nil { + return errors.Wrap(err, "Could not cleanup clusterrolebindings") + } + + err = CleanupNamespacesWithPoll(r.Ctx, r.Client, r.NSBaseName) + if err != nil { + return errors.Wrap(err, "Could cleanup retrieve namespaces") + } + + return nil +} + +func (r *RBACCase) Clean() error { + return r.Destroy() +} diff --git a/test/e2e/basic/resources-check/resources_check.go b/test/e2e/basic/resources-check/resources_check.go new file mode 100644 index 000000000..b3ddadcb9 --- /dev/null +++ b/test/e2e/basic/resources-check/resources_check.go @@ -0,0 +1,45 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the Licensm. +You may obtain a copy of the License at + + http://www.apachm.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the Licensm. +*/ + +/* +Copyright 2021 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package basic + +import . "github.com/vmware-tanzu/velero/test/e2e/test" + +func GetResourcesCheckTestCases() []VeleroBackupRestoreTest { + return []VeleroBackupRestoreTest{ + &NSAnnotationCase{}, + &MultiNSBackup{IsScalTest: false}, + &RBACCase{}, + } +} + +var ResourcesCheckTest func() = TestFuncWithMultiIt(GetResourcesCheckTestCases()) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 7f0712e31..0c582f600 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package e2e_test import ( "flag" @@ -23,37 +23,76 @@ import ( . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" -) -var ( - veleroCLI, veleroImage, cloudCredentialsFile, bslConfig, bslBucket, bslPrefix, vslConfig, cloudProvider, objectStoreProvider, veleroNamespace, crdsVersion string - additionalBSLProvider, additionalBSLBucket, additionalBSLPrefix, additionalBSLConfig, additionalBSLCredentials, registryCredentialFile string - installVelero bool + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/backup" + . "github.com/vmware-tanzu/velero/test/e2e/backups" + . "github.com/vmware-tanzu/velero/test/e2e/basic" + . "github.com/vmware-tanzu/velero/test/e2e/basic/resources-check" + . "github.com/vmware-tanzu/velero/test/e2e/privilegesmgmt" + . "github.com/vmware-tanzu/velero/test/e2e/resource-filtering" + . "github.com/vmware-tanzu/velero/test/e2e/scale" + . "github.com/vmware-tanzu/velero/test/e2e/upgrade" ) func init() { - flag.StringVar(&cloudProvider, "cloud-provider", "", "Cloud that Velero will be installed into. Required.") - flag.StringVar(&objectStoreProvider, "object-store-provider", "", "Provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.") - flag.StringVar(&bslBucket, "bucket", "", "name of the object storage bucket where backups from e2e tests should be stored. Required.") - flag.StringVar(&cloudCredentialsFile, "credentials-file", "", "file containing credentials for backup and volume provider. Required.") - flag.StringVar(&veleroCLI, "velerocli", "velero", "path to the velero application to use.") - flag.StringVar(&veleroImage, "velero-image", "velero/velero:main", "image for the velero server to be tested.") - flag.StringVar(&bslConfig, "bsl-config", "", "configuration to use for the backup storage location. Format is key1=value1,key2=value2") - flag.StringVar(&bslPrefix, "prefix", "", "prefix under which all Velero data should be stored within the bucket. Optional.") - flag.StringVar(&vslConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2") - flag.StringVar(&veleroNamespace, "velero-namespace", "velero", "Namespace to install Velero into") - flag.BoolVar(&installVelero, "install-velero", true, "Install/uninstall velero during the test. Optional.") - flag.StringVar(®istryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.") + flag.StringVar(&VeleroCfg.CloudProvider, "cloud-provider", "", "cloud that Velero will be installed into. Required.") + flag.StringVar(&VeleroCfg.ObjectStoreProvider, "object-store-provider", "", "provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.") + flag.StringVar(&VeleroCfg.BSLBucket, "bucket", "", "name of the object storage bucket where backups from e2e tests should be stored. Required.") + flag.StringVar(&VeleroCfg.CloudCredentialsFile, "credentials-file", "", "file containing credentials for backup and volume provider. Required.") + flag.StringVar(&VeleroCfg.VeleroCLI, "velerocli", "velero", "path to the velero application to use.") + flag.StringVar(&VeleroCfg.VeleroImage, "velero-image", "velero/velero:main", "image for the velero server to be tested.") + flag.StringVar(&VeleroCfg.Plugins, "plugins", "", "provider plugins to be tested.") + flag.StringVar(&VeleroCfg.AddBSLPlugins, "additional-bsl-plugins", "", "additional plugins to be tested.") + flag.StringVar(&VeleroCfg.VeleroVersion, "velero-version", "main", "image version for the velero server to be tested with.") + flag.StringVar(&VeleroCfg.ResticHelperImage, "restic-helper-image", "", "image for the velero restic restore helper to be tested.") + flag.StringVar(&VeleroCfg.UpgradeFromVeleroCLI, "upgrade-from-velero-cli", "", "path to the pre-upgrade velero application to use.") + flag.StringVar(&VeleroCfg.UpgradeFromVeleroVersion, "upgrade-from-velero-version", "v1.7.1", "image for the pre-upgrade velero server to be tested.") + flag.StringVar(&VeleroCfg.BSLConfig, "bsl-config", "", "configuration to use for the backup storage location. Format is key1=value1,key2=value2") + flag.StringVar(&VeleroCfg.BSLPrefix, "prefix", "", "prefix under which all Velero data should be stored within the bucket. Optional.") + flag.StringVar(&VeleroCfg.VSLConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2") + flag.StringVar(&VeleroCfg.VeleroNamespace, "velero-namespace", "velero", "namespace to install Velero into") + flag.BoolVar(&VeleroCfg.InstallVelero, "install-velero", true, "install/uninstall velero during the test. Optional.") + flag.StringVar(&VeleroCfg.RegistryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.") // Flags to create an additional BSL for multiple credentials test - flag.StringVar(&additionalBSLProvider, "additional-bsl-object-store-provider", "", "Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support.") - flag.StringVar(&additionalBSLBucket, "additional-bsl-bucket", "", "name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support.") - flag.StringVar(&additionalBSLPrefix, "additional-bsl-prefix", "", "prefix under which all Velero data should be stored within the bucket for additional backup storage location. Optional.") - flag.StringVar(&additionalBSLConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2") - flag.StringVar(&additionalBSLCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.") - flag.StringVar(&crdsVersion, "crds-version", "v1", "CRD apiVersion for velero CRD creation.") + flag.StringVar(&VeleroCfg.AdditionalBSLProvider, "additional-bsl-object-store-provider", "", "Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support.") + flag.StringVar(&VeleroCfg.AdditionalBSLBucket, "additional-bsl-bucket", "", "name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support.") + flag.StringVar(&VeleroCfg.AdditionalBSLPrefix, "additional-bsl-prefix", "", "prefix under which all Velero data should be stored within the bucket for additional backup storage location. Optional.") + flag.StringVar(&VeleroCfg.AdditionalBSLConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2") + flag.StringVar(&VeleroCfg.AdditionalBSLCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.") } +var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", APIGropuVersionsTest) + +// Test backup and restore of Kibishi using restic +var _ = Describe("[Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic) + +var _ = Describe("[Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots) + +var _ = Describe("[Basic] Backup/restore of cluster resources", ResourcesCheckTest) + +var _ = Describe("[Scale] Backup/restore of 2500 namespaces", MultiNSBackupRestore) + +// Upgrade test by Kibishi using restic +var _ = Describe("[Upgrade][Restic] Velero upgrade tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupUpgradeRestoreWithRestic) +var _ = Describe("[Upgrade][Snapshot] Velero upgrade tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupUpgradeRestoreWithSnapshots) + +// test filter objects by namespace, type, or labels when backup or restore. +var _ = Describe("[ResourceFiltering][ExcludeFromBackup] Resources with the label velero.io/exclude-from-backup=true are not included in backup", ExcludeFromBackupTest) +var _ = Describe("[ResourceFiltering][ExcludeNamespaces][Backup] Velero test on exclude namespace from the cluster backup", BackupWithExcludeNamespaces) +var _ = Describe("[ResourceFiltering][ExcludeNamespaces][Restore] Velero test on exclude namespace from the cluster restore", RestoreWithExcludeNamespaces) +var _ = Describe("[ResourceFiltering][ExcludeResources][Backup] Velero test on exclude resources from the cluster backup", BackupWithExcludeResources) +var _ = Describe("[ResourceFiltering][ExcludeResources][Restore] Velero test on exclude resources from the cluster restore", RestoreWithExcludeResources) +var _ = Describe("[ResourceFiltering][IncludeNamespaces][Backup] Velero test on include namespace from the cluster backup", BackupWithIncludeNamespaces) +var _ = Describe("[ResourceFiltering][IncludeNamespaces][Restore] Velero test on include namespace from the cluster restore", RestoreWithIncludeNamespaces) +var _ = Describe("[ResourceFiltering][IncludeResources][Backup] Velero test on include resources from the cluster backup", BackupWithIncludeResources) +var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on include resources from the cluster restore", RestoreWithIncludeResources) +var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector) +var _ = Describe("[Backups][Deletion] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", Backup_deletion_with_restic) +var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest) +var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest) + func TestE2e(t *testing.T) { // Skip running E2E tests when running only "short" tests because: // 1. E2E tests are long running tests involving installation of Velero and performing backup and restore operations. diff --git a/test/e2e/multiple_namespaces_test.go b/test/e2e/multiple_namespaces_test.go deleted file mode 100644 index 8a1673b80..000000000 --- a/test/e2e/multiple_namespaces_test.go +++ /dev/null @@ -1,158 +0,0 @@ -package e2e - -import ( - "context" - "flag" - "fmt" - "strings" - "time" - - "github.com/google/uuid" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/pkg/errors" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -var _ = Describe("[Basic] Backup/restore of 2 namespaces", func() { - - client, err := newTestClient() - Expect(err).To(Succeed(), "Failed to instantiate cluster client for multiple namespace tests") - - BeforeEach(func() { - var err error - flag.Parse() - uuidgen, err = uuid.NewRandom() - Expect(err).To(Succeed()) - if installVelero { - Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, false, - cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed()) - } - }) - - AfterEach(func() { - if installVelero { - err := veleroUninstall(context.Background(), veleroCLI, veleroNamespace) - Expect(err).To(Succeed()) - } - - }) - - Context("When I create 2 namespaces", func() { - It("should be successfully backed up and restored", func() { - backupName := "backup-" + uuidgen.String() - restoreName := "restore-" + uuidgen.String() - fiveMinTimeout, _ := context.WithTimeout(context.Background(), 5*time.Minute) - Expect(RunMultipleNamespaceTest(fiveMinTimeout, client, "nstest-"+uuidgen.String(), 2, - backupName, restoreName)).To(Succeed(), "Failed to successfully backup and restore multiple namespaces") - }) - }) -}) - -var _ = Describe("[Scale] Backup/restore of 2500 namespaces", func() { - - client, err := newTestClient() - Expect(err).To(Succeed(), "Failed to instantiate cluster client for multiple namespace tests") - - BeforeEach(func() { - var err error - flag.Parse() - uuidgen, err = uuid.NewRandom() - Expect(err).To(Succeed()) - if installVelero { - Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, false, - cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed()) - } - }) - - AfterEach(func() { - if installVelero { - err := veleroUninstall(context.Background(), veleroCLI, veleroNamespace) - Expect(err).To(Succeed()) - } - - }) - - Context("When I create 2500 namespaces", func() { - It("should be successfully backed up and restored", func() { - backupName := "backup-" + uuidgen.String() - restoreName := "restore-" + uuidgen.String() - twoHourTimeout, _ := context.WithTimeout(context.Background(), 2*time.Hour) - Expect(RunMultipleNamespaceTest(twoHourTimeout, client, "nstest-"+uuidgen.String(), 2500, - backupName, restoreName)).To(Succeed(), "Failed to successfully backup and restore multiple namespaces") - }) - }) -}) - -func RunMultipleNamespaceTest(ctx context.Context, client testClient, nsBaseName string, numberOfNamespaces int, backupName string, restoreName string) error { - defer cleanupNamespaces(ctx, client, nsBaseName) // Run at exit for final cleanup - var excludeNamespaces []string - - // Currently it's hard to build a large list of namespaces to include and wildcards do not work so instead - // we will exclude all of the namespaces that existed prior to the test from the backup - namespaces, err := client.clientGo.CoreV1().Namespaces().List(ctx, v1.ListOptions{}) - if err != nil { - return errors.Wrap(err, "Could not retrieve namespaces") - } - - for _, excludeNamespace := range namespaces.Items { - excludeNamespaces = append(excludeNamespaces, excludeNamespace.Name) - } - - fmt.Printf("Creating namespaces ...\n") - for nsNum := 0; nsNum < numberOfNamespaces; nsNum++ { - createNSName := fmt.Sprintf("%s-%00000d", nsBaseName, nsNum) - if err := createNamespace(ctx, client, createNSName); err != nil { - return errors.Wrapf(err, "Failed to create namespace %s", createNSName) - } - } - if err := veleroBackupExcludeNamespaces(ctx, veleroCLI, veleroNamespace, backupName, excludeNamespaces); err != nil { - veleroBackupLogs(ctx, veleroCLI, veleroNamespace, backupName) - return errors.Wrapf(err, "Failed to backup backup namespaces %s-*", nsBaseName) - } - - err = cleanupNamespaces(ctx, client, nsBaseName) - if err != nil { - return errors.Wrap(err, "Could cleanup retrieve namespaces") - } - - err = veleroRestore(ctx, veleroCLI, veleroNamespace, restoreName, backupName) - if err != nil { - veleroRestoreLogs(ctx, veleroCLI, veleroNamespace, restoreName) - return errors.Wrap(err, "Restore failed") - } - - // Verify that we got back all of the namespaces we created - for nsNum := 0; nsNum < numberOfNamespaces; nsNum++ { - checkNSName := fmt.Sprintf("%s-%00000d", nsBaseName, nsNum) - checkNS, err := getNamespace(ctx, client, checkNSName) - if err != nil { - return errors.Wrapf(err, "Could not retrieve test namespace %s", checkNSName) - } - if checkNS.Name != checkNSName { - return errors.Errorf("Retrieved namespace for %s has name %s instead", checkNSName, checkNS.Name) - } - } - // Cleanup is automatic on the way out - return nil -} - -func cleanupNamespaces(ctx context.Context, client testClient, nsBaseName string) error { - namespaces, err := client.clientGo.CoreV1().Namespaces().List(ctx, v1.ListOptions{}) - if err != nil { - return errors.Wrap(err, "Could not retrieve namespaces") - } - - fmt.Printf("Cleaning up namespaces ...\n") - for _, checkNamespace := range namespaces.Items { - if strings.HasPrefix(checkNamespace.Name, nsBaseName) { - err = client.clientGo.CoreV1().Namespaces().Delete(ctx, checkNamespace.Name, v1.DeleteOptions{}) - if err != nil { - return errors.Wrapf(err, "Could not delete namespace %s", checkNamespace.Name) - } - } - } - return nil -} diff --git a/test/e2e/namespace.go b/test/e2e/namespace.go deleted file mode 100644 index 7ac2efdac..000000000 --- a/test/e2e/namespace.go +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "context" - "fmt" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - - corev1api "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - waitutil "k8s.io/apimachinery/pkg/util/wait" - - "github.com/vmware-tanzu/velero/pkg/builder" -) - -func createNamespace(ctx context.Context, client testClient, namespace string) error { - ns := builder.ForNamespace(namespace).Result() - _, err := client.clientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) - if apierrors.IsAlreadyExists(err) { - return nil - } - return err -} - -func getNamespace(ctx context.Context, client testClient, namespace string) (*corev1api.Namespace, error) { - return client.clientGo.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) -} - -func deleteNamespace(ctx context.Context, client testClient, namespace string, wait bool) error { - if err := client.clientGo.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}); err != nil { - return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) - } - if !wait { - return nil - } - - return waitutil.PollImmediateInfinite(5*time.Second, - func() (bool, error) { - if _, err := client.clientGo.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}); err != nil { - if apierrors.IsNotFound(err) { - return true, nil - } - return false, err - } - logrus.Debugf("namespace %q is still being deleted...", namespace) - return false, nil - }) -} diff --git a/test/e2e/privilegesmgmt/ssr.go b/test/e2e/privilegesmgmt/ssr.go new file mode 100644 index 000000000..91af0f8b8 --- /dev/null +++ b/test/e2e/privilegesmgmt/ssr.go @@ -0,0 +1,128 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package privilegesmgmt + +import ( + "context" + "flag" + "fmt" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" + waitutil "k8s.io/apimachinery/pkg/util/wait" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + + v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" +) + +func SSRTest() { + testNS := "ssr-test" + client, err := NewTestClient() + if err != nil { + println(err.Error()) + } + Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + + BeforeEach(func() { + flag.Parse() + if VeleroCfg.InstallVelero { + Expect(VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed()) + } + }) + + AfterEach(func() { + if VeleroCfg.InstallVelero { + Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) + } + }) + + It(fmt.Sprintf("Should create an ssr object in the %s namespace and later removed by controller", VeleroCfg.VeleroNamespace), func() { + defer DeleteNamespace(context.TODO(), client, testNS, false) + ctx, _ := context.WithTimeout(context.Background(), time.Duration(time.Minute*10)) + By(fmt.Sprintf("Create %s namespace", testNS)) + Expect(CreateNamespace(ctx, client, testNS)).To(Succeed(), + fmt.Sprintf("Failed to create %s namespace", testNS)) + + By(fmt.Sprintf("Get version in %s namespace", testNS)) + cmd := []string{"version", "-n", testNS} + Expect(VeleroCmdExec(context.Background(), VeleroCfg.VeleroCLI, cmd)).To(Succeed(), + fmt.Sprintf("Failed to create an ssr object in the %s namespace", testNS)) + + By(fmt.Sprintf("Get version in %s namespace", VeleroCfg.VeleroNamespace)) + cmd = []string{"version", "-n", VeleroCfg.VeleroNamespace} + Expect(VeleroCmdExec(context.Background(), VeleroCfg.VeleroCLI, cmd)).To(Succeed(), + fmt.Sprintf("Failed to create an ssr object in %s namespace", VeleroCfg.VeleroNamespace)) + + ssrListResp := new(v1.ServerStatusRequestList) + By(fmt.Sprintf("Check ssr object in %s namespace", VeleroCfg.VeleroNamespace)) + err = waitutil.PollImmediate(5*time.Second, time.Minute, + func() (bool, error) { + if err = client.Kubebuilder.List(ctx, ssrListResp, &kbclient.ListOptions{Namespace: VeleroCfg.VeleroNamespace}); err != nil { + return false, fmt.Errorf("failed to list ssr object in %s namespace with err %v", VeleroCfg.VeleroNamespace, err) + } + if len(ssrListResp.Items) != 1 { + return false, fmt.Errorf("count of ssr object in %s namespace is not 1", VeleroCfg.VeleroNamespace) + } + + if ssrListResp.Items[0].Status.ServerVersion == "" { + fmt.Printf("ServerVersion of ssr object in %s namespace should not empty, current response result %v\n", VeleroCfg.VeleroNamespace, ssrListResp) + return false, nil + } + + if ssrListResp.Items[0].Status.Phase != "Processed" { + return false, fmt.Errorf("phase of ssr object in %s namespace should be Processed but got phase %s", VeleroCfg.VeleroNamespace, ssrListResp.Items[0].Status.Phase) + } + return true, nil + }) + if err == waitutil.ErrWaitTimeout { + fmt.Printf("exceed test case deadline and failed to check ssr object in %s namespace", VeleroCfg.VeleroNamespace) + } + Expect(err).To(Succeed(), fmt.Sprintf("Failed to check ssr object in %s namespace", VeleroCfg.VeleroNamespace)) + + By(fmt.Sprintf("Check ssr object in %s namespace", testNS)) + Expect(client.Kubebuilder.List(ctx, ssrListResp, &kbclient.ListOptions{Namespace: testNS})).To(Succeed(), + fmt.Sprintf("Failed to list ssr object in %s namespace", testNS)) + Expect(len(ssrListResp.Items)).To(BeNumerically("==", 1), + fmt.Sprintf("Count of ssr object in %s namespace is not 1", testNS)) + Expect(ssrListResp.Items[0].Status.Phase).To(BeEmpty(), + fmt.Sprintf("Status of ssr object in %s namespace should be empty", testNS)) + Expect(ssrListResp.Items[0].Status.ServerVersion).To(BeEmpty(), + fmt.Sprintf("ServerVersion of ssr object in %s namespace should be empty", testNS)) + + By(fmt.Sprintf("Waiting ssr object in %s namespace deleted", VeleroCfg.VeleroNamespace)) + err = waitutil.PollImmediateInfinite(5*time.Second, + func() (bool, error) { + if err = client.Kubebuilder.List(ctx, ssrListResp, &kbclient.ListOptions{Namespace: VeleroCfg.VeleroNamespace}); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + if len(ssrListResp.Items) != 0 { + return false, nil + } + return true, nil + }) + + Expect(err).To(Succeed(), fmt.Sprintf("ssr object in %s namespace is not been deleted by controller", VeleroCfg.VeleroNamespace)) + }) +} diff --git a/test/e2e/resource-filtering/base.go b/test/e2e/resource-filtering/base.go new file mode 100644 index 000000000..fcfd1fc37 --- /dev/null +++ b/test/e2e/resource-filtering/base.go @@ -0,0 +1,159 @@ +/* +Copyright 2021 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filtering + +import ( + "context" + "fmt" + "math/rand" + "time" + + "github.com/google/uuid" + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" +) + +type FilteringCase struct { + TestCase + IsTestInBackup bool + replica int32 + labels map[string]string + labelSelector string +} + +var testInBackup = FilteringCase{IsTestInBackup: true} +var testInRestore = FilteringCase{IsTestInBackup: false} + +func (f *FilteringCase) Init() error { + rand.Seed(time.Now().UnixNano()) + UUIDgen, _ = uuid.NewRandom() + f.replica = int32(2) + f.labels = map[string]string{"resourcefiltering": "true"} + f.labelSelector = "resourcefiltering" + f.Client = TestClientInstance + + f.NamespacesTotal = 3 + f.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", f.BackupName, + "--default-volumes-to-restic", "--wait", + } + + f.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", f.RestoreName, + "--from-backup", f.BackupName, "--wait", + } + + f.NSIncluded = &[]string{} + return nil +} + +func (f *FilteringCase) CreateResources() error { + f.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute) + for nsNum := 0; nsNum < f.NamespacesTotal; nsNum++ { + namespace := fmt.Sprintf("%s-%00000d", f.NSBaseName, nsNum) + fmt.Printf("Creating resources in namespace ...%s\n", namespace) + if err := CreateNamespace(f.Ctx, f.Client, namespace); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s", namespace) + } + serviceAccountName := "default" + // wait until the service account is created before patch the image pull secret + if err := WaitUntilServiceAccountCreated(f.Ctx, f.Client, namespace, serviceAccountName, 10*time.Minute); err != nil { + return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, namespace) + } + // add the image pull secret to avoid the image pull limit issue of Docker Hub + if err := PatchServiceAccountWithImagePullSecret(f.Ctx, f.Client, namespace, serviceAccountName, VeleroCfg.RegistryCredentialFile); err != nil { + return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, namespace) + } + //Create deployment + fmt.Printf("Creating deployment in namespaces ...%s\n", namespace) + deployment := NewDeployment(f.NSBaseName, namespace, f.replica, f.labels) + deployment, err := CreateDeployment(f.Client.ClientGo, namespace, deployment) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) + } + err = WaitForReadyDeployment(f.Client.ClientGo, namespace, deployment.Name) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to ensure job completion in namespace: %q", namespace)) + } + //Create Secret + secretName := f.NSBaseName + fmt.Printf("Creating secret %s in namespaces ...%s\n", secretName, namespace) + _, err = CreateSecret(f.Client.ClientGo, namespace, secretName, f.labels) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to create secret in the namespace %q", namespace)) + } + err = WaitForSecretsComplete(f.Client.ClientGo, namespace, secretName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to ensure secret completion in namespace: %q", namespace)) + } + //Create Configmap + configmaptName := f.NSBaseName + fmt.Printf("Creating configmap %s in namespaces ...%s\n", configmaptName, namespace) + _, err = CreateConfigMap(f.Client.ClientGo, namespace, configmaptName, f.labels) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to create configmap in the namespace %q", namespace)) + } + err = WaitForConfigMapComplete(f.Client.ClientGo, namespace, configmaptName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to ensure secret completion in namespace: %q", namespace)) + } + } + return nil +} + +func (f *FilteringCase) Verify() error { + for nsNum := 0; nsNum < f.NamespacesTotal; nsNum++ { + namespace := fmt.Sprintf("%s-%00000d", f.NSBaseName, nsNum) + fmt.Printf("Checking resources in namespaces ...%s\n", namespace) + //Check namespace + checkNS, err := GetNamespace(f.Ctx, f.Client, namespace) + if err != nil { + return errors.Wrapf(err, "Could not retrieve test namespace %s", namespace) + } + if checkNS.Name != namespace { + return errors.Errorf("Retrieved namespace for %s has name %s instead", namespace, checkNS.Name) + } + //Check deployment + _, err = GetDeployment(f.Client.ClientGo, namespace, f.NSBaseName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) + } + + //Check secrets + secretsList, err := f.Client.ClientGo.CoreV1().Secrets(namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: f.labelSelector}) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to list secrets in namespace: %q", namespace)) + } else if len(secretsList.Items) == 0 { + return errors.Wrap(err, fmt.Sprintf("no secrets found in namespace: %q", namespace)) + } + + //Check configmap + configmapList, err := f.Client.ClientGo.CoreV1().ConfigMaps(namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: f.labelSelector}) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to list configmap in namespace: %q", namespace)) + } else if len(configmapList.Items) == 0 { + return errors.Wrap(err, fmt.Sprintf("no configmap found in namespace: %q", namespace)) + } + } + return nil +} diff --git a/test/e2e/resource-filtering/exclude_label.go b/test/e2e/resource-filtering/exclude_label.go new file mode 100644 index 000000000..72307fe3f --- /dev/null +++ b/test/e2e/resource-filtering/exclude_label.go @@ -0,0 +1,137 @@ +/* +Copyright 2021 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filtering + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" +) + +/* +Resources with the label velero.io/exclude-from-backup=true are not included +in backup, even if it contains a matching selector label. +*/ + +type ExcludeFromBackup struct { + FilteringCase +} + +var ExcludeFromBackupTest func() = TestFunc(&ExcludeFromBackup{testInBackup}) + +func (e *ExcludeFromBackup) Init() error { + e.FilteringCase.Init() + e.BackupName = "backup-exclude-from-backup-" + UUIDgen.String() + e.RestoreName = "restore-" + UUIDgen.String() + e.NSBaseName = "exclude-from-backup-" + UUIDgen.String() + e.TestMsg = &TestMSG{ + Desc: "Backup with the label velero.io/exclude-from-backup=true are not included test", + Text: "Should not backup resources with the label velero.io/exclude-from-backup=true", + FailedMSG: "Failed to backup resources with the label velero.io/exclude-from-backup=true", + } + for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) + *e.NSIncluded = append(*e.NSIncluded, createNSName) + } + e.labels = map[string]string{ + "velero.io/exclude-from-backup": "true", + } + e.labelSelector = "velero.io/exclude-from-backup" + + e.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", e.BackupName, + "--include-namespaces", strings.Join(*e.NSIncluded, ","), + "--default-volumes-to-restic", "--wait", + } + + e.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", e.RestoreName, + "--from-backup", e.BackupName, "--wait", + } + return nil +} + +func (e *ExcludeFromBackup) CreateResources() error { + e.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute) + for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { + namespace := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) + fmt.Printf("Creating resources in namespace ...%s\n", namespace) + labels := e.labels + if nsNum%2 == 0 { + labels = map[string]string{ + "velero.io/exclude-from-backup": "false", + } + } + if err := CreateNamespaceWithLabel(e.Ctx, e.Client, namespace, labels); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s", namespace) + } + serviceAccountName := "default" + // wait until the service account is created before patch the image pull secret + if err := WaitUntilServiceAccountCreated(e.Ctx, e.Client, namespace, serviceAccountName, 10*time.Minute); err != nil { + return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, namespace) + } + // add the image pull secret to avoid the image pull limit issue of Docker Hub + if err := PatchServiceAccountWithImagePullSecret(e.Ctx, e.Client, namespace, serviceAccountName, VeleroCfg.RegistryCredentialFile); err != nil { + return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, namespace) + } + //Create deployment + fmt.Printf("Creating deployment in namespaces ...%s\n", namespace) + + deployment := NewDeployment(e.NSBaseName, namespace, e.replica, labels) + deployment, err := CreateDeployment(e.Client.ClientGo, namespace, deployment) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) + } + err = WaitForReadyDeployment(e.Client.ClientGo, namespace, deployment.Name) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to ensure deployment completion in namespace: %q", namespace)) + } + } + return nil +} + +func (e *ExcludeFromBackup) Verify() error { + for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { + namespace := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) + fmt.Printf("Checking resources in namespaces ...%s\n", namespace) + //Check deployment + _, err := GetDeployment(e.Client.ClientGo, namespace, e.NSBaseName) + if nsNum%2 == 0 { //include + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) + } + } else { //exclude + if err == nil { + return fmt.Errorf("failed to exclude deployment in namespaces %q", namespace) + } else { + if apierrors.IsNotFound(err) { //resource should be excluded + return nil + } + return errors.Wrap(err, fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) + } + } + } + return nil +} diff --git a/test/e2e/resource-filtering/exclude_namespaces.go b/test/e2e/resource-filtering/exclude_namespaces.go new file mode 100644 index 000000000..b76b15fb1 --- /dev/null +++ b/test/e2e/resource-filtering/exclude_namespaces.go @@ -0,0 +1,148 @@ +/* +Copyright 2021 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filtering + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" +) + +/* +exclude-namespaces +Exclude namespace1 from the cluster backup. +velero backup create --exclude-namespaces + +Exclude two namespaces during a restore. +velero restore create --exclude-namespaces , +*/ + +type ExcludeNamespaces struct { + FilteringCase + nsExcluded *[]string + namespacesExcluded int +} + +var BackupWithExcludeNamespaces func() = TestFunc(&ExcludeNamespaces{FilteringCase: testInBackup}) +var RestoreWithExcludeNamespaces func() = TestFunc(&ExcludeNamespaces{FilteringCase: testInRestore}) + +func (e *ExcludeNamespaces) Init() error { + e.FilteringCase.Init() + e.namespacesExcluded = e.NamespacesTotal / 2 + e.NSBaseName = "exclude-namespaces-" + UUIDgen.String() + if e.IsTestInBackup { + e.BackupName = "backup-exclude-namespaces-" + UUIDgen.String() + e.RestoreName = "restore-" + UUIDgen.String() + e.TestMsg = &TestMSG{ + Desc: "Backup resources with exclude namespace test", + FailedMSG: "Failed to backup and restore with namespace include", + Text: fmt.Sprintf("should not backup %d namespaces of %d", e.namespacesExcluded, e.NamespacesTotal), + } + } else { + e.BackupName = "backup-" + UUIDgen.String() + e.RestoreName = "restore-exclude-namespaces-" + UUIDgen.String() + e.TestMsg = &TestMSG{ + Desc: "Restore resources with exclude namespace test", + FailedMSG: "Failed to restore with namespace exclude", + Text: fmt.Sprintf("should not backup %d namespaces of %d", e.namespacesExcluded, e.NamespacesTotal), + } + } + e.nsExcluded = &[]string{} + for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) + if nsNum < e.namespacesExcluded { + *e.nsExcluded = append(*e.nsExcluded, createNSName) + } else { + *e.NSIncluded = append(*e.NSIncluded, createNSName) + } + } + if e.IsTestInBackup { + e.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", e.BackupName, + "--exclude-namespaces", strings.Join(*e.nsExcluded, ","), + "--include-namespaces", strings.Join(*e.NSIncluded, ","), + "--default-volumes-to-restic", "--wait", + } + + e.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", e.RestoreName, + "--from-backup", e.BackupName, "--wait", + } + + } else { + *e.NSIncluded = append(*e.NSIncluded, *e.nsExcluded...) + e.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", e.BackupName, + "--include-namespaces", strings.Join(*e.NSIncluded, ","), + "--default-volumes-to-restic", "--wait", + } + + e.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", e.RestoreName, + "--exclude-namespaces", strings.Join(*e.nsExcluded, ","), + "--from-backup", e.BackupName, "--wait", + } + } + return nil +} + +func (e *ExcludeNamespaces) CreateResources() error { + e.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute) + for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) + fmt.Printf("Creating namespaces ...%s\n", createNSName) + if err := CreateNamespace(e.Ctx, e.Client, createNSName); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s", createNSName) + } + } + return nil +} + +func (e *ExcludeNamespaces) Verify() error { + // Verify that we got back all of the namespaces we created + for nsNum := 0; nsNum < e.namespacesExcluded; nsNum++ { + excludeNSName := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) + _, err := GetNamespace(e.Ctx, e.Client, excludeNSName) + if err == nil { + return errors.Wrapf(err, "Resource filtering with exclude namespace but exclude namespace %s exist", excludeNSName) + } + + if !apierrors.IsNotFound(err) { + return errors.Wrapf(err, "Resource filtering with exclude namespace failed with checking namespace %s", excludeNSName) + } + } + + for nsNum := e.namespacesExcluded; nsNum < e.NamespacesTotal; nsNum++ { + checkNSName := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) + checkNS, err := GetNamespace(e.Ctx, e.Client, checkNSName) + if err != nil { + return errors.Wrapf(err, "Could not retrieve test namespace %s", checkNSName) + } + if checkNS.Name != checkNSName { + return errors.Errorf("Retrieved namespace for %s has name %s instead", checkNSName, checkNS.Name) + } + } + return nil +} diff --git a/test/e2e/resource-filtering/exclude_resources.go b/test/e2e/resource-filtering/exclude_resources.go new file mode 100644 index 000000000..1080ee70f --- /dev/null +++ b/test/e2e/resource-filtering/exclude_resources.go @@ -0,0 +1,129 @@ +/* +Copyright 2021 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filtering + +import ( + "context" + "fmt" + "strings" + + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" +) + +/* +exclude-resources +Exclude secrets from the backup. + +velero backup create --exclude-resources secrets +Exclude secrets and rolebindings. + +velero backup create --exclude-resources secrets +*/ + +type ExcludeResources struct { + FilteringCase +} + +var BackupWithExcludeResources func() = TestFunc(&ExcludeResources{testInBackup}) +var RestoreWithExcludeResources func() = TestFunc(&ExcludeResources{testInRestore}) + +func (e *ExcludeResources) Init() error { + e.FilteringCase.Init() + e.NSBaseName = "exclude-resources-" + UUIDgen.String() + for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) + *e.NSIncluded = append(*e.NSIncluded, createNSName) + } + if e.IsTestInBackup { // testing case backup with exclude-resources option + e.TestMsg = &TestMSG{ + Desc: "Backup resources with resources included test", + Text: "Should not backup resources which is excluded others should be backup", + FailedMSG: "Failed to backup with resource exclude", + } + e.BackupName = "backup-exclude-resources-" + UUIDgen.String() + e.RestoreName = "restore-" + UUIDgen.String() + e.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", e.BackupName, + "--include-namespaces", strings.Join(*e.NSIncluded, ","), + "--exclude-resources", "secrets", + "--default-volumes-to-restic", "--wait", + } + + e.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", e.RestoreName, + "--from-backup", e.BackupName, "--wait", + } + } else { // testing case restore with exclude-resources option + e.BackupName = "backup-" + UUIDgen.String() + e.RestoreName = "restore-exclude-resources-" + UUIDgen.String() + e.TestMsg = &TestMSG{ + Desc: "Restore resources with resources included test", + Text: "Should not restore resources which is excluded others should be backup", + FailedMSG: "Failed to restore with resource exclude", + } + e.BackupName = "backup-exclude-resources-" + UUIDgen.String() + e.RestoreName = "restore-exclude-resources-" + UUIDgen.String() + e.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", e.BackupName, + "--include-namespaces", strings.Join(*e.NSIncluded, ","), + "--default-volumes-to-restic", "--wait", + } + e.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", e.RestoreName, + "--exclude-resources", "secrets", + "--from-backup", e.BackupName, "--wait", + } + } + return nil +} + +func (e *ExcludeResources) Verify() error { + for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { + namespace := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) + fmt.Printf("Checking resources in namespaces ...%s\n", namespace) + //Check deployment + _, err := GetDeployment(e.Client.ClientGo, namespace, e.NSBaseName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) + } + //Check secrets + secretsList, err := e.Client.ClientGo.CoreV1().Secrets(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: e.labelSelector}) + if err != nil { + if apierrors.IsNotFound(err) { //resource should be excluded + return nil + } + return errors.Wrap(err, fmt.Sprintf("failed to list secrets in namespace: %q", namespace)) + } else if len(secretsList.Items) != 0 { + return errors.Errorf(fmt.Sprintf("Should no secrets found %s in namespace: %q", secretsList.Items[0].Name, namespace)) + } + + //Check configmap + configmapList, err := e.Client.ClientGo.CoreV1().ConfigMaps(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: e.labelSelector}) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to list configmap in namespace: %q", namespace)) + } else if len(configmapList.Items) == 0 { + return errors.Errorf(fmt.Sprintf("Should have configmap found in namespace: %q", namespace)) + } + } + return nil +} diff --git a/test/e2e/resource-filtering/include_namespaces.go b/test/e2e/resource-filtering/include_namespaces.go new file mode 100644 index 000000000..1d6fdf49d --- /dev/null +++ b/test/e2e/resource-filtering/include_namespaces.go @@ -0,0 +1,144 @@ +/* +Copyright 2021 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filtering + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" +) + +/* +include-namespaces +Backup a namespace and it's objects. + +velero backup create --include-namespaces +Restore two namespaces and their objects. + +velero restore create --include-namespaces , +*/ + +type IncludeNamespaces struct { + allTestNamespaces *[]string + namespacesIncluded int + FilteringCase +} + +var BackupWithIncludeNamespaces func() = TestFunc(&IncludeNamespaces{FilteringCase: testInBackup}) +var RestoreWithIncludeNamespaces func() = TestFunc(&IncludeNamespaces{FilteringCase: testInRestore}) + +func (i *IncludeNamespaces) Init() error { + i.FilteringCase.Init() + i.namespacesIncluded = i.NamespacesTotal / 2 + i.allTestNamespaces = &[]string{} + i.NSBaseName = "include-namespaces-" + UUIDgen.String() + for nsNum := 0; nsNum < i.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", i.NSBaseName, nsNum) + if nsNum < i.namespacesIncluded { + *i.NSIncluded = append(*i.NSIncluded, createNSName) + } + *i.allTestNamespaces = append(*i.allTestNamespaces, createNSName) + } + + if i.IsTestInBackup { + i.BackupName = "backup-include-namespaces-" + UUIDgen.String() + i.RestoreName = "restore-" + UUIDgen.String() + i.TestMsg = &TestMSG{ + Desc: "Backup resources with include namespace test", + FailedMSG: "Failed to backup with namespace include", + Text: fmt.Sprintf("should backup %d namespaces of %d", i.namespacesIncluded, i.NamespacesTotal), + } + i.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", i.BackupName, + "--include-namespaces", strings.Join(*i.NSIncluded, ","), + "--default-volumes-to-restic", "--wait", + } + + i.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", i.RestoreName, + "--from-backup", i.BackupName, "--wait", + } + + } else { + i.BackupName = "backup-" + UUIDgen.String() + i.RestoreName = "restore-include-namespaces-" + UUIDgen.String() + i.TestMsg = &TestMSG{ + Desc: "Restore resources with include namespace test", + FailedMSG: "Failed to restore with namespace include", + Text: fmt.Sprintf("should restore %d namespaces of %d", i.namespacesIncluded, i.NamespacesTotal), + } + i.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", i.BackupName, + "--include-namespaces", strings.Join(*i.allTestNamespaces, ","), + "--default-volumes-to-restic", "--wait", + } + + i.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", i.RestoreName, + "--include-namespaces", strings.Join(*i.NSIncluded, ","), + "--from-backup", i.BackupName, "--wait", + } + } + return nil +} + +func (i *IncludeNamespaces) CreateResources() error { + i.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute) + for nsNum := 0; nsNum < i.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", i.NSBaseName, nsNum) + fmt.Printf("Creating namespaces ...%s\n", createNSName) + if err := CreateNamespace(i.Ctx, i.Client, createNSName); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s", createNSName) + } + } + return nil +} + +func (i *IncludeNamespaces) Verify() error { + // Verify that we got back all of the namespaces we created + for nsNum := 0; nsNum < i.namespacesIncluded; nsNum++ { + checkNSName := fmt.Sprintf("%s-%00000d", i.NSBaseName, nsNum) + checkNS, err := GetNamespace(i.Ctx, i.Client, checkNSName) + if err != nil { + return errors.Wrapf(err, "Could not retrieve test namespace %s", checkNSName) + } + if checkNS.Name != checkNSName { + return errors.Errorf("Retrieved namespace for %s has name %s instead", checkNSName, checkNS.Name) + } + } + + for nsNum := i.namespacesIncluded; nsNum < i.NamespacesTotal; nsNum++ { + excludeNSName := fmt.Sprintf("%s-%00000d", i.NSBaseName, nsNum) + _, err := GetNamespace(i.Ctx, i.Client, excludeNSName) + if err == nil { + return errors.Wrapf(err, "Resource filtering with include namespace but exclude namespace %s exist", excludeNSName) + } + + if !apierrors.IsNotFound(err) { + return errors.Wrapf(err, "Resource filtering with include namespace failed with checking namespace %s", excludeNSName) + } + } + return nil +} diff --git a/test/e2e/resource-filtering/include_resources.go b/test/e2e/resource-filtering/include_resources.go new file mode 100644 index 000000000..102991bc6 --- /dev/null +++ b/test/e2e/resource-filtering/include_resources.go @@ -0,0 +1,124 @@ +/* +Copyright 2021 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filtering + +import ( + "context" + "fmt" + "strings" + + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" +) + +/* +Backup all deployments in the cluster. + velero backup create --include-resources deployments, configmaps + +Restore all deployments and configmaps in the cluster. + velero restore create --include-resources deployments,configmaps +*/ + +type IncludeResources struct { + FilteringCase +} + +var BackupWithIncludeResources func() = TestFunc(&IncludeResources{testInBackup}) +var RestoreWithIncludeResources func() = TestFunc(&IncludeResources{testInRestore}) + +func (i *IncludeResources) Init() error { + i.FilteringCase.Init() + i.NSBaseName = "include-resources-" + UUIDgen.String() + for nsNum := 0; nsNum < i.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", i.NSBaseName, nsNum) + *i.NSIncluded = append(*i.NSIncluded, createNSName) + } + if i.IsTestInBackup { // testing case backup with include-resources option + i.TestMsg = &TestMSG{ + Desc: "Backup resources with resources included test", + Text: "Should backup resources which is included others should not be backup", + FailedMSG: "Failed to backup with resource include", + } + i.BackupName = "backup-include-resources-" + UUIDgen.String() + i.RestoreName = "restore-" + UUIDgen.String() + i.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", i.BackupName, + "--include-resources", "deployments,configmaps", + "--default-volumes-to-restic", "--wait", + } + + i.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", i.RestoreName, + "--from-backup", i.BackupName, "--wait", + } + } else { // testing case restore with include-resources option + i.TestMsg = &TestMSG{ + Desc: "Restore resources with resources included test", + Text: "Should restore resources which is included others should not be backup", + FailedMSG: "Failed to restore with resource include", + } + i.BackupName = "backup-" + UUIDgen.String() + i.RestoreName = "restore-include-resources-" + UUIDgen.String() + i.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", i.BackupName, + "--include-namespaces", strings.Join(*i.NSIncluded, ","), + "--default-volumes-to-restic", "--wait", + } + i.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", i.RestoreName, + "--include-resources", "deployments,configmaps", + "--from-backup", i.BackupName, "--wait", + } + } + return nil +} + +func (i *IncludeResources) Verify() error { + for nsNum := 0; nsNum < i.NamespacesTotal; nsNum++ { + namespace := fmt.Sprintf("%s-%00000d", i.NSBaseName, nsNum) + fmt.Printf("Checking resources in namespaces ...%s\n", namespace) + //Check deployment + _, err := GetDeployment(i.Client.ClientGo, namespace, i.NSBaseName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) + } + //Check secrets + secretsList, err := i.Client.ClientGo.CoreV1().Secrets(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: i.labelSelector}) + if err != nil { + if apierrors.IsNotFound(err) { //resource should be excluded + return nil + } + return errors.Wrap(err, fmt.Sprintf("failed to list secrets in namespace: %q", namespace)) + } else if len(secretsList.Items) != 0 { + return errors.Errorf(fmt.Sprintf("Should no secrets found %s in namespace: %q", secretsList.Items[0].Name, namespace)) + } + + //Check configmap + configmapList, err := i.Client.ClientGo.CoreV1().ConfigMaps(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: i.labelSelector}) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to list configmap in namespace: %q", namespace)) + } else if len(configmapList.Items) == 0 { + return errors.Errorf(fmt.Sprintf("Should have configmap found in namespace: %q", namespace)) + } + } + return nil +} diff --git a/test/e2e/resource-filtering/label_selector.go b/test/e2e/resource-filtering/label_selector.go new file mode 100644 index 000000000..3b0edfd3d --- /dev/null +++ b/test/e2e/resource-filtering/label_selector.go @@ -0,0 +1,172 @@ +/* +Copyright 2021 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filtering + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" +) + +/* +Include resources matching the label selector. + velero backup create --selector = +*/ + +type LabelSelector struct { + FilteringCase +} + +var BackupWithLabelSelector func() = TestFunc(&LabelSelector{testInBackup}) + +func (l *LabelSelector) Init() error { + l.FilteringCase.Init() + l.BackupName = "backup-label-selector-" + UUIDgen.String() + l.RestoreName = "restore-" + UUIDgen.String() + l.NSBaseName = "backup-label-selector-" + UUIDgen.String() + for nsNum := 0; nsNum < l.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", l.NSBaseName, nsNum) + *l.NSIncluded = append(*l.NSIncluded, createNSName) + } + l.TestMsg = &TestMSG{ + Desc: "Backup with the label selector test", + Text: "Should backup resources with selected label resource", + FailedMSG: "Failed to backup resources with selected label", + } + l.labels = map[string]string{ + "resourcefiltering": "true", + } + l.labelSelector = "resourcefiltering" + l.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", l.BackupName, + "--selector", "resourcefiltering=true", + "--include-namespaces", strings.Join(*l.NSIncluded, ","), + "--default-volumes-to-restic", "--wait", + } + + l.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", l.RestoreName, + "--from-backup", l.BackupName, "--wait", + } + return nil +} + +func (l *LabelSelector) CreateResources() error { + l.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute) + for nsNum := 0; nsNum < l.NamespacesTotal; nsNum++ { + namespace := fmt.Sprintf("%s-%00000d", l.NSBaseName, nsNum) + fmt.Printf("Creating resources in namespace ...%s\n", namespace) + labels := l.labels + if nsNum%2 == 0 { + labels = map[string]string{ + "resourcefiltering": "false", + } + } + if err := CreateNamespaceWithLabel(l.Ctx, l.Client, namespace, labels); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s", namespace) + } + + serviceAccountName := "default" + // wait until the service account is created before patch the image pull secret + if err := WaitUntilServiceAccountCreated(l.Ctx, l.Client, namespace, serviceAccountName, 10*time.Minute); err != nil { + return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, namespace) + } + // add the image pull secret to avoid the image pull limit issue of Docker Hub + if err := PatchServiceAccountWithImagePullSecret(l.Ctx, l.Client, namespace, serviceAccountName, VeleroCfg.RegistryCredentialFile); err != nil { + return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, namespace) + } + //Create deployment + fmt.Printf("Creating deployment in namespaces ...%s\n", namespace) + + deployment := NewDeployment(l.NSBaseName, namespace, l.replica, labels) + deployment, err := CreateDeployment(l.Client.ClientGo, namespace, deployment) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) + } + err = WaitForReadyDeployment(l.Client.ClientGo, namespace, deployment.Name) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to ensure job completion in namespace: %q", namespace)) + } + //Create Secret + secretName := l.NSBaseName + fmt.Printf("Creating secret %s in namespaces ...%s\n", secretName, namespace) + _, err = CreateSecret(l.Client.ClientGo, namespace, secretName, l.labels) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to create secret in the namespace %q", namespace)) + } + err = WaitForSecretsComplete(l.Client.ClientGo, namespace, secretName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to ensure secret completion in namespace: %q", namespace)) + } + } + return nil +} + +func (l *LabelSelector) Verify() error { + for nsNum := 0; nsNum < l.NamespacesTotal; nsNum++ { + namespace := fmt.Sprintf("%s-%00000d", l.NSBaseName, nsNum) + fmt.Printf("Checking resources in namespaces ...%s\n", namespace) + //Check deployment + _, err := GetDeployment(l.Client.ClientGo, namespace, l.NSBaseName) + if nsNum%2 == 1 { //include + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) + } + } else { //exclude + if err == nil { + return fmt.Errorf("failed to exclude deployment in namespaces %q", namespace) + } else { + if apierrors.IsNotFound(err) { //resource should be excluded + return nil + } + return errors.Wrap(err, fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) + } + } + + //Check secrets + secretsList, err := l.Client.ClientGo.CoreV1().Secrets(namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: l.labelSelector, + }) + + if nsNum%2 == 0 { //include + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to list secrets in namespace: %q", namespace)) + } else if len(secretsList.Items) == 0 { + return errors.Errorf(fmt.Sprintf("no secrets found in namespace: %q", namespace)) + } + } else { //exclude + if err == nil { + return fmt.Errorf("failed to exclude secrets in namespaces %q", namespace) + } else { + if apierrors.IsNotFound(err) { //resource should be excluded + return nil + } + return errors.Wrap(err, fmt.Sprintf("failed to list secrets in namespace: %q", namespace)) + } + } + } + return nil +} diff --git a/test/e2e/scale/multiple_namespaces.go b/test/e2e/scale/multiple_namespaces.go new file mode 100644 index 000000000..d63880c6b --- /dev/null +++ b/test/e2e/scale/multiple_namespaces.go @@ -0,0 +1,24 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scale + +import ( + basic "github.com/vmware-tanzu/velero/test/e2e/basic/resources-check" + . "github.com/vmware-tanzu/velero/test/e2e/test" +) + +var MultiNSBackupRestore func() = TestFunc(&basic.MultiNSBackup{IsScalTest: true}) diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go new file mode 100644 index 000000000..5648c06de --- /dev/null +++ b/test/e2e/test/test.go @@ -0,0 +1,213 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "context" + "flag" + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/pkg/errors" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" +) + +/* +The VeleroBackupRestoreTest interface is just could be suit for the cases that follow the test flow of +create resources, backup, delete test resource, restore and verify. +And the cases have similar execute function and similar data. it's both fine for you to use it or not which +depends on your test patterns. +*/ +type VeleroBackupRestoreTest interface { + Init() error + StartRun() error + CreateResources() error + Backup() error + Destroy() error + Restore() error + Verify() error + Clean() error + GetTestMsg() *TestMSG +} + +type TestMSG struct { + Desc string + Text string + FailedMSG string +} + +type TestCase struct { + BackupName string + RestoreName string + NSBaseName string + BackupArgs []string + RestoreArgs []string + NamespacesTotal int + TestMsg *TestMSG + Client TestClient + Ctx context.Context + NSIncluded *[]string +} + +var TestClientInstance TestClient + +func TestFunc(test VeleroBackupRestoreTest) func() { + return func() { + var err error + TestClientInstance, err = NewTestClient() + Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + Expect(test.Init()).To(Succeed(), "Failed to instantiate test cases") + BeforeEach(func() { + flag.Parse() + if VeleroCfg.InstallVelero { + Expect(VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed()) + } + }) + AfterEach(func() { + if VeleroCfg.InstallVelero { + Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To((Succeed())) + } + }) + It(test.GetTestMsg().Text, func() { + Expect(RunTestCase(test)).To(Succeed(), test.GetTestMsg().FailedMSG) + }) + } +} + +func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() { + return func() { + var err error + var countIt int + TestClientInstance, err = NewTestClient() + Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + for k := range tests { + Expect(tests[k].Init()).To(Succeed(), fmt.Sprintf("Failed to instantiate test %s case", tests[k].GetTestMsg().Desc)) + } + + BeforeEach(func() { + flag.Parse() + if VeleroCfg.InstallVelero { + if countIt == 0 { + Expect(VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed()) + } + countIt++ + } + }) + + AfterEach(func() { + if VeleroCfg.InstallVelero { + if countIt == len(tests) { + Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To((Succeed())) + } + } + }) + + for k := range tests { + curTest := tests[k] + It(curTest.GetTestMsg().Text, func() { + Expect(RunTestCase(curTest)).To(Succeed(), curTest.GetTestMsg().FailedMSG) + }) + } + } +} + +func (t *TestCase) Init() error { + return nil +} + +func (t *TestCase) CreateResources() error { + return nil +} + +func (t *TestCase) StartRun() error { + return nil +} + +func (t *TestCase) Backup() error { + if err := VeleroCmdExec(t.Ctx, VeleroCfg.VeleroCLI, t.BackupArgs); err != nil { + RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.BackupName, "") + return errors.Wrapf(err, "Failed to backup resources") + } + return nil +} + +func (t *TestCase) Destroy() error { + err := CleanupNamespacesWithPoll(t.Ctx, t.Client, t.NSBaseName) + if err != nil { + return errors.Wrap(err, "Could cleanup retrieve namespaces") + } + return nil +} + +func (t *TestCase) Restore() error { + if err := VeleroCmdExec(t.Ctx, VeleroCfg.VeleroCLI, t.RestoreArgs); err != nil { + RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.BackupName, "") + return errors.Wrapf(err, "Failed to restore resources") + } + return nil +} + +func (t *TestCase) Verify() error { + return nil +} + +func (t *TestCase) Clean() error { + return CleanupNamespaces(t.Ctx, t.Client, t.NSBaseName) +} + +func (t *TestCase) GetTestMsg() *TestMSG { + return t.TestMsg +} + +func RunTestCase(test VeleroBackupRestoreTest) error { + fmt.Printf("Running test case %s\n", test.GetTestMsg().Desc) + if test == nil { + return errors.New("No case should be tested") + } + + defer test.Clean() + err := test.StartRun() + if err != nil { + return err + } + err = test.CreateResources() + if err != nil { + return err + } + err = test.Backup() + if err != nil { + return err + } + err = test.Destroy() + if err != nil { + return err + } + err = test.Restore() + if err != nil { + return err + } + err = test.Verify() + if err != nil { + return err + } + return nil +} diff --git a/test/e2e/types.go b/test/e2e/types.go new file mode 100644 index 000000000..b8c214434 --- /dev/null +++ b/test/e2e/types.go @@ -0,0 +1,51 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "github.com/google/uuid" +) + +var UUIDgen uuid.UUID + +var VeleroCfg VerleroConfig + +type VerleroConfig struct { + VeleroCLI string + VeleroImage string + VeleroVersion string + CloudCredentialsFile string + BSLConfig string + BSLBucket string + BSLPrefix string + VSLConfig string + CloudProvider string + ObjectStoreProvider string + VeleroNamespace string + AdditionalBSLProvider string + AdditionalBSLBucket string + AdditionalBSLPrefix string + AdditionalBSLConfig string + AdditionalBSLCredentials string + RegistryCredentialFile string + ResticHelperImage string + UpgradeFromVeleroVersion string + UpgradeFromVeleroCLI string + Plugins string + AddBSLPlugins string + InstallVelero bool +} diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go new file mode 100644 index 000000000..7a7d818a8 --- /dev/null +++ b/test/e2e/upgrade/upgrade.go @@ -0,0 +1,174 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package upgrade + +import ( + "context" + "flag" + "fmt" + "time" + + "github.com/google/uuid" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/pkg/errors" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/kibishii" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" +) + +const ( + upgradeNamespace = "upgrade-workload" +) + +func BackupUpgradeRestoreWithSnapshots() { + BackupUpgradeRestoreTest(true) +} + +func BackupUpgradeRestoreWithRestic() { + BackupUpgradeRestoreTest(false) +} + +func BackupUpgradeRestoreTest(useVolumeSnapshots bool) { + var ( + backupName, restoreName, upgradeFromVeleroCLI string + ) + + client, err := NewTestClient() + Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + + BeforeEach(func() { + if (len(VeleroCfg.UpgradeFromVeleroVersion)) == 0 { + Skip("An original velero version is required to run upgrade test, please run test with upgrade-from-velero-version=") + } + if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" { + Skip("Volume snapshots not supported on kind") + } + + var err error + flag.Parse() + UUIDgen, err = uuid.NewRandom() + Expect(err).To(Succeed()) + if VeleroCfg.InstallVelero { + //Set VeleroImage and ResticHelperImage to blank + //VeleroImage and ResticHelperImage should be the default value in originalCli + tmpCfg := VeleroCfg + tmpCfg.VeleroImage = "" + tmpCfg.ResticHelperImage = "" + tmpCfg.Plugins = "" + //Assume tag of velero server image is identical to velero CLI version + //Download velero CLI if it's empty according to velero CLI version + if (len(VeleroCfg.UpgradeFromVeleroCLI)) == 0 { + tmpCfg.VeleroCLI, err = InstallVeleroCLI(VeleroCfg.UpgradeFromVeleroVersion) + upgradeFromVeleroCLI = tmpCfg.VeleroCLI + Expect(err).To(Succeed()) + } + Expect(VeleroInstall(context.Background(), &tmpCfg, "", useVolumeSnapshots)).To(Succeed()) + Expect(CheckVeleroVersion(context.Background(), tmpCfg.VeleroCLI, tmpCfg.UpgradeFromVeleroVersion)).To(Succeed()) + } else { + Skip("Upgrade test is skipped since user don't want to install any other velero") + } + }) + + AfterEach(func() { + if VeleroCfg.InstallVelero { + err = VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) + Expect(err).To(Succeed()) + } + }) + + When("kibishii is the sample workload", func() { + It("should be successfully backed up and restored to the default BackupStorageLocation", func() { + backupName = "backup-" + UUIDgen.String() + restoreName = "restore-" + UUIDgen.String() + tmpCfg := VeleroCfg + if (len(VeleroCfg.UpgradeFromVeleroCLI)) == 0 { + tmpCfg.UpgradeFromVeleroCLI = upgradeFromVeleroCLI + Expect(err).To(Succeed()) + } + Expect(runUpgradeTests(client, &tmpCfg, backupName, restoreName, "", useVolumeSnapshots)).To(Succeed(), + "Failed to successfully backup and restore Kibishii namespace") + }) + }) +} + +// runUpgradeTests runs upgrade test on the provider by kibishii. +func runUpgradeTests(client TestClient, veleroCfg *VerleroConfig, backupName, restoreName, backupLocation string, + useVolumeSnapshots bool) error { + if veleroCfg.VeleroCLI == "" { + return errors.New("empty") + } + oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) + if err := CreateNamespace(oneHourTimeout, client, upgradeNamespace); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", upgradeNamespace) + } + defer func() { + if err := DeleteNamespace(context.Background(), client, upgradeNamespace, true); err != nil { + fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", upgradeNamespace)) + } + }() + if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, veleroCfg.CloudProvider, upgradeNamespace, veleroCfg.RegistryCredentialFile); err != nil { + return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", upgradeNamespace) + } + + if err := VeleroBackupNamespace(oneHourTimeout, veleroCfg.UpgradeFromVeleroCLI, veleroCfg.VeleroNamespace, backupName, upgradeNamespace, backupLocation, useVolumeSnapshots); err != nil { + // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command + // TODO move to "RunDebug" after we bump up to 1.7 in the upgrade case + VeleroBackupLogs(context.Background(), veleroCfg.UpgradeFromVeleroCLI, veleroCfg.VeleroNamespace, backupName) + return errors.Wrapf(err, "Failed to backup kibishii namespace %s", upgradeNamespace) + } + + if veleroCfg.CloudProvider == "vsphere" && useVolumeSnapshots { + // Wait for uploads started by the Velero Plug-in for vSphere to complete + // TODO - remove after upload progress monitoring is implemented + fmt.Println("Waiting for vSphere uploads to complete") + if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, upgradeNamespace); err != nil { + return errors.Wrapf(err, "Error waiting for uploads to complete") + } + } + fmt.Printf("Simulating a disaster by removing namespace %s\n", upgradeNamespace) + if err := DeleteNamespace(oneHourTimeout, client, upgradeNamespace, true); err != nil { + return errors.Wrapf(err, "failed to delete namespace %s", upgradeNamespace) + } + + // the snapshots of AWS may be still in pending status when do the restore, wait for a while + // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 + // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed + if veleroCfg.CloudProvider == "aws" && useVolumeSnapshots { + fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") + time.Sleep(5 * time.Minute) + } + + if err := VeleroInstall(context.Background(), veleroCfg, "", useVolumeSnapshots); err != nil { + return errors.Wrapf(err, "Failed to install velero from image %s", veleroCfg.VeleroImage) + } + if err := CheckVeleroVersion(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroVersion); err != nil { + return errors.Wrapf(err, "Velero install version mismatch.") + } + if err := VeleroRestore(oneHourTimeout, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, restoreName, backupName); err != nil { + RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", restoreName) + return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName) + } + + if err := KibishiiVerifyAfterRestore(client, upgradeNamespace, oneHourTimeout); err != nil { + return errors.Wrapf(err, "Error verifying kibishii after restore") + } + + fmt.Printf("Upgrade test completed successfully\n") + return nil +} diff --git a/test/e2e/client.go b/test/e2e/util/k8s/client.go similarity index 71% rename from test/e2e/client.go rename to test/e2e/util/k8s/client.go index 0b1993934..a6907045d 100644 --- a/test/e2e/client.go +++ b/test/e2e/util/k8s/client.go @@ -14,27 +14,29 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package k8s import ( + "sync" + "k8s.io/client-go/kubernetes" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/client" ) -// testClient contains different API clients that are in use throughout +// TestClient contains different API clients that are in use throughout // the e2e tests. -type testClient struct { - kubebuilder kbclient.Client +type TestClient struct { + Kubebuilder kbclient.Client // clientGo returns a client-go API client. // // Deprecated, TODO(2.0): presuming all controllers and resources are converted to the // controller runtime framework by v2.0, it is the intent to remove all // client-go API clients. Please use the controller runtime to make API calls for tests. - clientGo kubernetes.Interface + ClientGo kubernetes.Interface // dynamicFactory returns a client-go API client for retrieving dynamic clients // for GroupVersionResources and GroupVersionKinds. @@ -45,35 +47,48 @@ type testClient struct { dynamicFactory client.DynamicFactory } -// newTestClient returns a set of ready-to-use API clients. -func newTestClient() (testClient, error) { +var ( + once sync.Once + testClient TestClient + err error +) + +func NewTestClient() (TestClient, error) { + once.Do(func() { // <-- atomic, does not allow repeating + testClient, err = InitTestClient() // <-- thread safe + }) + return testClient, err +} + +// NewTestClient returns a set of ready-to-use API clients. +func InitTestClient() (TestClient, error) { config, err := client.LoadConfig() if err != nil { - return testClient{}, err + return TestClient{}, err } f := client.NewFactory("e2e", config) clientGo, err := f.KubeClient() if err != nil { - return testClient{}, err + return TestClient{}, err } kb, err := f.KubebuilderClient() if err != nil { - return testClient{}, err + return TestClient{}, err } dynamicClient, err := f.DynamicClient() if err != nil { - return testClient{}, err + return TestClient{}, err } factory := client.NewDynamicFactory(dynamicClient) - return testClient{ - kubebuilder: kb, - clientGo: clientGo, + return TestClient{ + Kubebuilder: kb, + ClientGo: clientGo, dynamicFactory: factory, }, nil } diff --git a/test/e2e/common.go b/test/e2e/util/k8s/common.go similarity index 85% rename from test/e2e/common.go rename to test/e2e/util/k8s/common.go index 25a2bd3ed..1e4a5eb91 100644 --- a/test/e2e/common.go +++ b/test/e2e/util/k8s/common.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package k8s import ( "fmt" @@ -32,11 +32,11 @@ import ( ) // ensureClusterExists returns whether or not a kubernetes cluster exists for tests to be run on. -func ensureClusterExists(ctx context.Context) error { +func EnsureClusterExists(ctx context.Context) error { return exec.CommandContext(ctx, "kubectl", "cluster-info").Run() } -func createSecretFromFiles(ctx context.Context, client testClient, namespace string, name string, files map[string]string) error { +func CreateSecretFromFiles(ctx context.Context, client TestClient, namespace string, name string, files map[string]string) error { data := make(map[string][]byte) for key, filePath := range files { @@ -49,17 +49,17 @@ func createSecretFromFiles(ctx context.Context, client testClient, namespace str } secret := builder.ForSecret(namespace, name).Data(data).Result() - _, err := client.clientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + _, err := client.ClientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) return err } -// waitForPods waits until all of the pods have gone to PodRunning state -func waitForPods(ctx context.Context, client testClient, namespace string, pods []string) error { +// WaitForPods waits until all of the pods have gone to PodRunning state +func WaitForPods(ctx context.Context, client TestClient, namespace string, pods []string) error { timeout := 10 * time.Minute interval := 5 * time.Second err := wait.PollImmediate(interval, timeout, func() (bool, error) { for _, podName := range pods { - checkPod, err := client.clientGo.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + checkPod, err := client.ClientGo.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) if err != nil { return false, errors.WithMessage(err, fmt.Sprintf("Failed to verify pod %s/%s is %s", namespace, podName, corev1api.PodRunning)) } diff --git a/test/e2e/util/k8s/configmap.go b/test/e2e/util/k8s/configmap.go new file mode 100644 index 000000000..91fcf97cd --- /dev/null +++ b/test/e2e/util/k8s/configmap.go @@ -0,0 +1,75 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8s + +import ( + "fmt" + "time" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/net/context" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + waitutil "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" +) + +func CreateConfigMap(c clientset.Interface, ns, name string, data map[string]string) (*v1.ConfigMap, error) { + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: data, + }, + Data: data, + } + return c.CoreV1().ConfigMaps(ns).Create(context.TODO(), cm, metav1.CreateOptions{}) +} + +// WaitForConfigMapComplete uses c to wait for completions to complete for the Job jobName in namespace ns. +func WaitForConfigMapComplete(c clientset.Interface, ns, configmapName string) error { + return wait.Poll(PollInterval, PollTimeout, func() (bool, error) { + _, err := c.CoreV1().ConfigMaps(ns).Get(context.TODO(), configmapName, metav1.GetOptions{}) + if err != nil { + return false, err + } + return true, nil + }) +} + +func GetConfigmap(c clientset.Interface, ns, secretName string) (*v1.ConfigMap, error) { + return c.CoreV1().ConfigMaps(ns).Get(context.TODO(), secretName, metav1.GetOptions{}) +} + +func WaitForConfigmapDelete(c clientset.Interface, ns, name string) error { + if err := c.CoreV1().ConfigMaps(ns).Delete(context.TODO(), name, metav1.DeleteOptions{}); err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to delete configmap in namespace %q", ns)) + } + return waitutil.PollImmediateInfinite(5*time.Second, + func() (bool, error) { + if _, err := c.CoreV1().ConfigMaps(ns).Get(context.TODO(), ns, metav1.GetOptions{}); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + logrus.Debugf("configmap %q in namespace %q is still being deleted...", name, ns) + return false, nil + }) +} diff --git a/test/e2e/util/k8s/deployment.go b/test/e2e/util/k8s/deployment.go new file mode 100644 index 000000000..bfa1252a2 --- /dev/null +++ b/test/e2e/util/k8s/deployment.go @@ -0,0 +1,95 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8s + +import ( + "fmt" + "time" + + "golang.org/x/net/context" + apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" +) + +const ( + JobSelectorKey = "job" + // Poll is how often to Poll pods, nodes and claims. + PollInterval = 2 * time.Second + PollTimeout = 15 * time.Minute +) + +// newDeployment returns a RollingUpdate Deployment with a fake container image +func NewDeployment(name, ns string, replicas int32, labels map[string]string) *apps.Deployment { + return &apps.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: name, + Labels: labels, + }, + Spec: apps.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Strategy: apps.DeploymentStrategy{ + Type: apps.RollingUpdateDeploymentStrategyType, + RollingUpdate: new(apps.RollingUpdateDeployment), + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: name, + Image: "gcr.io/velero-gcp/busybox:latest", + Command: []string{"sleep", "1000000"}, + }, + }, + }, + }, + }, + } +} + +func CreateDeployment(c clientset.Interface, ns string, deployment *apps.Deployment) (*apps.Deployment, error) { + return c.AppsV1().Deployments(ns).Create(context.TODO(), deployment, metav1.CreateOptions{}) +} + +func GetDeployment(c clientset.Interface, ns, name string) (*apps.Deployment, error) { + return c.AppsV1().Deployments(ns).Get(context.TODO(), name, metav1.GetOptions{}) +} + +// WaitForReadyDeployment waits for number of ready replicas to equal number of replicas. +func WaitForReadyDeployment(c clientset.Interface, ns, name string) error { + if err := wait.PollImmediate(PollInterval, PollTimeout, func() (bool, error) { + deployment, err := c.AppsV1().Deployments(ns).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("failed to get deployment %q: %v", name, err) + } + return deployment.Status.ReadyReplicas == *deployment.Spec.Replicas, nil + }); err != nil { + return fmt.Errorf("failed to wait for .readyReplicas to equal .replicas: %v", err) + } + return nil +} diff --git a/test/e2e/util/k8s/namespace.go b/test/e2e/util/k8s/namespace.go new file mode 100644 index 000000000..9b28873ef --- /dev/null +++ b/test/e2e/util/k8s/namespace.go @@ -0,0 +1,136 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8s + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + corev1api "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + waitutil "k8s.io/apimachinery/pkg/util/wait" + + "github.com/vmware-tanzu/velero/pkg/builder" +) + +func CreateNamespace(ctx context.Context, client TestClient, namespace string) error { + ns := builder.ForNamespace(namespace).Result() + _, err := client.ClientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + return nil + } + return err +} + +func CreateNamespaceWithLabel(ctx context.Context, client TestClient, namespace string, label map[string]string) error { + ns := builder.ForNamespace(namespace).Result() + ns.Labels = label + _, err := client.ClientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + return nil + } + return err +} + +func CreateNamespaceWithAnnotation(ctx context.Context, client TestClient, namespace string, annotation map[string]string) error { + ns := builder.ForNamespace(namespace).Result() + ns.ObjectMeta.Annotations = annotation + _, err := client.ClientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + return nil + } + return err +} + +func GetNamespace(ctx context.Context, client TestClient, namespace string) (*corev1api.Namespace, error) { + return client.ClientGo.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) +} + +func DeleteNamespace(ctx context.Context, client TestClient, namespace string, wait bool) error { + if err := client.ClientGo.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}); err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) + } + if !wait { + return nil + } + + return waitutil.PollImmediateInfinite(5*time.Second, + func() (bool, error) { + if _, err := client.ClientGo.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + logrus.Debugf("namespace %q is still being deleted...", namespace) + return false, nil + }) +} + +func CleanupNamespacesWithPoll(ctx context.Context, client TestClient, nsBaseName string) error { + namespaces, err := client.ClientGo.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + if err != nil { + return errors.Wrap(err, "Could not retrieve namespaces") + } + for _, checkNamespace := range namespaces.Items { + if strings.HasPrefix(checkNamespace.Name, nsBaseName) { + err := DeleteNamespace(ctx, client, checkNamespace.Name, true) + if err != nil { + return errors.Wrapf(err, "Could not delete namespace %s", checkNamespace.Name) + } + } + } + return nil +} + +func CleanupNamespaces(ctx context.Context, client TestClient, nsBaseName string) error { + namespaces, err := client.ClientGo.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + if err != nil { + return errors.Wrap(err, "Could not retrieve namespaces") + } + for _, checkNamespace := range namespaces.Items { + if strings.HasPrefix(checkNamespace.Name, nsBaseName) { + err = client.ClientGo.CoreV1().Namespaces().Delete(ctx, checkNamespace.Name, metav1.DeleteOptions{}) + if err != nil { + return errors.Wrapf(err, "Could not delete namespace %s", checkNamespace.Name) + } + } + } + return nil +} + +func WaitAllSelectedNSDeleted(ctx context.Context, client TestClient, label string) error { + return waitutil.PollImmediateInfinite(5*time.Second, + func() (bool, error) { + if ns, err := client.ClientGo.CoreV1().Namespaces().List(ctx, metav1.ListOptions{LabelSelector: label}); err != nil { + return false, err + } else if ns == nil { + return true, nil + } else if len(ns.Items) == 0 { + return true, nil + } else { + logrus.Debugf("%d namespaces is still being deleted...\n", len(ns.Items)) + return false, nil + } + }) +} diff --git a/test/e2e/util/k8s/rbac.go b/test/e2e/util/k8s/rbac.go new file mode 100644 index 000000000..7486e8761 --- /dev/null +++ b/test/e2e/util/k8s/rbac.go @@ -0,0 +1,114 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8s + +import ( + "context" + "fmt" + "strings" + + "github.com/pkg/errors" + v1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func CreateRBACWithBindingSA(ctx context.Context, client TestClient, namespace string, serviceaccount string, clusterrole string, clusterrolebinding string) error { + role := &v1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterrole, + }, + } + + _, err = client.ClientGo.RbacV1().ClusterRoles().Create(ctx, role, metav1.CreateOptions{}) + + if err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + + //creating role binding and binding it to the test service account + rolebinding := &v1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterrolebinding, + }, + Subjects: []v1.Subject{ + { + Kind: "ServiceAccount", + Name: serviceaccount, + Namespace: namespace, + }, + }, + RoleRef: v1.RoleRef{ + Kind: "ClusterRole", + Name: clusterrole, + }, + } + + _, err = client.ClientGo.RbacV1().ClusterRoleBindings().Create(ctx, rolebinding, metav1.CreateOptions{}) + + if err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + + return nil +} + +func GetClusterRole(ctx context.Context, client TestClient, role string) (*v1.ClusterRole, error) { + return client.ClientGo.RbacV1().ClusterRoles().Get(ctx, role, metav1.GetOptions{}) +} + +func GetClusterRoleBinding(ctx context.Context, client TestClient, rolebinding string) (*v1.ClusterRoleBinding, error) { + return client.ClientGo.RbacV1().ClusterRoleBindings().Get(ctx, rolebinding, metav1.GetOptions{}) +} + +func CleanupClusterRole(ctx context.Context, client TestClient, nsBaseName string) error { + + clusterroles, err := client.ClientGo.RbacV1().ClusterRoles().List(ctx, metav1.ListOptions{}) + if err != nil { + return errors.Wrap(err, "Could not retrieve clusterroles") + } + + for _, checkClusterRole := range clusterroles.Items { + if strings.HasPrefix(checkClusterRole.Name, "clusterrole-"+nsBaseName) { + fmt.Printf("Cleaning up clusterrole %s\n", checkClusterRole.Name) + err = client.ClientGo.RbacV1().ClusterRoles().Delete(ctx, checkClusterRole.Name, metav1.DeleteOptions{}) + if err != nil { + return errors.Wrapf(err, "Could not delete clusterrole %s", checkClusterRole.Name) + } + } + } + return nil +} + +func CleanupClusterRoleBinding(ctx context.Context, client TestClient, nsBaseName string) error { + + clusterrolebindings, err := client.ClientGo.RbacV1().ClusterRoleBindings().List(ctx, metav1.ListOptions{}) + if err != nil { + return errors.Wrap(err, "Could not retrieve clusterrolebindings") + } + + for _, checkClusterRoleBinding := range clusterrolebindings.Items { + if strings.HasPrefix(checkClusterRoleBinding.Name, "clusterrolebinding-"+nsBaseName) { + fmt.Printf("Cleaning up clusterrolebinding %s\n", checkClusterRoleBinding.Name) + err = client.ClientGo.RbacV1().ClusterRoleBindings().Delete(ctx, checkClusterRoleBinding.Name, metav1.DeleteOptions{}) + if err != nil { + return errors.Wrapf(err, "Could not delete clusterrolebinding %s", checkClusterRoleBinding.Name) + } + } + } + return nil +} diff --git a/test/e2e/util/k8s/secret.go b/test/e2e/util/k8s/secret.go new file mode 100644 index 000000000..3c3682c93 --- /dev/null +++ b/test/e2e/util/k8s/secret.go @@ -0,0 +1,97 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8s + +import ( + "fmt" + "time" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + "golang.org/x/net/context" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + waitutil "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" +) + +func CreateSecret(c clientset.Interface, ns, name string, labels map[string]string) (*v1.Secret, error) { + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: labels, + }, + } + return c.CoreV1().Secrets(ns).Create(context.TODO(), secret, metav1.CreateOptions{}) +} + +func WaitForSecretDelete(c clientset.Interface, ns, name string) error { + if err := c.CoreV1().Secrets(ns).Delete(context.TODO(), name, metav1.DeleteOptions{}); err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to delete secret in namespace %q", ns)) + } + return waitutil.PollImmediateInfinite(5*time.Second, + func() (bool, error) { + if _, err := c.CoreV1().Secrets(ns).Get(context.TODO(), ns, metav1.GetOptions{}); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + logrus.Debugf("secret %q in namespace %q is still being deleted...", name, ns) + return false, nil + }) +} + +// WaitForSecretsComplete uses c to wait for completions to complete for the Job jobName in namespace ns. +func WaitForSecretsComplete(c clientset.Interface, ns, secretName string) error { + return wait.Poll(PollInterval, PollTimeout, func() (bool, error) { + _, err := c.CoreV1().Secrets(ns).Get(context.TODO(), secretName, metav1.GetOptions{}) + if err != nil { + return false, err + } + return true, nil + }) +} + +func GetSecret(c clientset.Interface, ns, secretName string) (*v1.Secret, error) { + return c.CoreV1().Secrets(ns).Get(context.TODO(), secretName, metav1.GetOptions{}) +} + +//CreateVCCredentialSecret refer to https://github.com/vmware-tanzu/velero-plugin-for-vsphere/blob/v1.3.0/docs/vanilla.md +func CreateVCCredentialSecret(c clientset.Interface, veleroNamespace string) error { + secret, err := GetSecret(c, "kube-system", "vsphere-config-secret") + if err != nil { + return err + } + vsphereCfg, exist := secret.Data["csi-vsphere.conf"] + if !exist { + return errors.New("failed to retrieve csi-vsphere config") + } + se := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "velero-vsphere-config-secret", + Namespace: veleroNamespace, + }, + Type: v1.SecretTypeOpaque, + Data: map[string][]byte{"csi-vsphere.conf": vsphereCfg}, + } + _, err = c.CoreV1().Secrets(veleroNamespace).Create(context.TODO(), se, metav1.CreateOptions{}) + return err +} diff --git a/test/e2e/serviceaccount.go b/test/e2e/util/k8s/serviceaccount.go similarity index 63% rename from test/e2e/serviceaccount.go rename to test/e2e/util/k8s/serviceaccount.go index 30e46a85c..6f949e856 100644 --- a/test/e2e/serviceaccount.go +++ b/test/e2e/util/k8s/serviceaccount.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package k8s import ( "context" @@ -33,10 +33,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func waitUntilServiceAccountCreated(ctx context.Context, client testClient, namespace, serviceAccount string, timeout time.Duration) error { +func WaitUntilServiceAccountCreated(ctx context.Context, client TestClient, namespace, serviceAccount string, timeout time.Duration) error { return wait.PollImmediate(5*time.Second, timeout, func() (bool, error) { - if _, err := client.clientGo.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccount, metav1.GetOptions{}); err != nil { + if _, err := client.ClientGo.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccount, metav1.GetOptions{}); err != nil { if !apierrors.IsNotFound(err) { return false, err } @@ -46,7 +46,7 @@ func waitUntilServiceAccountCreated(ctx context.Context, client testClient, name }) } -func patchServiceAccountWithImagePullSecret(ctx context.Context, client testClient, namespace, serviceAccount, dockerCredentialFile string) error { +func PatchServiceAccountWithImagePullSecret(ctx context.Context, client TestClient, namespace, serviceAccount, dockerCredentialFile string) error { credential, err := ioutil.ReadFile(dockerCredentialFile) if err != nil { return errors.Wrapf(err, "failed to read the docker credential file %q", dockerCredentialFile) @@ -54,13 +54,33 @@ func patchServiceAccountWithImagePullSecret(ctx context.Context, client testClie secretName := "image-pull-secret" secret := builder.ForSecret(namespace, secretName).Data(map[string][]byte{".dockerconfigjson": credential}).Result() secret.Type = corev1.SecretTypeDockerConfigJson - if _, err = client.clientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil { + if _, err = client.ClientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil { return errors.Wrapf(err, "failed to create secret %q under namespace %q", secretName, namespace) } - if _, err = client.clientGo.CoreV1().ServiceAccounts(namespace).Patch(ctx, serviceAccount, types.StrategicMergePatchType, + if _, err = client.ClientGo.CoreV1().ServiceAccounts(namespace).Patch(ctx, serviceAccount, types.StrategicMergePatchType, []byte(fmt.Sprintf(`{"imagePullSecrets": [{"name": "%s"}]}`, secretName)), metav1.PatchOptions{}); err != nil { return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccount, namespace) } return nil } + +func CreateServiceAccount(ctx context.Context, client TestClient, namespace string, serviceaccount string) error { + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceaccount, + }, + AutomountServiceAccountToken: nil, + } + + _, err = client.ClientGo.CoreV1().ServiceAccounts(namespace).Create(ctx, sa, metav1.CreateOptions{}) + + if err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + return nil +} + +func GetServiceAccount(ctx context.Context, client TestClient, namespace string, serviceAccount string) (*corev1.ServiceAccount, error) { + return client.ClientGo.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccount, metav1.GetOptions{}) +} diff --git a/test/e2e/kibishii_tests.go b/test/e2e/util/kibishii/kibishii_utils.go similarity index 74% rename from test/e2e/kibishii_tests.go rename to test/e2e/util/kibishii/kibishii_utils.go index 27c32b8e9..945006c54 100644 --- a/test/e2e/kibishii_tests.go +++ b/test/e2e/util/kibishii/kibishii_utils.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package kibishii import ( "fmt" @@ -26,6 +26,8 @@ import ( "golang.org/x/net/context" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) const ( @@ -33,6 +35,61 @@ const ( jumpPadPod = "jump-pad" ) +// RunKibishiiTests runs kibishii tests on the provider. +func RunKibishiiTests(client TestClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, + useVolumeSnapshots bool, registryCredentialFile string) error { + oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) + if err := CreateNamespace(oneHourTimeout, client, kibishiiNamespace); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace) + } + defer func() { + if err := DeleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil { + fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace)) + } + }() + if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, kibishiiNamespace, registryCredentialFile); err != nil { + return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", kibishiiNamespace) + } + + if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil { + RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "") + return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace) + } + + if providerName == "vsphere" && useVolumeSnapshots { + // Wait for uploads started by the Velero Plug-in for vSphere to complete + // TODO - remove after upload progress monitoring is implemented + fmt.Println("Waiting for vSphere uploads to complete") + if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil { + return errors.Wrapf(err, "Error waiting for uploads to complete") + } + } + fmt.Printf("Simulating a disaster by removing namespace %s\n", kibishiiNamespace) + if err := DeleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil { + return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace) + } + + // the snapshots of AWS may be still in pending status when do the restore, wait for a while + // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 + // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed + if providerName == "aws" && useVolumeSnapshots { + fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") + time.Sleep(5 * time.Minute) + } + + if err := VeleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil { + RunDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName) + return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName) + } + + if err := KibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout); err != nil { + return errors.Wrapf(err, "Error verifying kibishii after restore") + } + + fmt.Printf("kibishii test completed successfully\n") + return nil +} + func installKibishii(ctx context.Context, namespace string, cloudPlatform string) error { // We use kustomize to generate YAML for Kibishii from the checked-in yaml directories kibishiiInstallCmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", namespace, "-k", @@ -81,33 +138,26 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel strconv.Itoa(blockSize), strconv.Itoa(passNum), strconv.Itoa(expectedNodes)) fmt.Printf("kibishiiVerifyCmd cmd =%v\n", kibishiiVerifyCmd) - _, stderr, err := veleroexec.RunCommand(kibishiiVerifyCmd) + stdout, stderr, err := veleroexec.RunCommand(kibishiiVerifyCmd) if err != nil { - return errors.Wrapf(err, "failed to verify, stderr=%s", stderr) + return errors.Wrapf(err, "failed to verify, stderr=%s, stdout=%s", stderr, stdout) } return nil } -// runKibishiiTests runs kibishii tests on the provider. -func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, - useVolumeSnapshots bool, registryCredentialFile string) error { - oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) +func waitForKibishiiPods(ctx context.Context, client TestClient, kibishiiNamespace string) error { + return WaitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"}) +} + +func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClient, providerName, kibishiiNamespace, registryCredentialFile string) error { serviceAccountName := "default" - if err := createNamespace(oneHourTimeout, client, kibishiiNamespace); err != nil { - return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace) - } - defer func() { - if err := deleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil { - fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace)) - } - }() // wait until the service account is created before patch the image pull secret - if err := waitUntilServiceAccountCreated(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, 10*time.Minute); err != nil { + if err := WaitUntilServiceAccountCreated(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, 10*time.Minute); err != nil { return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, kibishiiNamespace) } // add the image pull secret to avoid the image pull limit issue of Docker Hub - if err := patchServiceAccountWithImagePullSecret(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, registryCredentialFile); err != nil { + if err := PatchServiceAccountWithImagePullSecret(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, registryCredentialFile); err != nil { return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, kibishiiNamespace) } @@ -125,30 +175,10 @@ func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespac if err := generateData(oneHourTimeout, kibishiiNamespace, 2, 10, 10, 1024, 1024, 0, 2); err != nil { return errors.Wrap(err, "Failed to generate data") } + return nil +} - if err := veleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil { - veleroBackupLogs(oneHourTimeout, veleroCLI, veleroNamespace, backupName) - return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace) - } - - if providerName == "vsphere" && useVolumeSnapshots { - // Wait for uploads started by the Velero Plug-in for vSphere to complete - // TODO - remove after upload progress monitoring is implemented - fmt.Println("Waiting for vSphere uploads to complete") - if err := waitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil { - return errors.Wrapf(err, "Error waiting for uploads to complete") - } - } - fmt.Printf("Simulating a disaster by removing namespace %s\n", kibishiiNamespace) - if err := deleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil { - return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace) - } - - if err := veleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil { - veleroRestoreLogs(oneHourTimeout, veleroCLI, veleroNamespace, restoreName) - return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName) - } - +func KibishiiVerifyAfterRestore(client TestClient, kibishiiNamespace string, oneHourTimeout context.Context) error { // wait for kibishii pod startup // TODO - Fix kibishii so we can check that it is ready to go fmt.Printf("Waiting for kibishii pods to be ready\n") @@ -161,11 +191,5 @@ func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespac if err := verifyData(oneHourTimeout, kibishiiNamespace, 2, 10, 10, 1024, 1024, 0, 2); err != nil { return errors.Wrap(err, "Failed to verify data generated by kibishii") } - - fmt.Printf("kibishii test completed successfully\n") return nil } - -func waitForKibishiiPods(ctx context.Context, client testClient, kibishiiNamespace string) error { - return waitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"}) -} diff --git a/test/e2e/util/providers/aws_utils.go b/test/e2e/util/providers/aws_utils.go new file mode 100644 index 000000000..3e64b2b3a --- /dev/null +++ b/test/e2e/util/providers/aws_utils.go @@ -0,0 +1,141 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package providers + +import ( + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/pkg/errors" + + "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" +) + +type AWSStorage string + +func (s AWSStorage) ListItems(client *s3.S3, objectsV2Input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + res, err := client.ListObjectsV2(objectsV2Input) + if err != nil { + return nil, err + } + + return res, nil +} + +func (s AWSStorage) DeleteItem(client *s3.S3, deleteObjectV2Input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { + res, err := client.DeleteObject(deleteObjectV2Input) + if err != nil { + return nil, err + } + fmt.Println(res) + return res, nil +} +func (s AWSStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) { + config := flag.NewMap() + config.Set(bslConfig) + region := config.Data()["region"] + objectsInput := s3.ListObjectsV2Input{} + objectsInput.Bucket = aws.String(bslBucket) + objectsInput.Delimiter = aws.String("/") + s3url := "" + if bslPrefix != "" { + objectsInput.Prefix = aws.String(bslPrefix) + } + s3Config := &aws.Config{ + Region: aws.String(region), + Credentials: credentials.NewSharedCredentials(cloudCredentialsFile, ""), + } + if region == "minio" { + s3url = config.Data()["s3Url"] + s3Config = &aws.Config{ + Credentials: credentials.NewSharedCredentials(cloudCredentialsFile, ""), + Endpoint: aws.String(s3url), + Region: aws.String(region), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + } + } + + sess, err := session.NewSession(s3Config) + + if err != nil { + return false, errors.Wrapf(err, "Failed to create AWS session") + } + svc := s3.New(sess) + + bucketObjects, err := s.ListItems(svc, &objectsInput) + if err != nil { + return false, errors.Wrapf(err, "Couldn't retrieve bucket items") + } + + for _, item := range bucketObjects.Contents { + fmt.Println(*item) + } + var backupNameInStorage string + for _, item := range bucketObjects.CommonPrefixes { + backupNameInStorage = strings.TrimPrefix(*item.Prefix, strings.Trim(bslPrefix, "/")+"/") + fmt.Println(backupNameInStorage) + if strings.Contains(backupNameInStorage, backupObject) { + fmt.Printf("Backup %s was found under prefix %s \n", backupObject, bslPrefix) + return true, nil + } + } + fmt.Printf("Backup %s was not found under prefix %s \n", backupObject, bslPrefix) + return false, nil +} + +func (s AWSStorage) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error { + config := flag.NewMap() + config.Set(bslConfig) + region := config.Data()["region"] + s3url := "" + s3Config := &aws.Config{ + Region: aws.String(region), + Credentials: credentials.NewSharedCredentials(cloudCredentialsFile, ""), + } + if region == "minio" { + s3url = config.Data()["s3Url"] + s3Config = &aws.Config{ + Credentials: credentials.NewSharedCredentials(cloudCredentialsFile, ""), + Endpoint: aws.String(s3url), + Region: aws.String(region), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + } + } + sess, err := session.NewSession(s3Config) + if err != nil { + return errors.Wrapf(err, "Error waiting for uploads to complete") + } + svc := s3.New(sess) + fullPrefix := strings.Trim(bslPrefix, "/") + "/" + strings.Trim(backupObject, "/") + "/" + iter := s3manager.NewDeleteListIterator(svc, &s3.ListObjectsInput{ + Bucket: aws.String(bslBucket), + Prefix: aws.String(fullPrefix), + }) + + if err := s3manager.NewBatchDeleteWithClient(svc).Delete(aws.BackgroundContext(), iter); err != nil { + return errors.Wrapf(err, "Error waiting for uploads to complete") + } + fmt.Printf("Deleted object(s) from bucket: %s %s \n", bslBucket, fullPrefix) + return nil +} diff --git a/test/e2e/util/providers/azure_utils.go b/test/e2e/util/providers/azure_utils.go new file mode 100644 index 000000000..d9ee4c6ca --- /dev/null +++ b/test/e2e/util/providers/azure_utils.go @@ -0,0 +1,258 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package providers + +import ( + "fmt" + "log" + "net/url" + "os" + "strings" + + "github.com/Azure/azure-pipeline-go/pipeline" + storagemgmt "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-06-01/storage" + "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/Azure/go-autorest/autorest/azure/auth" + "github.com/joho/godotenv" + "github.com/pkg/errors" + "golang.org/x/net/context" + + "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" +) + +type AzureStorage string + +const ( + subscriptionIDEnvVar = "AZURE_SUBSCRIPTION_ID" + cloudNameEnvVar = "AZURE_CLOUD_NAME" + resourceGroupEnvVar = "AZURE_RESOURCE_GROUP" + storageAccountKey = "AZURE_STORAGE_ACCOUNT_ACCESS_KEY" + storageAccount = "storageAccount" + subscriptionID = "subscriptionId" + resourceGroup = "resourceGroup" +) + +func getStorageCredential(cloudCredentialsFile, bslConfig string) (string, string, error) { + config := flag.NewMap() + config.Set(bslConfig) + accountName := config.Data()[storageAccount] + // Account name must be provided in config + if len(accountName) == 0 { + return "", "", errors.New("Please provide bucket as Azure account name ") + } + subscriptionID := config.Data()[subscriptionID] + resourceGroupCfg := config.Data()[resourceGroup] + accountKey, err := getStorageAccountKey(cloudCredentialsFile, accountName, subscriptionID, resourceGroupCfg) + if err != nil { + return "", "", errors.Wrapf(err, "Fail to get storage key of bucket %s", accountName) + } + return accountName, accountKey, nil +} +func loadCredentialsIntoEnv(credentialsFile string) error { + if credentialsFile == "" { + return nil + } + + if err := godotenv.Overload(credentialsFile); err != nil { + return errors.Wrapf(err, "error loading environment from credentials file (%s)", credentialsFile) + } + return nil +} +func parseAzureEnvironment(cloudName string) (*azure.Environment, error) { + if cloudName == "" { + fmt.Println("cloudName is empty") + return &azure.PublicCloud, nil + } + + env, err := azure.EnvironmentFromName(cloudName) + return &env, errors.WithStack(err) +} +func getStorageAccountKey(credentialsFile, accountName, subscriptionID, resourceGroupCfg string) (string, error) { + if err := loadCredentialsIntoEnv(credentialsFile); err != nil { + return "", err + } + storageKey := os.Getenv(storageAccountKey) + if storageKey != "" { + return storageKey, nil + } + if os.Getenv(cloudNameEnvVar) == "" { + return "", errors.New("Credential file should contain AZURE_CLOUD_NAME") + } + var resourceGroup string + if os.Getenv(resourceGroupEnvVar) == "" { + if resourceGroupCfg == "" { + return "", errors.New("Credential file should contain AZURE_RESOURCE_GROUP or AZURE_STORAGE_ACCOUNT_ACCESS_KEY") + } else { + resourceGroup = resourceGroupCfg + } + } else { + resourceGroup = os.Getenv(resourceGroupEnvVar) + } + // get Azure cloud from AZURE_CLOUD_NAME, if it exists. If the env var does not + // exist, parseAzureEnvironment will return azure.PublicCloud. + env, err := parseAzureEnvironment(os.Getenv(cloudNameEnvVar)) + if err != nil { + return "", errors.Wrap(err, "unable to parse azure cloud name environment variable") + } + + // get subscription ID from object store config or AZURE_SUBSCRIPTION_ID environment variable + if subscriptionID == "" { + return "", errors.New("azure subscription ID not found in object store's config or in environment variable") + } + + authorizer, err := auth.NewAuthorizerFromEnvironment() + if err != nil { + return "", errors.Wrap(err, "error getting authorizer from environment") + } + + // get storageAccountsClient + storageAccountsClient := storagemgmt.NewAccountsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID) + storageAccountsClient.Authorizer = authorizer + + // get storage key + res, err := storageAccountsClient.ListKeys(context.TODO(), resourceGroup, accountName, storagemgmt.Kerb) + if err != nil { + return "", errors.WithStack(err) + } + if res.Keys == nil || len(*res.Keys) == 0 { + return "", errors.New("No storage keys found") + } + + for _, key := range *res.Keys { + // uppercase both strings for comparison because the ListKeys call returns e.g. "FULL" but + // the storagemgmt.Full constant in the SDK is defined as "Full". + if strings.EqualFold(string(key.Permissions), string(storagemgmt.Full)) { + storageKey = *key.Value + break + } + } + + if storageKey == "" { + return "", errors.New("No storage key with Full permissions found") + } + + return storageKey, nil +} +func handleErrors(err error) { + if err != nil { + if serr, ok := err.(azblob.StorageError); ok { // This error is a Service-specific + switch serr.ServiceCode() { // Compare serviceCode to ServiceCodeXxx constants + case azblob.ServiceCodeContainerAlreadyExists: + return + } + } + log.Fatal(err) + } +} + +func deleteBlob(p pipeline.Pipeline, accountName, containerName, blobName string) error { + ctx := context.Background() + + URL_BLOB, err := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s", accountName, containerName, blobName)) + if err != nil { + return errors.Wrapf(err, "Fail to url.Parse") + } + blobURL := azblob.NewBlobURL(*URL_BLOB, p) + _, err = blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) + return err +} +func (s AzureStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) { + accountName, accountKey, err := getStorageCredential(cloudCredentialsFile, bslConfig) + if err != nil { + log.Fatal("Fail to get : accountName and accountKey, " + err.Error()) + } + + credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) + if err != nil { + log.Fatal("Invalid credentials with error: " + err.Error()) + } + p := azblob.NewPipeline(credential, azblob.PipelineOptions{}) + + containerName := bslBucket + + URL, _ := url.Parse( + fmt.Sprintf("https://%s.blob.core.windows.net/%s", accountName, containerName)) + + containerURL := azblob.NewContainerURL(*URL, p) + + // Create the container, if container is already exist, then do nothing + ctx := context.Background() + _, err = containerURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone) + handleErrors(err) + + fmt.Printf("Finding backup %s blobs in Azure container/bucket %s\n", backupObject, containerName) + for marker := (azblob.Marker{}); marker.NotDone(); { + listBlob, err := containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{}) + if err != nil { + return false, errors.Wrapf(err, "Fail to create gcloud client") + } + marker = listBlob.NextMarker + + for _, blobInfo := range listBlob.Segment.BlobItems { + if strings.Contains(blobInfo.Name, backupObject) { + fmt.Printf("Blob name: %s exist in %s\n", backupObject, blobInfo.Name) + return true, nil + } + } + } + return false, nil +} + +func (s AzureStorage) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error { + ctx := context.Background() + accountName, accountKey, err := getStorageCredential(cloudCredentialsFile, bslConfig) + if err != nil { + return errors.Wrapf(err, "Fail to get storage account name and key of bucket %s", bslBucket) + } + + credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) + if err != nil { + log.Fatal("Invalid credentials with error: " + err.Error()) + } + p := azblob.NewPipeline(credential, azblob.PipelineOptions{}) + + containerName := bslBucket + + URL, _ := url.Parse( + fmt.Sprintf("https://%s.blob.core.windows.net/%s", accountName, containerName)) + + containerURL := azblob.NewContainerURL(*URL, p) + _, err = containerURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone) + handleErrors(err) + + fmt.Println("Listing the blobs in the container:") + for marker := (azblob.Marker{}); marker.NotDone(); { + listBlob, err := containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{}) + if err != nil { + return errors.Wrapf(err, "Fail to create gcloud client") + } + + marker = listBlob.NextMarker + for _, blobInfo := range listBlob.Segment.BlobItems { + + if strings.Contains(blobInfo.Name, bslPrefix+backupObject+"/") { + deleteBlob(p, accountName, containerName, blobInfo.Name) + if err != nil { + log.Fatal("Invalid credentials with error: " + err.Error()) + } + fmt.Printf("Deleted blob: %s according to backup resource %s\n", blobInfo.Name, bslPrefix+backupObject+"/") + } + } + } + return nil +} diff --git a/test/e2e/util/providers/common.go b/test/e2e/util/providers/common.go new file mode 100644 index 000000000..aa00065c9 --- /dev/null +++ b/test/e2e/util/providers/common.go @@ -0,0 +1,105 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package providers + +import ( + "fmt" + "strings" + "time" + + "github.com/pkg/errors" +) + +type ObjectsInStorage interface { + IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) + DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error +} + +func ObjectsShouldBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error { + fmt.Printf("|| VERIFICATION || - Backup %s should exist in storage %s", backupName, bslPrefix) + exist, _ := IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix) + if !exist { + return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s is not exist in object store after backup as expected", backupName)) + } + fmt.Printf("|| EXPECTED || - Backup %s exist in object storage bucket %s\n", backupName, bslBucket) + return nil +} +func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string, retryTimes int) error { + var err error + var exist bool + fmt.Printf("|| VERIFICATION || - Backup %s should not exist in storage %s", backupName, bslPrefix) + for i := 0; i < retryTimes; i++ { + exist, err = IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix) + if err != nil { + return errors.Wrapf(err, "|| UNEXPECTED || - Failed to get backup %s in object store", backupName) + } + if !exist { + fmt.Printf("|| EXPECTED || - Backup %s is not in object store\n", backupName) + return nil + } + time.Sleep(1 * time.Minute) + } + return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s still exist in object store after backup deletion", backupName)) +} +func getProvider(cloudProvider string) (ObjectsInStorage, error) { + var s ObjectsInStorage + switch cloudProvider { + case "aws", "vsphere": + aws := AWSStorage("") + s = &aws + case "gcp": + gcs := GCSStorage("") + s = &gcs + case "azure": + az := AzureStorage("") + s = &az + default: + return nil, errors.New(fmt.Sprintf("Cloud provider %s is not valid", cloudProvider)) + } + return s, nil +} +func getFullPrefix(bslPrefix, subPrefix string) string { + if bslPrefix == "" { + bslPrefix = subPrefix + "/" + } else { + //subPrefix must have surfix "/", so that objects under it can be listed + bslPrefix = strings.Trim(bslPrefix, "/") + "/" + strings.Trim(subPrefix, "/") + "/" + } + return bslPrefix +} +func IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) (bool, error) { + bslPrefix = getFullPrefix(bslPrefix, subPrefix) + s, err := getProvider(cloudProvider) + if err != nil { + return false, errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider)) + } + return s.IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName) +} + +func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error { + bslPrefix = getFullPrefix(bslPrefix, subPrefix) + fmt.Printf("|| VERIFICATION || - Delete backup %s in storage %s", backupName, bslPrefix) + s, err := getProvider(cloudProvider) + if err != nil { + return errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider)) + } + err = s.DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName) + if err != nil { + return errors.Wrapf(err, fmt.Sprintf("Fail to delete %s", bslPrefix)) + } + return nil +} diff --git a/test/e2e/util/providers/gcloud_utils.go b/test/e2e/util/providers/gcloud_utils.go new file mode 100644 index 000000000..fdc01b081 --- /dev/null +++ b/test/e2e/util/providers/gcloud_utils.go @@ -0,0 +1,99 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package providers + +import ( + "fmt" + "strings" + + "cloud.google.com/go/storage" + "github.com/pkg/errors" + "golang.org/x/net/context" + "google.golang.org/api/iterator" + "google.golang.org/api/option" +) + +type GCSStorage string + +func (s GCSStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) { + q := &storage.Query{ + Prefix: bslPrefix, + } + + ctx := context.Background() + client, err := storage.NewClient(ctx, option.WithCredentialsFile(cloudCredentialsFile)) + if err != nil { + return false, errors.Wrapf(err, "Fail to create gcloud client") + } + iter := client.Bucket(bslBucket).Objects(context.Background(), q) + for { + obj, err := iter.Next() + if err == iterator.Done { + return false, errors.Wrapf(err, fmt.Sprintf("Backup %s was not found under prefix %s \n", backupObject, bslPrefix)) + } + if err != nil { + return false, errors.WithStack(err) + } + if obj.Name == bslPrefix { + fmt.Println("Ignore GCS prefix itself") + continue + } + if strings.Contains(obj.Name, bslPrefix+backupObject+"/") { + fmt.Printf("Found delete-object %s of %s in bucket %s \n", backupObject, obj.Name, bslBucket) + return true, nil + } + } +} +func (s GCSStorage) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error { + q := &storage.Query{ + Prefix: bslPrefix, + } + + ctx := context.Background() + client, err := storage.NewClient(ctx, option.WithCredentialsFile(cloudCredentialsFile)) + if err != nil { + return errors.Wrapf(err, "Fail to create gcloud client") + } + bucket := client.Bucket(bslBucket) + iter := bucket.Objects(context.Background(), q) + deleted := false + for { + obj, err := iter.Next() + if err == iterator.Done { + fmt.Println(err) + if !deleted { + return errors.New("|| UNEXPECTED ||Backup object is not exist and was not deleted in object store") + } + return nil + } + if err != nil { + return errors.WithStack(err) + } + if obj.Name == bslPrefix { + fmt.Println("Ignore GCS prefix itself") + continue + } + // Only delete folder named as backupObject under prefix + if strings.Contains(obj.Name, bslPrefix+backupObject+"/") { + if err = bucket.Object(obj.Name).Delete(ctx); err != nil { + return errors.Wrapf(err, fmt.Sprintf("Fail to delete object %s in bucket %s", obj.Name, bslBucket)) + } + fmt.Printf("Delete item: %s\n", obj.Name) + deleted = true + } + } +} diff --git a/test/e2e/install.go b/test/e2e/util/velero/install.go similarity index 58% rename from test/e2e/install.go rename to test/e2e/util/velero/install.go index 0d31f8d82..9c5c30c4a 100644 --- a/test/e2e/install.go +++ b/test/e2e/util/velero/install.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package velero import ( "bytes" @@ -29,65 +29,71 @@ import ( "github.com/pkg/errors" apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" velerexec "github.com/vmware-tanzu/velero/pkg/util/exec" + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" ) // we provide more install options other than the standard install.InstallOptions in E2E test type installOptions struct { *install.InstallOptions RegistryCredentialFile string + ResticHelperImage string } -// TODO too many parameters for this function, better to make it a structure, we can introduces a structure `config` for the E2E to hold all configuration items -func veleroInstall(ctx context.Context, cli, veleroImage string, veleroNamespace string, cloudProvider string, objectStoreProvider string, useVolumeSnapshots bool, - cloudCredentialsFile string, bslBucket string, bslPrefix string, bslConfig string, vslConfig string, - crdsVersion string, features string, registryCredentialFile string) error { - - if cloudProvider != "kind" { - if objectStoreProvider != "" { +func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, features string, useVolumeSnapshots bool) error { + if veleroCfg.CloudProvider != "kind" { + if veleroCfg.ObjectStoreProvider != "" { return errors.New("For cloud platforms, object store plugin cannot be overridden") // Can't set an object store provider that is different than your cloud } - objectStoreProvider = cloudProvider + veleroCfg.ObjectStoreProvider = veleroCfg.CloudProvider } else { - if objectStoreProvider == "" { + if veleroCfg.ObjectStoreProvider == "" { return errors.New("No object store provider specified - must be specified when using kind as the cloud provider") // Gotta have an object store provider } } - // Fetch the plugins for the provider before checking for the object store provider below. - providerPlugins := getProviderPlugins(objectStoreProvider) - - // TODO - handle this better - if cloudProvider == "vsphere" { - // We overrider the objectStoreProvider here for vSphere because we want to use the aws plugin for the - // backup, but needed to pick up the provider plugins earlier. vSphere plugin no longer needs a Volume - // Snapshot location specified - objectStoreProvider = "aws" + providerPluginsTmp, err := getProviderPlugins(ctx, veleroCfg.VeleroCLI, veleroCfg.ObjectStoreProvider, veleroCfg.Plugins) + if err != nil { + return errors.WithMessage(err, "Failed to get provider plugins") } - err := ensureClusterExists(ctx) + err = EnsureClusterExists(ctx) if err != nil { return errors.WithMessage(err, "Failed to ensure Kubernetes cluster exists") } - veleroInstallOptions, err := getProviderVeleroInstallOptions(objectStoreProvider, cloudCredentialsFile, bslBucket, - bslPrefix, bslConfig, vslConfig, providerPlugins, features) + // TODO - handle this better + if veleroCfg.CloudProvider == "vsphere" { + // We overrider the ObjectStoreProvider here for vSphere because we want to use the aws plugin for the + // backup, but needed to pick up the provider plugins earlier. vSphere plugin no longer needs a Volume + // Snapshot location specified + veleroCfg.ObjectStoreProvider = "aws" + if err := configvSpherePlugin(); err != nil { + return errors.WithMessagef(err, "Failed to config vsphere plugin") + } + } + + veleroInstallOptions, err := getProviderVeleroInstallOptions(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, + veleroCfg.BSLPrefix, veleroCfg.BSLConfig, veleroCfg.VSLConfig, providerPluginsTmp, features) if err != nil { - return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", objectStoreProvider) + return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider) } veleroInstallOptions.UseVolumeSnapshots = useVolumeSnapshots veleroInstallOptions.UseRestic = !useVolumeSnapshots - veleroInstallOptions.Image = veleroImage - veleroInstallOptions.CRDsVersion = crdsVersion - veleroInstallOptions.Namespace = veleroNamespace + veleroInstallOptions.Image = veleroCfg.VeleroImage + veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace - err = installVeleroServer(ctx, cli, &installOptions{ + err = installVeleroServer(ctx, veleroCfg.VeleroCLI, &installOptions{ InstallOptions: veleroInstallOptions, - RegistryCredentialFile: registryCredentialFile, + RegistryCredentialFile: veleroCfg.RegistryCredentialFile, + ResticHelperImage: veleroCfg.ResticHelperImage, }) if err != nil { return errors.WithMessagef(err, "Failed to install Velero in the cluster") @@ -96,6 +102,64 @@ func veleroInstall(ctx context.Context, cli, veleroImage string, veleroNamespace return nil } +//configvSpherePlugin refers to https://github.com/vmware-tanzu/velero-plugin-for-vsphere/blob/v1.3.0/docs/vanilla.md +func configvSpherePlugin() error { + cli, err := NewTestClient() + if err != nil { + return errors.WithMessagef(err, "Failed to instantiate cluster client to config vsphere plugin") + } + vsphereSecret := "velero-vsphere-config-secret" + configmaptName := "velero-vsphere-plugin-config" + if err := clearupvSpherePluginConfig(cli.ClientGo, VeleroCfg.VeleroNamespace, vsphereSecret, configmaptName); err != nil { + return errors.WithMessagef(err, "Failed to clear up vsphere plugin config %s namespace", VeleroCfg.VeleroNamespace) + } + if err := CreateNamespace(context.Background(), cli, VeleroCfg.VeleroNamespace); err != nil { + return errors.WithMessagef(err, "Failed to create Velero %s namespace", VeleroCfg.VeleroNamespace) + } + if err := CreateVCCredentialSecret(cli.ClientGo, VeleroCfg.VeleroNamespace); err != nil { + return errors.WithMessagef(err, "Failed to create virtual center credential secret in %s namespace", VeleroCfg.VeleroNamespace) + } + if err := WaitForSecretsComplete(cli.ClientGo, VeleroCfg.VeleroNamespace, vsphereSecret); err != nil { + return errors.Wrap(err, "Failed to ensure velero-vsphere-config-secret secret completion in namespace kube-system") + } + _, err = CreateConfigMap(cli.ClientGo, VeleroCfg.VeleroNamespace, configmaptName, map[string]string{ + "cluster_flavor": "VANILLA", + "vsphere_secret_name": vsphereSecret, + "vsphere_secret_namespace": VeleroCfg.VeleroNamespace, + }) + if err != nil { + return errors.WithMessagef(err, "Failed to create velero-vsphere-plugin-config configmap in %s namespace", VeleroCfg.VeleroNamespace) + } + err = WaitForConfigMapComplete(cli.ClientGo, VeleroCfg.VeleroNamespace, configmaptName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Failed to ensure configmap %s completion in namespace: %s", configmaptName, VeleroCfg.VeleroNamespace)) + } + return nil +} + +func clearupvSpherePluginConfig(c clientset.Interface, ns, secretName, configMapName string) error { + //clear secret + _, err := GetSecret(c, ns, secretName) + if err == nil { //exist + if err := WaitForSecretDelete(c, ns, secretName); err != nil { + return errors.WithMessagef(err, "Failed to clear up vsphere plugin secret in %s namespace", ns) + } + } else if !apierrors.IsNotFound(err) { + return errors.WithMessagef(err, "Failed to retrieve vsphere plugin secret in %s namespace", ns) + } + + //clear configmap + _, err = GetConfigmap(c, ns, configMapName) + if err == nil { + if err := WaitForConfigmapDelete(c, ns, configMapName); err != nil { + return errors.WithMessagef(err, "Failed to clear up vsphere plugin configmap in %s namespace", ns) + } + } else if !apierrors.IsNotFound(err) { + return errors.WithMessagef(err, "Failed to retrieve vsphere plugin configmap in %s namespace", ns) + } + return nil +} + func installVeleroServer(ctx context.Context, cli string, options *installOptions) error { args := []string{"install"} namespace := "velero" @@ -103,9 +167,6 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption args = append(args, "--namespace", options.Namespace) namespace = options.Namespace } - if len(options.CRDsVersion) > 0 { - args = append(args, "--crds-version", options.CRDsVersion) - } if len(options.Image) > 0 { args = append(args, "--image", options.Image) } @@ -140,14 +201,14 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption args = append(args, "--features", options.Features) } - if err := createVelereResources(ctx, cli, namespace, args, options.RegistryCredentialFile); err != nil { + if err := createVelereResources(ctx, cli, namespace, args, options.RegistryCredentialFile, options.ResticHelperImage); err != nil { return err } return waitVeleroReady(ctx, namespace, options.UseRestic) } -func createVelereResources(ctx context.Context, cli, namespace string, args []string, registryCredentialFile string) error { +func createVelereResources(ctx context.Context, cli, namespace string, args []string, registryCredentialFile, resticHelperImage string) error { args = append(args, "--dry-run", "--output", "json", "--crds-only") // get the CRD definitions @@ -192,7 +253,7 @@ func createVelereResources(ctx context.Context, cli, namespace string, args []st return errors.Wrapf(err, "failed to unmarshal the resources: %s", string(stdout)) } - if err = patchResources(ctx, resources, namespace, registryCredentialFile); err != nil { + if err = patchResources(ctx, resources, namespace, registryCredentialFile, VeleroCfg.ResticHelperImage); err != nil { return errors.Wrapf(err, "failed to patch resources") } @@ -214,7 +275,7 @@ func createVelereResources(ctx context.Context, cli, namespace string, args []st } // patch the velero resources -func patchResources(ctx context.Context, resources *unstructured.UnstructuredList, namespace, registryCredentialFile string) error { +func patchResources(ctx context.Context, resources *unstructured.UnstructuredList, namespace, registryCredentialFile, resticHelperImage string) error { // apply the image pull secret to avoid the image pull limit of Docker Hub if len(registryCredentialFile) > 0 { credential, err := ioutil.ReadFile(registryCredentialFile) @@ -257,6 +318,34 @@ func patchResources(ctx context.Context, resources *unstructured.UnstructuredLis resources.Items = append(resources.Items, un) } + // customize the restic restore helper image + if len(VeleroCfg.ResticHelperImage) > 0 { + restoreActionConfig := corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "restic-restore-action-config", + Namespace: namespace, + Labels: map[string]string{ + "velero.io/plugin-config": "", + "velero.io/restic": "RestoreItemAction", + }, + }, + Data: map[string]string{ + "image": VeleroCfg.ResticHelperImage, + }, + } + + un, err := toUnstructured(restoreActionConfig) + if err != nil { + return errors.Wrapf(err, "failed to convert restore action config to unstructure") + } + resources.Items = append(resources.Items, un) + fmt.Printf("the restic restore helper image is set by the configmap %q \n", "restic-restore-action-config") + } + return nil } @@ -272,15 +361,17 @@ func toUnstructured(res interface{}) (unstructured.Unstructured, error) { func waitVeleroReady(ctx context.Context, namespace string, useRestic bool) error { fmt.Println("Waiting for Velero deployment to be ready.") - stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, "kubectl", "wait", "--for=condition=available", - "deployment/velero", "-n", namespace, "--timeout=600s")) + // when doing upgrade by the "kubectl apply" the command "kubectl wait --for=condition=available deployment/velero -n velero --timeout=600s" returns directly + // use "rollout status" instead to avoid this. For more detail information, refer to https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#complete-deployment + stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, "kubectl", "rollout", "status", + "deployment/velero", "-n", namespace)) if err != nil { return errors.Wrapf(err, "fail to wait for the velero deployment ready, stdout=%s, stderr=%s", stdout, stderr) } if useRestic { fmt.Println("Waiting for Velero restic daemonset to be ready.") - wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) { + err := wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) { stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, "kubectl", "get", "daemonset/restic", "-o", "json", "-n", namespace)) if err != nil { @@ -295,13 +386,16 @@ func waitVeleroReady(ctx context.Context, namespace string, useRestic bool) erro } return false, nil }) + if err != nil { + return errors.Wrap(err, "fail to wait for the velero restic ready") + } } fmt.Printf("Velero is installed and ready to be tested in the %s namespace! ⛵ \n", namespace) return nil } -func veleroUninstall(ctx context.Context, cli, namespace string) error { +func VeleroUninstall(ctx context.Context, cli, namespace string) error { stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, cli, "uninstall", "--force", "-n", namespace)) if err != nil { return errors.Wrapf(err, "failed to uninstall velero, stdout=%s, stderr=%s", stdout, stderr) diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go new file mode 100644 index 000000000..2dff360f5 --- /dev/null +++ b/test/e2e/util/velero/velero_utils.go @@ -0,0 +1,636 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package velero + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "time" + + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/util/wait" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + cliinstall "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" + "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" + veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" +) + +const ( + BackupObjectsPrefix = "backups" +) + +var pluginsMatrix = map[string]map[string][]string{ + "v1.4": { + "aws": {"velero/velero-plugin-for-aws:v1.1.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.1.2"}, + "vsphere": {"velero/velero-plugin-for-aws:v1.1.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.0.2"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.1.0"}, + }, + "v1.5": { + "aws": {"velero/velero-plugin-for-aws:v1.1.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.1.2"}, + "vsphere": {"velero/velero-plugin-for-aws:v1.1.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.1.0"}, + }, + "v1.6": { + "aws": {"velero/velero-plugin-for-aws:v1.2.1"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.2.1"}, + "vsphere": {"velero/velero-plugin-for-aws:v1.2.1", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.2.1"}, + }, + "v1.7": { + "aws": {"velero/velero-plugin-for-aws:v1.3.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.3.0"}, + "vsphere": {"velero/velero-plugin-for-aws:v1.3.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.0"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.3.0"}, + }, + "v1.8": { + "aws": {"velero/velero-plugin-for-aws:v1.4.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.4.0"}, + "vsphere": {"velero/velero-plugin-for-aws:v1.4.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.1"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.4.0"}, + }, + "main": { + "aws": {"velero/velero-plugin-for-aws:main"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:main"}, + "vsphere": {"velero/velero-plugin-for-aws:main", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.1"}, + "gcp": {"velero/velero-plugin-for-gcp:main"}, + }, +} + +func getProviderPluginsByVersion(version, providerName string) ([]string, error) { + var cloudMap map[string][]string + arr := strings.Split(version, ".") + if len(arr) >= 3 { + cloudMap = pluginsMatrix[arr[0]+"."+arr[1]] + } + if len(cloudMap) == 0 { + cloudMap = pluginsMatrix["main"] + if len(cloudMap) == 0 { + return nil, errors.Errorf("fail to get plugins by version: main") + } + } + plugins, ok := cloudMap[providerName] + if !ok { + return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, providerName) + } + return plugins, nil +} + +// getProviderVeleroInstallOptions returns Velero InstallOptions for the provider. +func getProviderVeleroInstallOptions( + pluginProvider, + credentialsFile, + objectStoreBucket, + objectStorePrefix, + bslConfig, + vslConfig string, + plugins []string, + features string, +) (*cliinstall.InstallOptions, error) { + + if credentialsFile == "" { + return nil, errors.Errorf("No credentials were supplied to use for E2E tests") + } + + realPath, err := filepath.Abs(credentialsFile) + if err != nil { + return nil, err + } + + io := cliinstall.NewInstallOptions() + // always wait for velero and restic pods to be running. + io.Wait = true + io.ProviderName = pluginProvider + io.SecretFile = credentialsFile + + io.BucketName = objectStoreBucket + io.Prefix = objectStorePrefix + io.BackupStorageConfig = flag.NewMap() + io.BackupStorageConfig.Set(bslConfig) + + io.VolumeSnapshotConfig = flag.NewMap() + io.VolumeSnapshotConfig.Set(vslConfig) + + io.SecretFile = realPath + io.Plugins = flag.NewStringArray(plugins...) + io.Features = features + return io, nil +} + +// checkBackupPhase uses VeleroCLI to inspect the phase of a Velero backup. +func checkBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, + expectedPhase velerov1api.BackupPhase) error { + checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "get", "-o", "json", + backupName) + + fmt.Printf("get backup cmd =%v\n", checkCMD) + stdoutPipe, err := checkCMD.StdoutPipe() + if err != nil { + return err + } + + jsonBuf := make([]byte, 16*1024) // If the YAML is bigger than 16K, there's probably something bad happening + + err = checkCMD.Start() + if err != nil { + return err + } + + bytesRead, err := io.ReadFull(stdoutPipe, jsonBuf) + + if err != nil && err != io.ErrUnexpectedEOF { + return err + } + if bytesRead == len(jsonBuf) { + return errors.New("yaml returned bigger than max allowed") + } + + jsonBuf = jsonBuf[0:bytesRead] + err = checkCMD.Wait() + if err != nil { + return err + } + backup := velerov1api.Backup{} + err = json.Unmarshal(jsonBuf, &backup) + if err != nil { + return err + } + if backup.Status.Phase != expectedPhase { + return errors.Errorf("Unexpected backup phase got %s, expecting %s", backup.Status.Phase, expectedPhase) + } + return nil +} + +// checkRestorePhase uses VeleroCLI to inspect the phase of a Velero restore. +func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, + expectedPhase velerov1api.RestorePhase) error { + checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "get", "-o", "json", + restoreName) + + fmt.Printf("get restore cmd =%v\n", checkCMD) + stdoutPipe, err := checkCMD.StdoutPipe() + if err != nil { + return err + } + + jsonBuf := make([]byte, 16*1024) // If the YAML is bigger than 16K, there's probably something bad happening + + err = checkCMD.Start() + if err != nil { + return err + } + + bytesRead, err := io.ReadFull(stdoutPipe, jsonBuf) + + if err != nil && err != io.ErrUnexpectedEOF { + return err + } + if bytesRead == len(jsonBuf) { + return errors.New("yaml returned bigger than max allowed") + } + + jsonBuf = jsonBuf[0:bytesRead] + err = checkCMD.Wait() + if err != nil { + return err + } + restore := velerov1api.Restore{} + err = json.Unmarshal(jsonBuf, &restore) + if err != nil { + return err + } + if restore.Status.Phase != expectedPhase { + return errors.Errorf("Unexpected restore phase got %s, expecting %s", restore.Status.Phase, expectedPhase) + } + return nil +} + +// VeleroBackupNamespace uses the veleroCLI to backup a namespace. +func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string, backupLocation string, + useVolumeSnapshots bool) error { + args := []string{ + "--namespace", veleroNamespace, + "create", "backup", backupName, + "--include-namespaces", namespace, + "--wait", + } + + if useVolumeSnapshots { + args = append(args, "--snapshot-volumes") + } else { + args = append(args, "--default-volumes-to-restic") + // To workaround https://github.com/vmware-tanzu/velero-plugin-for-vsphere/issues/347 for vsphere plugin v1.1.1 + // if the "--snapshot-volumes=false" isn't specified explicitly, the vSphere plugin will always take snapshots + // for the volumes even though the "--default-volumes-to-restic" is specified + // TODO This can be removed if the logic of vSphere plugin bump up to 1.3 + args = append(args, "--snapshot-volumes=false") + } + if backupLocation != "" { + args = append(args, "--storage-location", backupLocation) + } + + return VeleroBackupExec(ctx, veleroCLI, veleroNamespace, backupName, args) +} + +// VeleroBackupExcludeNamespaces uses the veleroCLI to backup a namespace. +func VeleroBackupExcludeNamespaces(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, excludeNamespaces []string) error { + namespaces := strings.Join(excludeNamespaces, ",") + args := []string{ + "--namespace", veleroNamespace, "create", "backup", backupName, + "--exclude-namespaces", namespaces, + "--default-volumes-to-restic", "--wait", + } + return VeleroBackupExec(ctx, veleroCLI, veleroNamespace, backupName, args) +} + +// VeleroBackupIncludeNamespaces uses the veleroCLI to backup a namespace. +func VeleroBackupIncludeNamespaces(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, includeNamespaces []string) error { + namespaces := strings.Join(includeNamespaces, ",") + args := []string{ + "--namespace", veleroNamespace, "create", "backup", backupName, + "--include-namespaces", namespaces, + "--default-volumes-to-restic", "--wait", + } + return VeleroBackupExec(ctx, veleroCLI, veleroNamespace, backupName, args) +} + +// VeleroRestore uses the VeleroCLI to restore from a Velero backup. +func VeleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, backupName string) error { + args := []string{ + "--namespace", veleroNamespace, "create", "restore", restoreName, + "--from-backup", backupName, "--wait", + } + return VeleroRestoreExec(ctx, veleroCLI, veleroNamespace, restoreName, args) +} + +func VeleroRestoreExec(ctx context.Context, veleroCLI, veleroNamespace, restoreName string, args []string) error { + if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil { + return err + } + return checkRestorePhase(ctx, veleroCLI, veleroNamespace, restoreName, velerov1api.RestorePhaseCompleted) +} + +func VeleroBackupExec(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, args []string) error { + if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil { + return err + } + return checkBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted) +} + +func VeleroCmdExec(ctx context.Context, veleroCLI string, args []string) error { + cmd := exec.CommandContext(ctx, veleroCLI, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + fmt.Printf("velero cmd =%v\n", cmd) + err := cmd.Run() + if err != nil { + return err + } + return err +} + +func VeleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error { + args := []string{ + "--namespace", veleroNamespace, "backup", "describe", backupName, + } + if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil { + return err + } + args = []string{ + "--namespace", veleroNamespace, "backup", "logs", backupName, + } + return VeleroCmdExec(ctx, veleroCLI, args) +} + +func RunDebug(ctx context.Context, veleroCLI, veleroNamespace, backup, restore string) { + output := fmt.Sprintf("debug-bundle-%d.tar.gz", time.Now().UnixNano()) + args := []string{"debug", "--namespace", veleroNamespace, "--output", output, "--verbose"} + if len(backup) > 0 { + args = append(args, "--backup", backup) + } + if len(restore) > 0 { + args = append(args, "--restore", restore) + } + fmt.Printf("Generating the debug tarball at %s\n", output) + if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil { + fmt.Println(errors.Wrapf(err, "failed to run the debug command")) + } +} + +func VeleroCreateBackupLocation(ctx context.Context, + veleroCLI, + veleroNamespace, + name, + objectStoreProvider, + bucket, + prefix, + config, + secretName, + secretKey string, +) error { + args := []string{ + "--namespace", veleroNamespace, + "create", "backup-location", name, + "--provider", objectStoreProvider, + "--bucket", bucket, + } + + if prefix != "" { + args = append(args, "--prefix", prefix) + } + + if config != "" { + args = append(args, "--config", config) + } + + if secretName != "" && secretKey != "" { + args = append(args, "--credential", fmt.Sprintf("%s=%s", secretName, secretKey)) + } + return VeleroCmdExec(ctx, veleroCLI, args) +} + +func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, providerPlugins string) ([]string, error) { + // Fetch the plugins for the provider before checking for the object store provider below. + var plugins []string + if len(providerPlugins) > 0 { + plugins = strings.Split(providerPlugins, ",") + } else { + version, err := getVeleroVersion(ctx, veleroCLI, true) + if err != nil { + return nil, errors.WithMessage(err, "failed to get velero version") + } + plugins, err = getProviderPluginsByVersion(version, objectStoreProvider) + if err != nil { + return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", objectStoreProvider, version) + } + } + return plugins, nil +} + +// VeleroAddPluginsForProvider determines which plugins need to be installed for a provider and +// installs them in the current Velero installation, skipping over those that are already installed. +func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins string) error { + plugins, err := getProviderPlugins(ctx, veleroCLI, provider, addPlugins) + fmt.Printf("addPlugins cmd =%v\n", addPlugins) + fmt.Printf("provider cmd = %v\n", provider) + fmt.Printf("plugins cmd = %v\n", plugins) + if err != nil { + return errors.WithMessage(err, "Failed to get plugins") + } + for _, plugin := range plugins { + stdoutBuf := new(bytes.Buffer) + stderrBuf := new(bytes.Buffer) + + installPluginCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "plugin", "add", plugin) + fmt.Printf("installPluginCmd cmd =%v\n", installPluginCmd) + installPluginCmd.Stdout = stdoutBuf + installPluginCmd.Stderr = stderrBuf + + err := installPluginCmd.Run() + + fmt.Fprint(os.Stdout, stdoutBuf) + fmt.Fprint(os.Stderr, stderrBuf) + + if err != nil { + // If the plugin failed to install as it was already installed, ignore the error and continue + // TODO: Check which plugins are already installed by inspecting `velero plugin get` + if !strings.Contains(stderrBuf.String(), "Duplicate value") { + return errors.WithMessagef(err, "error installing plugin %s", plugin) + } + } + } + + return nil +} + +// WaitForVSphereUploadCompletion waits for uploads started by the Velero Plug-in for vSphere to complete +// TODO - remove after upload progress monitoring is implemented +func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, namespace string) error { + err := wait.PollImmediate(time.Minute, timeout, func() (bool, error) { + checkSnapshotCmd := exec.CommandContext(ctx, "kubectl", + "get", "-n", namespace, "snapshots.backupdriver.cnsdp.vmware.com", "-o=jsonpath='{range .items[*]}{.spec.resourceHandle.name}{\"=\"}{.status.phase}{\"\\n\"}{end}'") + fmt.Printf("checkSnapshotCmd cmd =%v\n", checkSnapshotCmd) + stdout, stderr, err := veleroexec.RunCommand(checkSnapshotCmd) + if err != nil { + fmt.Print(stdout) + fmt.Print(stderr) + return false, errors.Wrap(err, "failed to verify") + } + lines := strings.Split(stdout, "\n") + complete := true + for _, curLine := range lines { + fmt.Println(curLine) + comps := strings.Split(curLine, "=") + // SnapshotPhase represents the lifecycle phase of a Snapshot. + // New - No work yet, next phase is InProgress + // InProgress - snapshot being taken + // Snapshotted - local snapshot complete, next phase is Protecting or SnapshotFailed + // SnapshotFailed - end state, snapshot was not able to be taken + // Uploading - snapshot is being moved to durable storage + // Uploaded - end state, snapshot has been protected + // UploadFailed - end state, unable to move to durable storage + // Canceling - when the SanpshotCancel flag is set, if the Snapshot has not already moved into a terminal state, the + // status will move to Canceling. The snapshot ID will be removed from the status status if has been filled in + // and the snapshot ID will not longer be valid for a Clone operation + // Canceled - the operation was canceled, the snapshot ID is not valid + // Canceled - the operation was canceled, the snapshot ID is not valid + if len(comps) == 2 { + phase := comps[1] + switch phase { + case "Uploaded": + case "New", "InProgress", "Snapshotted", "Uploading": + complete = false + default: + return false, fmt.Errorf("unexpected snapshot phase: %s", phase) + } + } + } + return complete, nil + }) + + return err +} + +func getVeleroVersion(ctx context.Context, veleroCLI string, clientOnly bool) (string, error) { + args := []string{"version", "--timeout", "60s"} + if clientOnly { + args = append(args, "--client-only") + } + cmd := exec.CommandContext(ctx, veleroCLI, args...) + fmt.Println("Get Version Command:" + cmd.String()) + stdout, stderr, err := veleroexec.RunCommand(cmd) + if err != nil { + return "", errors.Wrapf(err, "failed to get velero version, stdout=%s, stderr=%s", stdout, stderr) + } + + output := strings.Replace(stdout, "\n", " ", -1) + fmt.Println("Version:" + output) + resultCount := 3 + regexpRule := `(?i)client\s*:\s*version\s*:\s*(\S+).+server\s*:\s*version\s*:\s*(\S+)` + if clientOnly { + resultCount = 2 + regexpRule = `(?i)client\s*:\s*version\s*:\s*(\S+)` + } + regCompiler := regexp.MustCompile(regexpRule) + versionMatches := regCompiler.FindStringSubmatch(output) + if len(versionMatches) != resultCount { + return "", errors.New("failed to parse velero version from output") + } + if !clientOnly { + if versionMatches[1] != versionMatches[2] { + return "", errors.New("velero server and client version are not matched") + } + } + return versionMatches[1], nil +} + +func CheckVeleroVersion(ctx context.Context, veleroCLI string, expectedVer string) error { + tag := expectedVer + tagInstalled, err := getVeleroVersion(ctx, veleroCLI, false) + if err != nil { + return errors.WithMessagef(err, "failed to get Velero version") + } + if strings.Trim(tag, " ") != strings.Trim(tagInstalled, " ") { + return errors.New(fmt.Sprintf("velero version %s is not as expected %s", tagInstalled, tag)) + } + fmt.Printf("Velero version %s is as expected %s\n", tagInstalled, tag) + return nil +} + +func InstallVeleroCLI(version string) (string, error) { + name := "velero-" + version + "-" + runtime.GOOS + "-" + runtime.GOARCH + postfix := ".tar.gz" + tarball := name + postfix + tempFile, err := getVeleroCliTarball("https://github.com/vmware-tanzu/velero/releases/download/" + version + "/" + tarball) + if err != nil { + return "", errors.WithMessagef(err, "failed to get Velero CLI tarball") + } + tempVeleroCliDir, err := ioutil.TempDir("", "velero-test") + if err != nil { + return "", errors.WithMessagef(err, "failed to create temp dir for tarball extraction") + } + + cmd := exec.Command("tar", "-xvf", tempFile.Name(), "-C", tempVeleroCliDir) + defer os.Remove(tempFile.Name()) + + if _, err := cmd.Output(); err != nil { + return "", errors.WithMessagef(err, "failed to extract file from velero CLI tarball") + } + return tempVeleroCliDir + "/" + name + "/velero", nil +} + +func getVeleroCliTarball(cliTarballUrl string) (*os.File, error) { + lastInd := strings.LastIndex(cliTarballUrl, "/") + tarball := cliTarballUrl[lastInd+1:] + + resp, err := http.Get(cliTarballUrl) + if err != nil { + return nil, errors.WithMessagef(err, "failed to access Velero CLI tarball") + } + defer resp.Body.Close() + + tarballBuf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.WithMessagef(err, "failed to read buffer for tarball %s.", tarball) + } + tmpfile, err := ioutil.TempFile("", tarball) + if err != nil { + return nil, errors.WithMessagef(err, "failed to create temp file for tarball %s locally.", tarball) + } + + if _, err := tmpfile.Write(tarballBuf); err != nil { + return nil, errors.WithMessagef(err, "failed to write tarball file %s locally.", tarball) + } + + return tmpfile, nil +} +func DeleteBackupResource(ctx context.Context, veleroCLI string, backupName string) error { + args := []string{"backup", "delete", backupName, "--confirm"} + + cmd := exec.CommandContext(ctx, veleroCLI, args...) + fmt.Println("Delete backup Command:" + cmd.String()) + stdout, stderr, err := veleroexec.RunCommand(cmd) + if err != nil { + return errors.Wrapf(err, "Fail to get delete backup, stdout=%s, stderr=%s", stdout, stderr) + } + + output := strings.Replace(stdout, "\n", " ", -1) + fmt.Println("Backup delete command output:" + output) + + args = []string{"backup", "get", backupName} + + retryTimes := 5 + for i := 1; i < retryTimes+1; i++ { + cmd = exec.CommandContext(ctx, veleroCLI, args...) + fmt.Printf("Try %d times to delete backup %s \n", i, cmd.String()) + stdout, stderr, err = veleroexec.RunCommand(cmd) + if err != nil { + if strings.Contains(stderr, "not found") { + fmt.Printf("|| EXPECTED || - Backup %s was deleted successfully according to message %s\n", backupName, stderr) + return nil + } + return errors.Wrapf(err, "Fail to get delete backup, stdout=%s, stderr=%s", stdout, stderr) + } + time.Sleep(1 * time.Minute) + } + return nil +} + +func GetBackup(ctx context.Context, veleroCLI string, backupName string) (string, string, error) { + args := []string{"backup", "get", backupName} + cmd := exec.CommandContext(ctx, veleroCLI, args...) + return veleroexec.RunCommand(cmd) +} + +func IsBackupExist(ctx context.Context, veleroCLI string, backupName string) (bool, error) { + if _, outerr, err := GetBackup(ctx, veleroCLI, backupName); err != nil { + if err != nil { + if strings.Contains(outerr, "not found") { + return false, nil + } + return false, err + } + } + return true, nil +} + +func WaitBackupDeleted(ctx context.Context, veleroCLI string, backupName string, timeout time.Duration) error { + return wait.PollImmediate(10*time.Second, timeout, func() (bool, error) { + if exist, err := IsBackupExist(ctx, veleroCLI, backupName); err != nil { + return false, err + } else { + if exist { + return false, nil + } else { + return true, nil + } + } + }) +} diff --git a/test/e2e/velero_utils.go b/test/e2e/velero_utils.go deleted file mode 100644 index fcdfec0b5..000000000 --- a/test/e2e/velero_utils.go +++ /dev/null @@ -1,393 +0,0 @@ -/* -Copyright the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "strings" - "time" - - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/util/wait" - - velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - cliinstall "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" - "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" - veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" -) - -func getProviderPlugins(providerName string) []string { - // TODO: make plugin images configurable - switch providerName { - case "aws": - return []string{"velero/velero-plugin-for-aws:v1.2.1"} - case "azure": - return []string{"velero/velero-plugin-for-microsoft-azure:v1.2.0"} - case "vsphere": - return []string{"velero/velero-plugin-for-aws:v1.2.1", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"} - default: - return []string{""} - } -} - -// getProviderVeleroInstallOptions returns Velero InstallOptions for the provider. -func getProviderVeleroInstallOptions( - pluginProvider, - credentialsFile, - objectStoreBucket, - objectStorePrefix string, - bslConfig, - vslConfig string, - plugins []string, - features string, -) (*cliinstall.InstallOptions, error) { - - if credentialsFile == "" { - return nil, errors.Errorf("No credentials were supplied to use for E2E tests") - } - - realPath, err := filepath.Abs(credentialsFile) - if err != nil { - return nil, err - } - - io := cliinstall.NewInstallOptions() - // always wait for velero and restic pods to be running. - io.Wait = true - io.ProviderName = pluginProvider - io.SecretFile = credentialsFile - - io.BucketName = objectStoreBucket - io.Prefix = objectStorePrefix - io.BackupStorageConfig = flag.NewMap() - io.BackupStorageConfig.Set(bslConfig) - - io.VolumeSnapshotConfig = flag.NewMap() - io.VolumeSnapshotConfig.Set(vslConfig) - - io.SecretFile = realPath - io.Plugins = flag.NewStringArray(plugins...) - io.Features = features - return io, nil -} - -// checkBackupPhase uses veleroCLI to inspect the phase of a Velero backup. -func checkBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, - expectedPhase velerov1api.BackupPhase) error { - checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "get", "-o", "json", - backupName) - - fmt.Printf("get backup cmd =%v\n", checkCMD) - stdoutPipe, err := checkCMD.StdoutPipe() - if err != nil { - return err - } - - jsonBuf := make([]byte, 16*1024) // If the YAML is bigger than 16K, there's probably something bad happening - - err = checkCMD.Start() - if err != nil { - return err - } - - bytesRead, err := io.ReadFull(stdoutPipe, jsonBuf) - - if err != nil && err != io.ErrUnexpectedEOF { - return err - } - if bytesRead == len(jsonBuf) { - return errors.New("yaml returned bigger than max allowed") - } - - jsonBuf = jsonBuf[0:bytesRead] - err = checkCMD.Wait() - if err != nil { - return err - } - backup := velerov1api.Backup{} - err = json.Unmarshal(jsonBuf, &backup) - if err != nil { - return err - } - if backup.Status.Phase != expectedPhase { - return errors.Errorf("Unexpected backup phase got %s, expecting %s", backup.Status.Phase, expectedPhase) - } - return nil -} - -// checkRestorePhase uses veleroCLI to inspect the phase of a Velero restore. -func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, - expectedPhase velerov1api.RestorePhase) error { - checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "get", "-o", "json", - restoreName) - - fmt.Printf("get restore cmd =%v\n", checkCMD) - stdoutPipe, err := checkCMD.StdoutPipe() - if err != nil { - return err - } - - jsonBuf := make([]byte, 16*1024) // If the YAML is bigger than 16K, there's probably something bad happening - - err = checkCMD.Start() - if err != nil { - return err - } - - bytesRead, err := io.ReadFull(stdoutPipe, jsonBuf) - - if err != nil && err != io.ErrUnexpectedEOF { - return err - } - if bytesRead == len(jsonBuf) { - return errors.New("yaml returned bigger than max allowed") - } - - jsonBuf = jsonBuf[0:bytesRead] - err = checkCMD.Wait() - if err != nil { - return err - } - restore := velerov1api.Restore{} - err = json.Unmarshal(jsonBuf, &restore) - if err != nil { - return err - } - if restore.Status.Phase != expectedPhase { - return errors.Errorf("Unexpected restore phase got %s, expecting %s", restore.Status.Phase, expectedPhase) - } - return nil -} - -// veleroBackupNamespace uses the veleroCLI to backup a namespace. -func veleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string, backupLocation string, - useVolumeSnapshots bool) error { - args := []string{ - "--namespace", veleroNamespace, - "create", "backup", backupName, - "--include-namespaces", namespace, - "--wait", - } - - if useVolumeSnapshots { - args = append(args, "--snapshot-volumes") - } else { - args = append(args, "--default-volumes-to-restic") - } - if backupLocation != "" { - args = append(args, "--storage-location", backupLocation) - } - - backupCmd := exec.CommandContext(ctx, veleroCLI, args...) - backupCmd.Stdout = os.Stdout - backupCmd.Stderr = os.Stderr - fmt.Printf("backup cmd =%v\n", backupCmd) - err := backupCmd.Run() - if err != nil { - return err - } - err = checkBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted) - - return err -} - -// veleroBackupExcludeNamespaces uses the veleroCLI to backup a namespace. -func veleroBackupExcludeNamespaces(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, excludeNamespaces []string) error { - namespaces := strings.Join(excludeNamespaces, ",") - backupCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "create", "backup", backupName, - "--exclude-namespaces", namespaces, - "--default-volumes-to-restic", "--wait") - backupCmd.Stdout = os.Stdout - backupCmd.Stderr = os.Stderr - fmt.Printf("backup cmd =%v\n", backupCmd) - err := backupCmd.Run() - if err != nil { - return err - } - err = checkBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted) - - return err -} - -// veleroRestore uses the veleroCLI to restore from a Velero backup. -func veleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, backupName string) error { - restoreCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "create", "restore", restoreName, - "--from-backup", backupName, "--wait") - - restoreCmd.Stdout = os.Stdout - restoreCmd.Stderr = os.Stderr - fmt.Printf("restore cmd =%v\n", restoreCmd) - err := restoreCmd.Run() - if err != nil { - return err - } - return checkRestorePhase(ctx, veleroCLI, veleroNamespace, restoreName, velerov1api.RestorePhaseCompleted) -} - -func veleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error { - describeCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "describe", backupName) - describeCmd.Stdout = os.Stdout - describeCmd.Stderr = os.Stderr - err := describeCmd.Run() - if err != nil { - return err - } - logCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "logs", backupName) - logCmd.Stdout = os.Stdout - logCmd.Stderr = os.Stderr - err = logCmd.Run() - if err != nil { - return err - } - return nil -} - -func veleroRestoreLogs(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string) error { - describeCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "describe", restoreName) - describeCmd.Stdout = os.Stdout - describeCmd.Stderr = os.Stderr - err := describeCmd.Run() - if err != nil { - return err - } - logCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "logs", restoreName) - logCmd.Stdout = os.Stdout - logCmd.Stderr = os.Stderr - err = logCmd.Run() - if err != nil { - return err - } - return nil -} - -func veleroCreateBackupLocation(ctx context.Context, - veleroCLI string, - veleroNamespace string, - name string, - objectStoreProvider string, - bucket string, - prefix string, - config string, - secretName string, - secretKey string, -) error { - args := []string{ - "--namespace", veleroNamespace, - "create", "backup-location", name, - "--provider", objectStoreProvider, - "--bucket", bucket, - } - - if prefix != "" { - args = append(args, "--prefix", prefix) - } - - if config != "" { - args = append(args, "--config", config) - } - - if secretName != "" && secretKey != "" { - args = append(args, "--credential", fmt.Sprintf("%s=%s", secretName, secretKey)) - } - - bslCreateCmd := exec.CommandContext(ctx, veleroCLI, args...) - bslCreateCmd.Stdout = os.Stdout - bslCreateCmd.Stderr = os.Stderr - - return bslCreateCmd.Run() -} - -// veleroAddPluginsForProvider determines which plugins need to be installed for a provider and -// installs them in the current Velero installation, skipping over those that are already installed. -func veleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string) error { - for _, plugin := range getProviderPlugins(provider) { - stdoutBuf := new(bytes.Buffer) - stderrBuf := new(bytes.Buffer) - - installPluginCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "plugin", "add", plugin) - installPluginCmd.Stdout = stdoutBuf - installPluginCmd.Stderr = stderrBuf - - err := installPluginCmd.Run() - - fmt.Fprint(os.Stdout, stdoutBuf) - fmt.Fprint(os.Stderr, stderrBuf) - - if err != nil { - // If the plugin failed to install as it was already installed, ignore the error and continue - // TODO: Check which plugins are already installed by inspecting `velero plugin get` - if !strings.Contains(stderrBuf.String(), "Duplicate value") { - return errors.WithMessagef(err, "error installing plugin %s", plugin) - } - } - } - - return nil -} - -// waitForVSphereUploadCompletion waits for uploads started by the Velero Plug-in for vSphere to complete -// TODO - remove after upload progress monitoring is implemented -func waitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, namespace string) error { - err := wait.PollImmediate(time.Minute, timeout, func() (bool, error) { - checkSnapshotCmd := exec.CommandContext(ctx, "kubectl", - "get", "-n", namespace, "snapshots.backupdriver.cnsdp.vmware.com", "-o=jsonpath='{range .items[*]}{.spec.resourceHandle.name}{\"=\"}{.status.phase}{\"\\n\"}'") - fmt.Printf("checkSnapshotCmd cmd =%v\n", checkSnapshotCmd) - stdout, stderr, err := veleroexec.RunCommand(checkSnapshotCmd) - if err != nil { - fmt.Print(stdout) - fmt.Print(stderr) - return false, errors.Wrap(err, "failed to verify") - } - lines := strings.Split(stdout, "\n") - complete := true - for _, curLine := range lines { - fmt.Println(curLine) - comps := strings.Split(curLine, "=") - // SnapshotPhase represents the lifecycle phase of a Snapshot. - // New - No work yet, next phase is InProgress - // InProgress - snapshot being taken - // Snapshotted - local snapshot complete, next phase is Protecting or SnapshotFailed - // SnapshotFailed - end state, snapshot was not able to be taken - // Uploading - snapshot is being moved to durable storage - // Uploaded - end state, snapshot has been protected - // UploadFailed - end state, unable to move to durable storage - // Canceling - when the SanpshotCancel flag is set, if the Snapshot has not already moved into a terminal state, the - // status will move to Canceling. The snapshot ID will be removed from the status status if has been filled in - // and the snapshot ID will not longer be valid for a Clone operation - // Canceled - the operation was canceled, the snapshot ID is not valid - if len(comps) == 2 { - phase := comps[1] - if phase == "New" || - phase == "InProgress" || - phase == "Snapshotted" || - phase == "Uploading" { - complete = false - } - } - } - return complete, nil - }) - - return err -} diff --git a/tilt-resources/examples/deployment.yaml b/tilt-resources/examples/deployment.yaml index 292966a75..b5bf4fdf6 100644 --- a/tilt-resources/examples/deployment.yaml +++ b/tilt-resources/examples/deployment.yaml @@ -39,7 +39,7 @@ spec: value: /plugins - name: AWS_SHARED_CREDENTIALS_FILE value: /credentials/cloud - - name: AZURE_SHARED_CREDENTIALS_FILE + - name: AZURE_CREDENTIALS_FILE value: /credentials/cloud - name: GOOGLE_APPLICATION_CREDENTIALS value: /credentials/cloud diff --git a/tilt-resources/examples/restic.yaml b/tilt-resources/examples/restic.yaml index ab8ad0d16..86d8a6f06 100644 --- a/tilt-resources/examples/restic.yaml +++ b/tilt-resources/examples/restic.yaml @@ -37,7 +37,7 @@ spec: value: /scratch - name: AWS_SHARED_CREDENTIALS_FILE value: /credentials/cloud - - name: AZURE_SHARED_CREDENTIALS_FILE + - name: AZURE_CREDENTIALS_FILE value: /credentials/cloud - name: GOOGLE_APPLICATION_CREDENTIALS value: /credentials/cloud