diff --git a/.github/workflows/nightly-trivy-scan.yml b/.github/workflows/nightly-trivy-scan.yml index 0f5fe0a68..85ce3cdc5 100644 --- a/.github/workflows/nightly-trivy-scan.yml +++ b/.github/workflows/nightly-trivy-scan.yml @@ -22,7 +22,7 @@ jobs: uses: actions/checkout@v6 - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@master + uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 with: image-ref: 'docker.io/velero/${{ matrix.images }}:${{ matrix.versions }}' severity: 'CRITICAL,HIGH,MEDIUM' diff --git a/.github/workflows/pr-filepath-check.yml b/.github/workflows/pr-filepath-check.yml new file mode 100644 index 000000000..2e4b3d6ea --- /dev/null +++ b/.github/workflows/pr-filepath-check.yml @@ -0,0 +1,93 @@ +name: Pull Request File Path Check +on: [pull_request] +jobs: + + filepath-check: + name: Check for invalid characters in file paths + runs-on: ubuntu-latest + steps: + + - name: Check out the code + uses: actions/checkout@v6 + + - name: Validate file paths for Go module compatibility + run: | + # Go's module zip rejects filenames containing certain characters. + # See golang.org/x/mod/module fileNameOK() for the full specification. + # + # Allowed ASCII: letters, digits, and: !#$%&()+,-.=@[]^_{}~ and space + # Allowed non-ASCII: unicode letters only + # Rejected: " ' * < > ? ` | / \ : and any non-letter unicode (control + # chars, format chars like U+200E LEFT-TO-RIGHT MARK, etc.) + # + # This check catches issues like the U+200E incident in PR #9552. + + EXIT_STATUS=0 + + git ls-files -z | python3 -c " + import sys, unicodedata + + data = sys.stdin.buffer.read() + files = data.split(b'\x00') + + # Characters explicitly rejected by Go's fileNameOK + # (path separators / and \ are inherent to paths so we check per-element) + bad_ascii = set('\"' + \"'\" + '*<>?\`|:') + + allowed_ascii = set('!#$%&()+,-.=@[]^_{}~ ') + + def is_ok(ch): + if ch.isascii(): + return ch.isalnum() or ch in allowed_ascii + return ch.isalpha() + + bad_files = [] # list of (original_path, clean_path, char_desc) + for f in files: + if not f: + continue + try: + name = f.decode('utf-8') + except UnicodeDecodeError: + print(f'::error::Non-UTF-8 bytes in filename: {f!r}') + bad_files.append((repr(f), None, 'non-UTF-8 bytes')) + continue + + # Check each path element (split on /) + for element in name.split('/'): + for ch in element: + if not is_ok(ch): + cp = ord(ch) + char_name = unicodedata.name(ch, f'U+{cp:04X}') + char_desc = f'U+{cp:04X} ({char_name})' + # Build cleaned path by stripping invalid chars + clean = '/'.join( + ''.join(c for c in elem if is_ok(c)) + for elem in name.split('/') + ) + print(f'::error file={name}::File \"{name}\" contains invalid char {char_desc}') + bad_files.append((name, clean, char_desc)) + break + + if bad_files: + print() + print('The following files have characters that are invalid in Go module zip archives:') + print() + for original, clean, desc in bad_files: + print(f' {original} — {desc}') + print() + print('To fix, rename the files to remove the problematic characters:') + print() + for original, clean, desc in bad_files: + if clean: + print(f' mv \"{original}\" \"{clean}\" && git add \"{clean}\"') + print(f' # or: git mv \"{original}\" \"{clean}\"') + else: + print(f' # {original} — cannot auto-suggest rename (non-UTF-8)') + print() + print('See https://github.com/vmware-tanzu/velero/pull/9552 for context.') + sys.exit(1) + else: + print('All file paths are valid for Go module zip.') + " || EXIT_STATUS=1 + + exit $EXIT_STATUS diff --git a/Dockerfile b/Dockerfile index 6ce46ca3b..da2355166 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ # limitations under the License. # Velero binary build section -FROM --platform=$BUILDPLATFORM golang:1.25-bookworm AS velero-builder +FROM --platform=$BUILDPLATFORM golang:1.25-trixie AS velero-builder ARG GOPROXY ARG BIN @@ -49,7 +49,7 @@ RUN mkdir -p /output/usr/bin && \ go clean -modcache -cache # Restic binary build section -FROM --platform=$BUILDPLATFORM golang:1.25-bookworm AS restic-builder +FROM --platform=$BUILDPLATFORM golang:1.25-trixie AS restic-builder ARG GOPROXY ARG BIN diff --git a/Dockerfile-Windows b/Dockerfile-Windows index ac22531dc..757da8f80 100644 --- a/Dockerfile-Windows +++ b/Dockerfile-Windows @@ -15,7 +15,7 @@ ARG OS_VERSION=1809 # Velero binary build section -FROM --platform=$BUILDPLATFORM golang:1.25-bookworm AS velero-builder +FROM --platform=$BUILDPLATFORM golang:1.25-trixie AS velero-builder ARG GOPROXY ARG BIN diff --git a/changelogs/CHANGELOG-1.18.md b/changelogs/CHANGELOG-1.18.md index d8a8aa6d8..66c486ddf 100644 --- a/changelogs/CHANGELOG-1.18.md +++ b/changelogs/CHANGELOG-1.18.md @@ -16,7 +16,7 @@ https://velero.io/docs/v1.18/upgrade-to-1.18/ #### Concurrent backup In v1.18, Velero is capable to process multiple backups concurrently. This is a significant usability improvement, especially for multiple tenants or multiple users case, backups submitted from different users could run their backups simultaneously without interfering with each other. -Check design https://github.com/vmware-tanzu/velero/blob/main/design/concurrent-backup-processing.md for more details. +Check design https://github.com/vmware-tanzu/velero/blob/main/design/Implemented/concurrent-backup-processing.md for more details. #### Cache volume for data movers In v1.18, Velero allows users to configure cache volumes for data mover pods during restore for CSI snapshot data movement and fs-backup. This brings below benefits: @@ -24,7 +24,7 @@ In v1.18, Velero allows users to configure cache volumes for data mover pods dur - Solve the problem that multiple data mover pods fail to run concurrently in one node when the node's ephemeral disk is limited - Working together with backup repository's cache limit configuration, cache volume with appropriate size helps to improve the restore throughput -Check design https://github.com/vmware-tanzu/velero/blob/main/design/backup-repo-cache-volume.md for more details. +Check design https://github.com/vmware-tanzu/velero/blob/main/design/Implemented/backup-repo-cache-volume.md for more details. #### Incremental size for data movers In v1.18, Velero allows users to observe the incremental size of data movers backups for CSI snapshot data movement and fs-backup, so that users could visually see the data reduction due to incremental backup. diff --git a/changelogs/unreleased/9403-GabriFedi97 b/changelogs/unreleased/9403-GabriFedi97 new file mode 100644 index 000000000..515549220 --- /dev/null +++ b/changelogs/unreleased/9403-GabriFedi97 @@ -0,0 +1 @@ +Include InitContainer configured as Sidecars when validating the existence of the target containers configured for the Backup Hooks diff --git a/changelogs/unreleased/9502-Joeavaikath b/changelogs/unreleased/9502-Joeavaikath new file mode 100644 index 000000000..419a5850a --- /dev/null +++ b/changelogs/unreleased/9502-Joeavaikath @@ -0,0 +1 @@ +Support all glob wildcard characters in namespace validation diff --git a/changelogs/unreleased/9508-kaovilai b/changelogs/unreleased/9508-kaovilai new file mode 100644 index 000000000..3c224aee3 --- /dev/null +++ b/changelogs/unreleased/9508-kaovilai @@ -0,0 +1 @@ +Fix VolumePolicy PVC phase condition filter for unbound PVCs (#9507) diff --git a/changelogs/unreleased/9516-shubham-pampattiwar b/changelogs/unreleased/9516-shubham-pampattiwar new file mode 100644 index 000000000..14f518c41 --- /dev/null +++ b/changelogs/unreleased/9516-shubham-pampattiwar @@ -0,0 +1 @@ +Fix VolumeGroupSnapshot restore failure with Ceph RBD CSI driver by creating stub VolumeGroupSnapshotContent during restore and looking up VolumeSnapshotClass by driver for credential support diff --git a/changelogs/unreleased/9532-Lyndon-Li b/changelogs/unreleased/9532-Lyndon-Li new file mode 100644 index 000000000..0d5094c22 --- /dev/null +++ b/changelogs/unreleased/9532-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #9343, include PV topology to data mover pod affinities \ No newline at end of file diff --git a/changelogs/unreleased/9533-Lyndon-Li b/changelogs/unreleased/9533-Lyndon-Li new file mode 100644 index 000000000..acd2b37cb --- /dev/null +++ b/changelogs/unreleased/9533-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #9496, support customized host os \ No newline at end of file diff --git a/changelogs/unreleased/9540-sseago b/changelogs/unreleased/9540-sseago new file mode 100644 index 000000000..3606d4f30 --- /dev/null +++ b/changelogs/unreleased/9540-sseago @@ -0,0 +1 @@ +Add custom action type to volume policies diff --git a/changelogs/unreleased/9547-blackpiglet b/changelogs/unreleased/9547-blackpiglet new file mode 100644 index 000000000..f1e546be6 --- /dev/null +++ b/changelogs/unreleased/9547-blackpiglet @@ -0,0 +1 @@ +If BIA return updateObj with SkipFromBackupAnnotation, treat it as skip the resource from backup. \ No newline at end of file diff --git a/changelogs/unreleased/9554-testsabirweb b/changelogs/unreleased/9554-testsabirweb new file mode 100644 index 000000000..18b925797 --- /dev/null +++ b/changelogs/unreleased/9554-testsabirweb @@ -0,0 +1 @@ +Issue #9544: Add test coverage for S3 bucket name in MRAP ARN notation and fix bucket validation to accept ARN format \ No newline at end of file diff --git a/changelogs/unreleased/9560-Lyndon-Li b/changelogs/unreleased/9560-Lyndon-Li new file mode 100644 index 000000000..b90d971fe --- /dev/null +++ b/changelogs/unreleased/9560-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #9475, use node-selector instead of nodName for generic restore \ No newline at end of file diff --git a/changelogs/unreleased/9561-Lyndon-Li b/changelogs/unreleased/9561-Lyndon-Li new file mode 100644 index 000000000..e6661e44d --- /dev/null +++ b/changelogs/unreleased/9561-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #9460, flush buffer before data mover completes \ No newline at end of file diff --git a/changelogs/unreleased/9570-H-M-Quang-Ngo b/changelogs/unreleased/9570-H-M-Quang-Ngo new file mode 100644 index 000000000..603cd75e5 --- /dev/null +++ b/changelogs/unreleased/9570-H-M-Quang-Ngo @@ -0,0 +1 @@ +Add schedule_expected_interval_seconds metric for dynamic backup alerting thresholds (#9559) diff --git a/changelogs/unreleased/9574-blackpiglet b/changelogs/unreleased/9574-blackpiglet new file mode 100644 index 000000000..d2b31f486 --- /dev/null +++ b/changelogs/unreleased/9574-blackpiglet @@ -0,0 +1 @@ +Add ephemeral storage limit and request support for data mover and maintenance job \ No newline at end of file diff --git a/changelogs/unreleased/9581-shubham-pampattiwar b/changelogs/unreleased/9581-shubham-pampattiwar new file mode 100644 index 000000000..f369a8af5 --- /dev/null +++ b/changelogs/unreleased/9581-shubham-pampattiwar @@ -0,0 +1 @@ +Fix DBR stuck when CSI snapshot no longer exists in cloud provider diff --git a/changelogs/unreleased/9614-blackpiglet b/changelogs/unreleased/9614-blackpiglet new file mode 100644 index 000000000..6d15e4f2e --- /dev/null +++ b/changelogs/unreleased/9614-blackpiglet @@ -0,0 +1 @@ +Add check for file extraction from tarball. \ No newline at end of file diff --git a/changelogs/unreleased/9628-priyansh17 b/changelogs/unreleased/9628-priyansh17 new file mode 100644 index 000000000..f65b55e16 --- /dev/null +++ b/changelogs/unreleased/9628-priyansh17 @@ -0,0 +1 @@ +Implement original VolumeSnapshotContent deletion for legacy backups \ No newline at end of file diff --git a/changelogs/unreleased/9634-Lyndon-Li b/changelogs/unreleased/9634-Lyndon-Li new file mode 100644 index 000000000..d0f12babc --- /dev/null +++ b/changelogs/unreleased/9634-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #9626, let go for uninitialized repo under readonly mode \ No newline at end of file diff --git a/changelogs/unreleased/9638-adam-jian-zhang b/changelogs/unreleased/9638-adam-jian-zhang new file mode 100644 index 000000000..3fd99e5e7 --- /dev/null +++ b/changelogs/unreleased/9638-adam-jian-zhang @@ -0,0 +1 @@ +Fix issue #9636, fix configmap lookup in non-default namespaces diff --git a/changelogs/unreleased/9643-priyansh17 b/changelogs/unreleased/9643-priyansh17 new file mode 100644 index 000000000..10870c871 --- /dev/null +++ b/changelogs/unreleased/9643-priyansh17 @@ -0,0 +1 @@ +Fix issue #9641, Remove redundant ReadyToUse polling in CSI VolumeSnapshotContent delete plugin \ No newline at end of file diff --git a/changelogs/unreleased/9653-BassinD b/changelogs/unreleased/9653-BassinD new file mode 100644 index 000000000..d4eefe02a --- /dev/null +++ b/changelogs/unreleased/9653-BassinD @@ -0,0 +1 @@ +Fix service restore with null healthCheckNodePort in last-applied-configuration label diff --git a/changelogs/unreleased/9663-Lyndon-Li b/changelogs/unreleased/9663-Lyndon-Li new file mode 100644 index 000000000..35ae4d5cb --- /dev/null +++ b/changelogs/unreleased/9663-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #9659, in the case that PVB/PVR/DU/DD is cancelled before the data path is really started, call EndEvent to prevent data mover pod from crashing because of delay event distribution \ No newline at end of file diff --git a/changelogs/unreleased/9668-adam-jian-zhang b/changelogs/unreleased/9668-adam-jian-zhang new file mode 100644 index 000000000..a998ded6f --- /dev/null +++ b/changelogs/unreleased/9668-adam-jian-zhang @@ -0,0 +1 @@ +Fix issue #9666, fix node-agent node detection in multiple instances scenario diff --git a/changelogs/unreleased/9676-Lyndon-Li b/changelogs/unreleased/9676-Lyndon-Li new file mode 100644 index 000000000..2fb765e29 --- /dev/null +++ b/changelogs/unreleased/9676-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #9470, remove restic from repository \ No newline at end of file diff --git a/changelogs/unreleased/9677-Lyndon-Li b/changelogs/unreleased/9677-Lyndon-Li new file mode 100644 index 000000000..f722008e9 --- /dev/null +++ b/changelogs/unreleased/9677-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #9469, remove restic for uploader \ No newline at end of file diff --git a/changelogs/unreleased/9682-adam-jian-zhang b/changelogs/unreleased/9682-adam-jian-zhang new file mode 100644 index 000000000..0cb724f5b --- /dev/null +++ b/changelogs/unreleased/9682-adam-jian-zhang @@ -0,0 +1 @@ +Fix issue #9681, fix restores and podvolumerestores list options to only list in installed namespace diff --git a/changelogs/unreleased/9683-Lyndon-Li b/changelogs/unreleased/9683-Lyndon-Li new file mode 100644 index 000000000..25b247bc1 --- /dev/null +++ b/changelogs/unreleased/9683-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #9428, increase repo maintenance history queue length from 3 to 25 \ No newline at end of file diff --git a/changelogs/unreleased/9695-shubham-pampattiwar b/changelogs/unreleased/9695-shubham-pampattiwar new file mode 100644 index 000000000..53deadee3 --- /dev/null +++ b/changelogs/unreleased/9695-shubham-pampattiwar @@ -0,0 +1 @@ +Bump external-snapshotter to v8.4.0 and migrate VolumeGroupSnapshot API from v1beta1 to v1beta2 for Kubernetes 1.34+ compatibility diff --git a/changelogs/unreleased/9701-emirot b/changelogs/unreleased/9701-emirot new file mode 100644 index 000000000..585c1c42a --- /dev/null +++ b/changelogs/unreleased/9701-emirot @@ -0,0 +1 @@ +Update Debian base image from bookworm to trixie \ No newline at end of file diff --git a/changelogs/unreleased/9704-adam-jian-zhang b/changelogs/unreleased/9704-adam-jian-zhang new file mode 100644 index 000000000..59ae81e9a --- /dev/null +++ b/changelogs/unreleased/9704-adam-jian-zhang @@ -0,0 +1 @@ +Fix issue #9703, fix CSI PVC Backup Plugin list options to only list in installed namespace diff --git a/config/crd/v1/bases/velero.io_backuprepositories.yaml b/config/crd/v1/bases/velero.io_backuprepositories.yaml index 19e4ce0dc..a7a2510bd 100644 --- a/config/crd/v1/bases/velero.io_backuprepositories.yaml +++ b/config/crd/v1/bases/velero.io_backuprepositories.yaml @@ -69,9 +69,7 @@ spec: - "" type: string resticIdentifier: - description: |- - ResticIdentifier is the full restic-compatible string for identifying - this repository. This field is only used when RepositoryType is "restic". + description: Deprecated type: string volumeNamespace: description: |- diff --git a/config/crd/v1/crds/crds.go b/config/crd/v1/crds/crds.go index d26e9cfd8..032a1ac84 100644 --- a/config/crd/v1/crds/crds.go +++ b/config/crd/v1/crds/crds.go @@ -29,7 +29,7 @@ import ( ) var rawCRDs = [][]byte{ - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xccXK\x8f۶\x13\xbf\xfbS\f\xf6\x7f\xfd\xcbnP\xb4(|K\xdc\x06\b\x9a\x04\v{\x91;-\x8dlf)\x92%\x87N\xdd\xc7w/\x86\x94lY\xa2\xad\xf5\x1e\x8a\xf2&r\u07bfy\xd9EQ̄\x95_\xd0yi\xf4\x12\x84\x95\xf8;\xa1\xe6/?\x7f\xfe\xc9ϥY\x1c\xde̞\xa5\xae\x96\xb0\n\x9eL\xb3Fo\x82+\xf1g\xac\xa5\x96$\x8d\x9e5H\xa2\x12$\x963\x00\xa1\xb5!\xc1מ?\x01J\xa3\xc9\x19\xa5\xd0\x15;\xd4\xf3\xe7\xb0\xc5m\x90\xaaB\x17\x85w\xaa\x0f\xdf\xcd\xdf\xfc8\xffa\x06\xa0E\x83K؊\xf29X\x87\xd6xI\xc6I\xf4\xf3\x03*tf.\xcd\xcc[,Y\xfaΙ`\x97p~Hܭ\xe6d\xf5\xbb(h\xdd\t:\xc6'%=\xfd\x9a}\xfe(=E\x12\xab\x82\x13*gH|\xf6R\xef\x82\x12nD\xc0\n|i,.\xe13\xdbbE\x89\xd5\f\xa0\xf54\xdaV\x80\xa8\xaa\x18;\xa1\x1e\x9dԄneTh\xba\x98\x15\xf0\xd5\x1b\xfd(h\xbf\x84y\x17\xddy\xe90\x06\xf6I6\xe8I46\xd2v\x01{\xbb\xc3\xf6\x9b\x8e\xac\xbc\x12\x84ca\x1c\xb9\xf9\xd9֧\xa3\xc5\v)\xe7@@\xef-I\xf4\xe4\xa4\xde\xcd\xceć7)\x14\xe5\x1e\x1b\xb1li\x8dE\xfd\xf6\xf1×\xef7\x17\xd7\x00\xd6\x19\x8b\x8ed\aO:\xbd\xf4\xeb\xdd\x02T\xe8K'-\xc5\xe4\xf8\xab\xb8x\x03`\x05\x89\v*\xceC\xf4@{\xecb\x8cUk\x13\x98\x1ah/=8\xb4\x0e=ꔙ|-4\x98\xedW,i>\x10\xbdA\xc7b\xc0\xefMP\x15\xa7\xef\x01\x1d\x81\xc3\xd2\xec\xb4\xfc\xe3$\xdb\x03\x99\xa8T\tBO\x10Q\xd4B\xc1A\xa8\x80\xff\a\xa1\xab\x81\xe4F\x1c\xc1!넠{\xf2\"\x83\x1f\xda\xf1\xc98\x04\xa9k\xb3\x84=\x91\xf5\xcb\xc5b'\xa9+\xca\xd24MВ\x8e\x8bX_r\x1b\xc88\xbf\xa8\xf0\x80j\xe1\xe5\xae\x10\xae\xdcK\u0092\x82Å\xb0\xb2\x88\x8e\xe8X\x98\xf3\xa6\xfa\x9fk\xcb\xd8_\xa8\x1d\x01\x9dN\xac\xa4;\xe0\xe1\xd2\x02\xe9A\xb4\xa2\x92\x8bg\x14\xf8\x8aC\xb7\xfee\xf3\x04\x9d%\t\xa9\x04ʙt\x14\x97\x0e\x1f\x8e\xa6\xd45\xba\xc4W;\xd3D\x99\xa8+k\xa4\xa6\xf8Q*\x89\x9a\xc0\x87m#\x89\xd3ව\x9e\x18\xba\xa1\xd8Ul\\\xb0E\b\x96K\xa7\x1a\x12|а\x12\r\xaa\x95\xf0\xf8/cŨ\xf8\x82Ax\x11Z\xfdv<$N\xe1\xed=t\xad\xf4\n\xb4\xc3\xf6\xb8\xb1X2\xb2\x1c\\f\x95\xb5,SM\xd5Ɓ\x18\xd1_F*\xdf\x02\xf8\xa4&\xba!\xe3\xc4\x0e?\x9a$sH4\x95v|\xde\xe5\x04u\x16s\xdbJ=\x01\xf3\x84\x19\x81\xb4\x17\xd4k\x06$\xa4>\xf5\x94\xac\x937\x90\x89\xe8\b\xee\x14Z\xe8\x12\xdf\xc7|\xd4\xe5q\xc2\xd1O\x19\x16vio\xbe\x81\xa9\tu_hkkƓ-\x82\v\xfa.c\xcf>\xae\x8c\xae\xe5nlh\x7f\x90]\x03wB\xc9\xc0\xdb\xf5@'{\xca\xc9u\xb6\xa5\xe82\x8f\x01\xa9\xe5.\xb8k\xe0\xd5\x12U5j!\x00:(%\xb6\n\x97@.\xe0\x95\x88\x8cj\xe52\"<\x1f'\x80[_\x10\x83\xd4\x15WK;\xacXI\x97\x8c\x9c\xfe\xa8+p\x97kJ\xff\xa0\x0e\xcdX]\x01\xcf\xc6J\x91\xb9w\xe8I\x96\x99\x87\x87\x87\xfb2\x80\xc5|\xa8\xb8\x1d\xd5\x12\xddkjr=\x90ѕc\x1d\x94j\x15\x14\xa5i\xac \xb9U\xd8\xcd\f\xc6\\&\x9ec.i`T\x86\xf0\x14'\x01c\xce*\x8cVG\b\x1e+\xf8\xb6G=\x02\xc3\xc3C\xd2\xfdpWI\x1cxQ\xc3\xd3j\xf7\x9ax|\xb9\x14\xd1\xefN\xe9\":\x96ZbϿ\xae\xfd\xf8\x8cHk\xaaֲ\x96/\xd6\xcc\x1d\x8eq_\x91\x0e\ac\xbe\xc87\xe6\x01M\xae\xa5\rH\x06Q{\xd1d\"A\xc1\xdf3\x9b\"C\x17\xcd28\x17g\x7f\xba\xe5\x95\xef\xd5\xd3I\tO\xbd&\xcc\v\xf8\x04\xee\x1f\xc7\x1c\x9da,\f\x88/\x18\xda~\xf02\xb8\xfaP\x96\x88\xd5x\x1d\x01Ʒ\x11\x94\x16\xfd\x82彮\xcb\xe5\x87\x14z/vSN~JTi\xd3kY@lM\xa0+\b\xd0>\xe7\xe3mT&,\xb5{\xe1\xa7\xec|d\x9a\\^\f\x96\x81[&\\k\xbf\x9f\xf1[\xe6v\x8d\xa2\x1a\xb7\xf0\x02>\x1b\xca?\xdd\xec\xc0%\xea~2M\x0e\x9d\x01={~\x81A+r\x94\x7fc\xaf%a\x93\x1d\xe7\xd7k%\x1dn\xe7\n\tO\xbfU\xf3d\x03\xd3WC\xae\x13h\xe9\x81W\xb9X9Ws\xa9\vٔc\xe9L\x97P:\x13\x85\x94\xce\xcd\r\an\x15U&\x12\xf7\x96\xd6\xd5P$\xb8_\x16\x8eI\x0f\x1c\xfa\xa0\xe8E\x0e\xac#i\x87_b<\xa7\xdf\xcb\xec\xc9\xd7\\:\x05l\xba\xd6x\x95⽐\xea\xea\U000e4cde\x84\xa3\xfb\xf2ws\xc1r\xfa\x9dķ\xfd\xbc\xfdO\xe6獝\xb7{\x14Ή\xe3\xf4\xe8\x1e]z\xfe\xc9^\xf5\x8c\xf3i\x9d\xe8߄\xed\xe9\x1f\x89%\xfc\xf9\xf7\xec\x9f\x00\x00\x00\xff\xff\x18\xd9g\x90\x9b\x14\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xccW͎\xdb6\x10\xbe\xfb)\x06鵲\x1b\x14-\n\xdf\x12\xa7\x01\x82&\xc1\xc2\x0e\xf6NSc\x99Y\x8adɡS\xf7\xe7\u074b!%[\x96dk\xbd\x87\xa2\xbc\x89\x9c\xffo\xe6#U\x14\xc5L8\xf5\x88>(k\x96 \x9c\xc2?\b\r\x7f\x85\xf9\xd3/a\xae\xec\xe2\xf0z\xf6\xa4L\xb9\x84U\fd\xeb5\x06\x1b\xbd\xc4w\xb8SF\x91\xb2fV#\x89R\x90X\xce\x00\x841\x96\x04o\a\xfe\x04\x90\u0590\xb7Z\xa3/*4\xf3\xa7\xb8\xc5mT\xbaD\x9f\x8c\xb7\xae\x0f?\xcc_\xff<\xffi\x06`D\x8dK\xd8\n\xf9\x14\x9dGg\x83\"\xeb\x15\x86\xf9\x015z;Wv\x16\x1cJ\xb6^y\x1b\xdd\x12\xce\aY\xbb\xf1\x9c\xa3~\x9b\f\xad[C\xc7t\xa4U\xa0\xdfF\x8f?\xaa@I\xc4\xe9\xe8\x85\x1e\v$\x1d\ae\xaa\xa8\x85\x1f\b\xb0\x83 \xad\xc3%|\xe6X\x9c\x90X\xce\x00\x9aLSl\x05\x88\xb2L\xb5\x13\xfa\xc1+C\xe8WVǺ\xadY\x01_\x835\x0f\x82\xf6K\x98\xb7՝K\x8f\xa9\xb0_T\x8d\x81D\xed\x92l[\xb07\x156\xdftd\xe7\xa5 \x1c\x1a\xe3\xca\xcdϱ~9:\xbc\xb0r.\x04tβ\xc5@^\x99jv\x16>\xbcΥ\x90{\xacŲ\x91\xb5\x0e͛\x87\x0f\x8f?n.\xb6\x01\x9c\xb7\x0e=\xa9\x16\x9e\xbc:\xed\xd7\xd9\x05(1H\xaf\x1c\xa5\xe6\xf8\xbb\xb88\x03`\aY\vJ\xeeC\f@{lk\x8ce\x13\x13\xd8\x1d\xd0^\x05\xf0\xe8<\x064\xb93y[\x18\xb0ۯ(i\xde3\xbdA\xcff \xecm\xd4%\xb7\xef\x01=\x81Gi+\xa3\xfe<\xd9\x0e@69Ղ0\x10$\x14\x8d\xd0p\x10:\xe2\xf7 Lٳ\\\x8b#xd\x9f\x10M\xc7^R\b\xfd8>Y\x8f\xa0\xcc\xce.aO\xe4\xc2r\xb1\xa8\x14\xb5C)m]G\xa3\xe8\xb8H\U000e5d91\xac\x0f\x8b\x12\x0f\xa8\x17AU\x85\xf0r\xaf\b%E\x8f\v\xe1T\x91\x121i0\xe7u\xf9\x9do\xc68\\\xb8\x1d\x00\x9dW\x9a\xa4;\xe0\xe1\xd1\x02\x15@4\xa6r\x8ag\x14x\x8bK\xb7\xfeu\xf3\x05\xdaH2R\x19\x94\xb3\xe8\xa0.->\\Mev\xe8\xb3\xde\xce\xdb:\xd9DS:\xab\f\xa5\x0f\xa9\x15\x1a\x82\x10\xb7\xb5\"n\x83\xdf#\x06b\xe8\xfafW\x89\xb8`\x8b\x10\x1d\x8fN\xd9\x17\xf8``%j\xd4+\x11\xf0?ƊQ\t\x05\x83\xf0,\xb4\xbat\xdc\x17\xce\xe5\xed\x1c\xb4Tz\x05\xda>=n\x1cJF\x96\x8b˪j\xa7d\x9e\xa9\x9d\xf5 \x06\xf2\x97\x95\x1a\xa7\x00^\x99D7d\xbd\xa8\xf0\xa3\xcd6\xfbBSm\xc7\xeb혡6b\xa6\xad\xcc\t8.8b\x90\xf6\x82:d@B\x99\x13\xa7\x8c&y\x03\x99\x84\x8e`\xa60\xc2H|\x9f\xfa\xd1\xc8\xe3D\xa2\x9fFT8\xa5\xbd\xfd\x06vGh\xbaF\x9bXG2\xd9\"\xf8h\xee\n\xf6\x9c\xe3ʚ\x9d\xaa\x86\x81v/\xb2k\xe0N8\xe9e\xbb\xee\xf9\xe4L\xb9\xb9α\x14m\xe71 ;UE\x7f\r\xbc\x9dB]\x0e(\x04\xc0D\xad\xc5V\xe3\x12\xc8G\xbcR\x91\xc1\xac\\V\x84\xef\xc7\t\xe0\xd6\x17\u00a0L\xc9\xd3\xd2\\V\xec\xa4mFn\x7f4%\xf8\xcbgJw\xa1\x89\xf5\xd0]\x01O\xd6)1\xb2\xef1\x90\x92#\a\xaf^\xdd\xd7\x01l\xe6C\xc9t\xb4S\xe8'2~Ǽ\xcd9\x0e\x1b\xf0\x86\x93\x03?~\xf0\xf4\\z\xc9\xdc?^\x9a\xe8N|\xdeH3\x9bi\xa6S\xe6v\xa4ÈIg\xcb&\xb2F/\xf5\xe1\x1d\xf3ó\xaa<\xf6\xae\xceb\x9c\xecz2c4\xd1\x13\xe9U\xedYlO\x82b\xb8\x87\xef\x93B[M\x19\xbdO\xf7i\xde\xe5gԋ\x19_\x8b@\x1db\xe3G\xed\x04\xee\x1f\x87\x1am`l\f\x887\x18\xdan\xf1Fp\rQJ\xc4rx\xc5\x03\xe3[\vʏ\xe7\x82\xed\xbd\x8c9Ɖ\x1fC\x10\xd5T\x92\x9f\xb2T~=5* \xb66\xd2\x15\x04h?\x96\xe3mT&\"u{\x11\xa6\xe2|`\x99\xb1\xbe\xe8]\xb0\xb7B\xb8Fi\x9f\xf1\xdb\xc8\xee\x1aE9\xa4\xc5\x02>[\x1a?\xba\xc9j\x12M\xb7\x99&\x89\xbc'ϙ_`И\x1c\xf4\xdf0kEX\x8f^\x91\xd7g%/ik\xa7\x91\xf0\xf4\xff7.\xd6\v}\xd5\xd7:\x81\x96\x0f\xf8y\x94&\xe7j/\xb5%\x9bJ,\xaf\xe9\x11\xcakb\x90\xf2\xba\xf9j\x80[C5R\x89{G\xebj)2\xdc\xcf+\xc7d\x06\x1eC\xd4\xf4\xac\x04\xd6I\xb4\xc5/+\x9e\xdb\xefy\xf1\x8c\xcf\\^\x05lZj\xbc*\xf1^(}\xf5x2\xd9@\xc2\xd3}\xfd\xbb\xb9P9\xfd{\xf0n\xb7o\xff\x97\xfdy\xe3\x1d\xd9\x1e\n\xef\xc5q\xfa\xea\x1el\x06\xfe\r.;\xc1\x85\xfc\x9c\xe8\xee\xc4\xed\xe9/\x7f\t\x7f\xfd3\xfb7\x00\x00\xff\xff\x96֥5\xef\x13\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=]s\xdb8\x92\xef\xf9\x15(\xdd\xc3\xecnI\xf6\xa6\ue8ee\xfc\x96q\x92\x1d\xd5\xcc$\xde\xd8\xe3}\x86Ȗ\x841\bp\x00P\xb6\xf6\xee\xfe\xfb\x15\x1a\x00?DP\x04eٓ\xdd\r_\x12\x8b`\x03\xfd\xddh4\x80\xc5b\xf1\x86\x96\xec\x1e\x94fR\\\x11Z2x2 \xec_\xfa\xe2\xe1\xbf\xf5\x05\x93\x97\xbb\xb7o\x1e\x98ȯ\xc8u\xa5\x8d,\xbe\x80\x96\x95\xca\xe0=\xac\x99`\x86I\xf1\xa6\x00Csj\xe8\xd5\x1bB\xa8\x10\xd2P\xfb\xb3\xb6\x7f\x12\x92Ia\x94\xe4\x1c\xd4b\x03\xe2\xe2\xa1Z\xc1\xaab<\a\x85\xc0C\u05fb?_\xbc\xfd\xaf\x8b\xff|C\x88\xa0\x05\\\x91\x15\xcd\x1e\xaaR_쀃\x92\x17L\xbe\xd1%d\x16\xe4Fɪ\xbc\"\xcd\v\xf7\x89\xef\xce\r\xf5{\xfc\x1a\x7f\xe0L\x9b\x1f[?\xfeĴ\xc1\x17%\xaf\x14\xe5uO\xf8\x9bfbSq\xaa¯o\bљ,\xe1\x8a|\xb2]\x944\x83\xfc\r!~\xd4\xd8\xe5\xc2\x0fx\xf7\xd6AȶPP7\x16Bd\t\xe2\xdd\xcd\xf2\xfe\xdfo;?\x13\x92\x83\xce\x14+\r\xe2\xfe\xbf\x8b\xfaw\xe2GI\x98&\x94\xdc#\x8eDy\x92\x13\xb3\xa5\x86((\x15h\x10F\x13\xb3\x05\x92\xd1\xd2T\n\x88\\\x93\x1f\xab\x15(\x01\x06t\v^\xc6+m@\x11m\xa8\x01B\r\xa1\xa4\x94L\x18\xc2\x041\xac\x00\xf2\x87w7K\"W\xbfBf4\xa1\"'Tk\x991j ';ɫ\x02ܷ\x7f\xbc\xa8\xa1\x96J\x96\xa0\f\vDwOK\x92Z\xbf\x1e\xc3\xd5>\x96<\xee+\x92[\x91\x02\x87\x96'1䞢\x16?\xb3e\xbaA\x1f\x85\xcc\xfeL\x85\x1f\xfe\xc5\x01\xe8[P\x16\f\xd1[Y\xf1\xdcJ\xe2\x0e\x94%`&7\x82\xfd\xbd\x86\xad\x89\x91\xd8)\xa7\x06\xb4\xa5\x8c\x01%(';\xca+\x98[\xa2\x1c@.\xe8\x9e(\xb0}\x92J\xb4\xe0\xe1\a\xfap\x1c?K\x05\x84\x89\xb5\xbc\"[cJ}uy\xb9a&\xe8W&\x8b\xa2\x12\xcc\xec/QUت2R\xe9\xcb\x1cv\xc0/5\xdb,\xa8ʶ\xcc@f\xd9|IK\xb6@D\x04\xea\xd8E\x91\xff[\x10\x0f\xdd\xe9\xd6\xec\xad\xd8j\xa3\x98ش^\xa0~L`\x8fU\x1d'\x8c\x0e\x94C\xb1\xe1\x82\xfdɒ\xeeˇۻ\xb6\xa02\xed\x99Ғ\xd7!\xfeXj2\xb1\x06\xe5\xbe[+Y L\x10\xb9\x13U\x94s\xce@\x18\xa2\xabU\xc1\x8c\x15\x83\xdf*\xd0V\a\xe4!\xd8k\xb4Ad\x05\xa4*s+Ƈ\r\x96\x82\\\xd3\x02\xf85\xd5\xf0ʼ\xb2\\\xd1\v˄$n\xb5-\xebacG\xde\u058b` \aX\xeb\f\xcbm\tYG\xd1\xecWl\xcd2\xa7Nk\xa9\x1a\xbb\xe3l`\x97BqշO\xa6٭\xa0\xa5\xdeJs\xc7\n\x90\x959l1&kȼ\xdb\xe5\x01\x940B?^\xb4Y\x95\x86\xdc*\xed#e\x06\xc7|}\xbb$\xf7h\xac\xc2\xd7h\xb4*ML\xa5\x84\x95\x92H__\x80\xe6\xfb;\xf9\x8b\x06\x92W(ܙ\x02\xa4Ü\xac`m%A\x81\xfd\u07be\x02\xa5,m4\x0e@V=cc\x9f\xbb-X\xdaҊ\x1b\xaf'L\x93\xb7\x7f&\x05\x13\x95\xe9\x89\xda בR\xd4\xd0B\xee@\x9dB\xc4\xf7\xd4П\xed\xc7\a\xb4\xb3@\tB\xb5\xc4[y:\xae\xf6\xf82\xc6m\xf7,\xd7-\x88L\x93ٌHEf\xce\x03\xcf\xe6\xee\xeb\x8aq\xb3`\xa2\xdd\xc7#\xe3<\xf42\ryGC\xc7P}'?j'\xbc'\xd1b\x00V\x8b4\x8f[0[P\xa4\x94\xb5\xc7[3\x0eD﵁\xc2\x13&x\x11\x8fO\xa4'\xd4\x1d\xce=\bm\xe9\xea\x11\xe9#/*\xce\xe9\x8a\xc3\x151\xaa\x82\x01ڬ\xa4\xe4@\xc5\bq\xbe\x806,;\ai\x1c\xa4\ba\x94\x7fѡ\x00:M\xfa\x00\x84F@{\x9aY\xef\xccy\x8b\xb0]\xaaD\xc7T*Ȭվ\xf2ހ\x01G\x0f$$\xe1Rl@\xb9\xdem\xa4\x12\x04L\x81\x15\xb8\x9cXC\xab\x80[oB֕\xb5\xc1\x17\xc4j\xf7\xa0\f0\xa1\rЈp>\x83?\xf0\x94\xf1*\x87\xfc\xda\x05^\xb76~\xccC\xd4ܳ\x9a)|\xfap\x14\xa2\xf7Μe\x18\x04\xfaxo\x81qkLL\x1b'\xbd/\xc1\x85Ζ\x95~؍\xf7=j\x0f4\x18\xfb\xd1\xecO\xb39r\xb8\xdbk\xb7\x0fM\xa8\x82\x9a,\xc9v\x13\x8a\xd2\xec\xfb\xad\x99\x81\"Bţ\xf6$\x91\x9fT)\xba\x1f\xe0f\x1d\xff\x9f\x91\x9fC0\x0f8*B\xb3W\xe6\xe9a\xbf\xff\xcc\\=\x0f\x1f5\xcev)\x13\x96\x7fv\xe2\xd9a\x9fv\xf37K6!M\x04\x1e\x13\x0e\x1eN͎p\xebw\"\xd6Yd~H\xc8k\xd9\xf2\xc2\xfb\x0fI\xa9\xad\x94\x0fc\xd4\xf9\xc1\xb6i&E$ì\nY\xc1\x96\xee\x98T\x1e\xf5\xc6\xd5\xc2\x13d\x95\x89j=5$g\xeb5(\v\xa7\xdcR\r\xdaM\x93\x87\t2\x1c\xbe\x93\x96\x19\x89\xbe<\xc0\xa3a\xa4e\x13b>4t\x1bG\x1cz\xc9\xf0\u0601\xda\xf0\x1a\x9dq\xcev,\xaf(G\xbfLE\xe6\xf0\xa1\xf5\xb8bV\xe6\b\x93{c\x8eJ\xa6{\\@\x10\x90\xb2L\xea̔\xa4\x00\x1b\xf3\x16vN\xd0o:\x8c\xf9\x8a\xdaXE\x0eaO\x90Y\xaa\xe2\xa0}W9\x86\x91\x8d͘7L\xc1D\x04\xe1t\x05\x9ch\xe0\x90\x19\xa9\xe2\x14\x19\xe3\xb3{R\x8c\xe0\x00!#\x96\xaf;\xd3h\x108\x02\x92\xe0\x14n˲\xad\v\xf5\xac\x10!\x1c\x92K\xb0\x01\x9f!\xb4,y\xc4]4\xcfQ\xe6\xfbN\x8e\xe9z\xf3\x8ch\xfd!\xbc\x98\xfe7O\x82\xcdl\x9e(i\x1b\xfd\xeaR\xb6\x16\x87\xf8\x9c\xb6y\xfe9\t\x1b,\xff\tB{D\xfb\tf\x85\x92ezPn-U\x19\xe8\v\x1bNa\xa43'̄_\xc74\xa1\x13s\xf5\x92e\x1d\"|ݼ\x99.\xf4\x89\xacIщ\x17bL\xdd\xc5? _\xd0e\xdcz\x8f\x91̓\x9f\xda_\xcd\t[\xd7D\xcf\xe7d\u0378\x01u@\xfd\x93L}\xe0\xcc9\x88\x91\xe2\xf5\b\xa6\xefM\xb6\xfd\xf0dC0ݬT%\xd2\xe5\xf0c\x17Ȇh\xbf\xeb\x9eG\xe0\x12Lc3\x05\x05\xa6\xc7q\xc6\xd4\xfe\x05C\xabw\x9f\xde\xc7\xe7W\xed'A\xf2z\x88\x8c(\x9d{\xde\x1d`\xd4\x1e\x9f\x0f\xe1\xc3\x1b\x8c\x81\xea\t\x90[\n\x99\x13J\x1e`\xefB\x17*\x88\xe5\x0f\r\x8d\x13\xbaW\x80k2(g\x0f\xb0G0\xf1E\x96\xfe\x93*\r\xeey\x80}J\xb3\x03\x1a\xda11\xed\x17\x8f,\x9d\xec\x0fH\b̭\xa7\x8a\x81{\xbc*D\x964\xe2O\xa2-\tO\xa0\xfd\th&\x89J\xbb\x8f\xf6*%J\xc0w\xda\xf1\xd2j̖\x95hV1\xe3 \xd7\xc9\fu\xcf=\xe5,\xaf;r:\xb2\x14s\xf2I\x1a\xfbχ'\xa6\xfdB\xe6{\t\xfa\x934\xf8ˋP\xd4\r\xfc%\xe9\xe9z@E\x13\xce\xca[\x82\xb5\x97\xe2\x9cO\xb3\xd2VӞi\xb2\x14v\xba\xe2H\x92\xd8\x15\xae\xba\xba\xee\\GE\xa5q\x15MH\xb1pi\x9bXO\x9e\xdeRu\xc8\xfd\xecN}\x87w\xd6Y\xb87n\xed\x97\xd3\f\xf2\xb0\\\x83\x8b\x92\xd4\xc0\x86e\x89\xfd\x15\xa06@Jk\xc2\xd3$\"Ѱzl\xa6\x89O\x9a\xf7n?O\x8b\x87z\x8d\x7fa]\xce\xc2C0\xb2H\xa0\x81\xb7\xdd\xf98>\v\xab\xb3\t\xad\x82$\x8c6\x1dX\xb3\x1cn\x9aB\x94g\x90\x03\xbd8\x868\xa3ܥy\x8eu.\x94\xdfL\xf0(\x13da\xaaih\x8dݹ\xe0\x82\xe2R\xcb\xffXO\x8b\xda\xf4\x7f\xa4\xa4L\xe9\v\xf2\x0eKZ8t\xde\xf9\xa4Y\vLB\x97X\x92b\xe5gG\xb9\xf5\xfdր\v\x02\xdcE\x02r\u074b\x8b\xe6\xe4q+\xb5s\xdb\xf5\"\xce\xec\x01\xf6n\xc5p\xb4˶\x91\x99-\xc5\xcc\xc5\x10=\x83Q\a\x1cR\xf0=\x99\xe1\xbb\xd9sB\xa9DIMl\xd6\x11т\x96i\x12\x8a%E\xa9\x81\xba\x9d\xb0\x86 \xc4~X\x97\xca\xd8 \xfb\x18\xb6I\"ZJ\x1dY\xc8\x1f\x18ʈ\xf0\xdeHm\\\xbe\xac\x133G\x13j2$\xd1\b]\xbb\xfa%\xa9B\xb1\x895\xcac\xa9\xdf\xf6s\xb7\x05\r~\xbd\xc2'\xe6\x1cP;\xb3\x9b5\xfa\xed\xac\xfḓ\x97`'4È\x05\xbf-\x95\xcc@Gײ\x9b'\xc1_D\xaa2ڸ\xd79G\xeafI\xae$\xe3x\n4<\xe9!\xaf%\xc4\xc4\xf9\u0087\xa7VB\xd4\xea\xbe\xfd{LƦ\x8e\x8b`\xc9`Q\xd0\xc32\xa5\xa4!^\xbb/\x836x@n\xf2\xa16\x15Z\x82T_^\v\xe0\xd7\x10(\x14L,\xb1\x03\xf2\xf6\x05\x02\voCc\xc5&\xb1\xe7\xb4P\xf6:t\xd2p\xa7\xfe\xc1\xa9r)q\xa9@A\x87y\xfd\xac:ơB\x9aVBbB\xb8Y\xca\xfc;M\xd6Li\xd3\x1e\x82\x1e(S\x89\x82\x998\xf1\x12\x1f\x94:i\xde\xf5\xd9}\xd9Jwm\xe5c(\xcfr\x84I\xc4\x1cח\x80\xb05a\x86\x80\xc8d%0\x81c\xf5\x18\xbbp\xc4u\x16\x96\xa5*I\x9a\xf6\xdb\aDU\xa4\x11`\x81\x92\xc2\xc4\xd1LO\xbb\xf9G\xca\xf8K\xb0\xcd\fU\xb1Ş\xd3t\"\x94\xb8\xb5\v\xf2\n\xfaĊ\xaa \xb4\xb0\x1f\xc0\xb0\x8c\x0f\xa1\xdd+\xc5\xc8E\xc5\r+9.\xa4\xeeX\x1eM6\x98-\xec\xeb\x034~\x95\xb8\xf5ԟ\x04\xf3\xf9K-\xb5\x17\a\x91>\xd5\xe4\x118'4\xa6W=\xcc3w\x12S&\x17`\xfd\x91\xd5N\x7f0\x88?\xbei\xee\xc4\x1dwעW+b)&*\x86O\x91\x19t\x1c)\xf6\xa6\x17\xc1\xba8\x1c\x7f\xfb\xad\x02\xb5'x\x8eM\x1d\xe74\x9b\xc0\xbcbj;\x11\v\xa6\u009b\xad\xa1\xfcy/\xe8oT\x99\xbc\x13\xce\xeb\x1e\x8e\a\xbf\xb16\xa2\x99\xd4X\xc3g\xe7+\xd1>\x06>\x17\xb2\xfe:\xf2\xd9X\x80\x9c\xba[\xeae\xa78\xd3'9\xa3QEz\xe4\xf7;\xed\x82:e\xf7SZ\x01\xc0\xe8n\xa7\x97\x9a\xf2\x8cMz\x92㼴\xddL\xd3\x16\v_p\xf7\xd2K\xecZJ\xa4T\xca.\xa5itz\x85]I\xaf\xba\x1b\xe9\xb5v!%\xef>J*qI^\x05N-Q9q;\xcd\xf8\x1a\xef\xf1\xddD\t\xbb\x88\x12V\x7fǑ<\x01\xbd\x84]B\xd3v\a%\xf0,U\x15_q\x17\xd0+\xee\xfey\xed]?#\x925\xf2z\xda\ue793\x97,\xa4\xcaA\x1d]\xf6I\x95£\xf2\x972\xb7\xe9\x0e\xe4`\xbd#\x9c\xfag[u\xe2et\x0f\xfe\xa0Q\x97G\xb3\xa2\xe4{;C!\xb3\xf6\a\xa7I@T\xdaBo7\x92\xb3,\x12\xbbE\xcffr\x8d{\x87e\xe0\x89QY\xbbd\xa0\xb4\r\xe3\xa1\x1b\x86y\xdd#0גs\xf98q\xeeOK\xf6\x17<\xb9\xfb\x19١w7K\x84\x11\xc4\x03\x8f\x02\xaf\x8b\xb3jlV`\xddr\x83\xe7\x90\xee/\xd7\x1d\x88\xdd:\xc7\xf6Ḑ\xbbs\x90CX\xe0Mg&\xadu\xb9Y\xbaq\f\xf5be\x86\x8a=\x91XQc\xb6L勒*\xb3w\x85\x1a\xf3\xce\x18\x82/=\x96\xdd\x19\xf4\x1e\xfd\xb3\x9d\xa3\xe4\rG:\xe3\n\xe5\xbe\xec.\xfa\x1e\xd2\xee\x94q\f\xef^\x1cݷx\xc6q\f\x87%\v\xa4T\xe4\xe7h\xe5\xd7ٲfڟL\xfc\xb3\xdc\xc1\xfbh\xf6\xacC\x9eۃ\xe6\x91\xf2\xac\x00\xd1\x1d\xba;X\xa5\xba\x02<\x90\xb7\xff\xea\x19\xf5V\xa1k\x7f\xa6\xea)\x89\xb2\xdb.\x88\b~\xe1\x84\xd9\xd0Y\xcc>\xe1\x01\xf0{rs\x8fs\xb4ڴy\x15\xf5s\xb4\x90*\v\x8b\xc1\x118\xfe\x83\xef\xcf_\x9a\xa6\x8dTt\x03?Iw\xc6\xf6\x18ۻ\xad;g\xaf\xfb\xa8'ԏ\x06\xa5\x89\x1d\xc0\xebO\xfb>\x00\xd6\xd4|\xf7\x0e5\xb6\xa3\x9cxL\xb31\xfc\x14\xbe\xdf\xdd\xfd\xe4\xb02\xac\x80\x8b\xf7\x95+w\xb06Q\x83%q\xc0\xd6AZ\xd9\xffn\xe5#\x1e\xfe\x1b\xcfc\x86;\x13\x1ad\x14`\xb19\x96 NB\xa9*\xb9\xa49\xa8k)\xd6l3\x82\xdd/\x9d\xc6\an6\xc3\x1f=r\xb5\x8f\n\xf0\xcf\\\x83`c\x1e\u0381\x7fd\x1c\xb4\x1bV\x82\x01\xbe\xe9\x7fU\xdb\xe3\xaaX\xb9\x18nm_\xd6\x1d\f\xf88\x87\x16\xa6\xa2KP6\x8arI\xebJ\aY\x1dF\xbc\xe1\b\x13\x066П\x05\x1e\xb1\xc0\xeeTit\x9f\xc1\x9c\xe0\\\xe6\xc7X~\xab\x83\xfc\xfd\xf0\x97\a\x9cl\xa5\xbcb'\xee\xb9 \xe4\xe6\xfeZ\x93J\xe4\x98.\xbe\xff\xcb\xed$\xa9\xdbuN\xae\x0f\xda:fT\xef\xe3_\xb5\x82㖽pѱ\\G\x10\x18\x82Ӻ\a\xe4\x91\x19\x7fp\xd7yOZ\x1d\x9a\xf2\f\xddp\x80G\xfa\x8f\xdfq\xe0N\xfe\xf77\xa3xu\xac\x14\x1e\x93\xeao\x05\xc0cEO\xba\xe6`U\x17l\xd5\xc5_\xfa\x9d1P\x94&\x16k\x8c\x9b\xc3\xef\x8f\x01\xac\xe34i(oi%\r\rb\x91\xb6ދ\xecXa\x99\xb7FG\xb8yL\x1fc\x04\xb8\xf6\xfb!\xceF\x80\x1a\xe0\x10\x01t\x95e\xa0\xf5\xba\xe2|_o\xc7\xf8J\xa8\xf1\x912~>R8h\x83\x82`\xd1;\ni\x14a_\xee\r\"\x0f\x9a\x1e\xb6*M#\x85炯\x86Ԇ\x16']\xd8p\xdd\a\x83W\xf6\xa8\xbcUTI\xeb\xb1Sݰ?\xe6\\\x1ap\xeeK\x9cdYh\x90\x13\u0601 \xd6;;\x12\x87;\xa7&B\xf1;\\\x9d\x87\v\xfe.\xa4B\xa2\x17\x13\x11\x9f\xed\xd0x\x01\xcew\xba\x86\x89\xb5\xa2x\x9fI\x9f\b\xfd\xe0\xd7e+\xael\xf4\x0f\v\v\u2d285j\x9b3ͺ~\xe1yF\xee\xfav9\x04\xee\x14\x13\u05ff\xee\xe5\x99j\xdcG\xf7Y&\xad\x8f\xee$\x83\x16\x81X\xcb\xf8\xf9qGU?\xedPw\xfc\xd2\x05\x1cY\xd8CG9\xf7\x1b\x1d\vКn\xc2i\xee\x8fv\xea\xb1\x01\x01.=\xe7\x16O\"@\x9b]qݳ̝\xca\xd0\xccT\xd4w\x10\n|[\xad\xbeӄ\xcb\x18T\xbcЅ\x85\x9b\xc2\u009cl\"\xa1\x9eJ\xa6R\xe6p\x1fꆖ6\x18\t#w\x9a\xbb݀\xb3\r\xb3s\x1d˹\rU+\xba\x81E&9\a\xb4\xd6\xfdq\xbd\xa4\xae\xfb\xbd\x87_\x80\xeaQ\xd4>\xb6\xdb\xfa\x15@\xc7m\xb7\xf0M]\xb9;\xde\xdee\x98\x82\xe6\"\xbdހ$v<)PvT\x88\xde2\xd7\x1fi\xbbm\xd0:o\x96}\x9e\xd7_27\xf7y\x81\xb8<\x16\xf4W\xa9\xe6\xa4`\xc2\xfeCE\xee\x16\xf0\xc2Ǔƿ\x95\xf2\xe16\x12\xc4\xf6\x06\xffCݰY\xea`\xc2\r\x1b7\x8c\xaed\xe5W\xdf\xeb\x806\xbe\xac\x82'\xf3\x9fy\xba\x890\x8f\xf8\x83\x1e:\x83\x19\xdd\x1f:\x90F]\x81\xeby\x00\xd6m\xb8Ɍ\xf3\xfd\xfc\x10\xf2\xc1\xad\x89\r\xec\xd6\xcd\x05>\fh\xce#\x18\xe8(\xacHE\x81\xd4\a_\xb4\r\xfa)\xb3^O\xe6\xa1`\xb2G\xe3\x1f\x9a\xd6Ctt\xc3l\x85{\x03\bv\x82\xc0\xf3N\xd8\xf1\x9a\x8a\x11\u1ff1m\xea\xb3\vZ\x13\xb7P%6\x98\xa5\x8b\xef}_\x90O\xd0_\xaeX\x90\xbfVPEh\xb0\b\x17\xc3\xdd\x1a\xaa\xfa)_\xb7\r\x1er\xac\xe8@m\x8c4Y\x8a\x1b%7\nt_X\x17\xe4o\x94\x19&6\x1f\xa5\xba\xe1Ն\x89\xcf\xc3[~\x8e5\xbe\xa1\xca0+\xecn<\xb1\x812A9\xfb{̮\xb5_\x8e\x03\xba\x1e\x9c`-H\xc20\x86^\xbc\a\x1b\xe3\x0e\xe6\x05\xa2&\xb4\xf4t=%^\t<\x19\xb3\xa9u,\xd1\xc4\"\xa1\xdb\v\xf2IF\r\x83/\x87b]\x986$\x03m\x16\xb0^Ke\xdcj\xf5bA\xd8:$\x1f\xac\xcd\xc1\xbc\x99\xbb\xab\x92\xb0\xd82s]hҸ/Lz+\xf4\xc2x\x94}A\xf7ne\x8afYe#\xacKm(\x8f\x048\xcf2\xfc\x98\xe5\xb1\xca\a\xf9/\xcfZ\xc9[\xb6\x01\xf5\x93\x8e؏#)\x1e\xa6\xe1\xa2>nQ\x04A\x1e\x153\xc6\xc6T\xf2H)\x81'\x95\xb1\xb1\x15\xe7D[R\x9f\x94}$Ό.\x87Kr\xd2P\xbe\xab\xa1\f\x99g\x8f5\xde̸B\xda\x10\x1b\xf7b\xf5\x91oeٜm\xa9\xd8\f\x9eP\xb0U\xb2\xdal\x83$\x0f\x04\xd3$\xaf\x00\x93\xb5hRt\xb8X\xd8TJ\xb4J\t\x8el\xfb&A\x18p\xb84{ U9\xf7\x17\xf7\xfa{\x99/\xfd\x1d(\x8b\xb5\x92\xc5\xc2\xf7\x8b\xb9Թ_\xc9WL\xda\xc8\xc5l\xa3T'.j\xf7\xd7\f\xa0$\x94%\bB\xb5\xef9ᤨ\x93\xdd\xd4o\xd65\xdcH\xcd\x12\xa2\xfd(\xc7\xff\xda\x06\x10\x18^\x86\xbf\xbb\xcc\xf03\x18\xec3\x86\xc7g\xbf\x05\x1fvT\x187\x9d\xa8]\xe4\xcc9\xb1٤\x89\x8c\xb6\x8e\xedYI\x9a\xdb\x0e\x84\x91\xfc\fv\x17gѭ/\xd7p\a\x81]\xfb\xebWk\xc0s\xa2\x99\b\x17_\xbb\xd2\x0f'\xfdѕ@\x81\x17UJ\x15\xaf\xc6<\x9ep\xe9\"\xf4\xba\xb9\x96]\x1dI|8y*~\x7f\x00\xe3`S7\xdeKZ7\t\xd3\xe7?\xb0\xd8z\x00\x96\xf1f\x16\x95?\xfe\ue6f5wIS\xbd8E\x8e\xcd\xfcpR7<\x85\xeb\xdeCz\xc3\xc1j\x9b\x06\xe8N*'\xe9\xdc\xee\x8cٴs\xa6\xd2\xc2\x15\xef\xe7\xc9%\xedΘD{\xb1\f\xdayQ~\xa4xA\xf4IZ\xfb7\xffm$\x85\xe6\xc1\x9e;\x89\xd6ʡ\x85\x81\xbfj\x16-\xeas{?\xa2\x9d\xce[\xd6\xc2\xf7\xe4\x7f\xf9\xff\x00\x00\x00\xff\xff<\x82OF\xb8\x82\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xccZK\x93۸\x11\xbe\xebWt\xed\x1e\xf6\xb2\x94줒J\xe96\x96\x93*W\xc6\xf1\xd4hvr]\bhJ\xb0@\x80\x01@\xc9\xca㿧\x1a\x0f\x89\xe2C\x0f;q\u008b-\x12h\xf4\xf3\xeb\x0f\xc0\x14E1a\xb5|E\xeb\xa4\xd1s`\xb5\xc4/\x1e5\xfdr\xd3\xed\x1f\xdcT\x9a\xd9\xee\xedd+\xb5\x98âq\xdeT\xcf\xe8Lc9\xbe\xc7Rj\xe9\xa5ѓ\n=\x13̳\xf9\x04\x80im<\xa3\u05ce~\x02p\xa3\xbd5J\xa1-֨\xa7\xdbf\x85\xabF*\x816\b\xcfK\xef\xdeL\xdf\xfe~\xfa\xbb\t\x80f\x15\xcea\xc5\xf8\xb6\xa9\x9d7\x96\xadQ\x19\x1eENw\xa8К\xa94\x13W#\xa7\x15\xd6\xd64\xf5\x1cN\x1f\xa2\x84\xb4z\xd4\xfc]\x10\xb6\x8c\xc2\x1e\x93\xb0\xf0]I\xe7\xff<>\xe6Q:\x1f\xc6ժ\xb1L\x8d\xa9\x15\x86\xb8\x8d\xb1\xfe/\xa7\xa5\vX9\x15\xbfH\xbdn\x14\xb3#\xd3'\x00\x8e\x9b\x1a\xe7\x10f\u05cc\xa3\x98\x00$\xd7\x04i\x050!\x82\xb3\x99z\xb2R{\xb4\v\xa3\x9aJ\x1f\xd7\x12踕\xb5\x0fΌ\xb6@2\x06\xb25\xe0<\xf3\x8d\x03\xd7\xf0\r0\a\x0f;&\x15[)\x9c\xfd\xa2Y\xfe\x7f\x90\a\xf0\xd9\x19\xfd\xc4\xfcf\x0e\xd38kZo\x98\xcb_c\x8c\x9eZo\xfc\x81\fp\xdeJ\xbd\x1eR\xe9\x919\xffʔ\x14A\x93\x17Y!H\a~\x83\xa0\x98\xf3\xe0\xe9\x05\xfd\x8a\x1e\x02r\x11B\xf6\x10\xec\x99K\xeb\x00좔\xe0\xa3aMUo\xad3\xb5I\x15x\xedH\x89\xfaӛ\xa4}Kl\xce\xef)\xb7x\x14\xe9<\xab\xea3\xb9\x0fk\x1c\x13v\xe6\x8a\xf7X\xb2F\xf9\xb6\xa9\x14%\xd5\xce\xcbs\xb3j\xe4S\x11g\x9d\xad\xf8\xfe\xec]\\ue\x8cB\x16\xa5\xc4Q\xbb\xb71\v\xf9\x06+6O\x83M\x8d\xfa\xe1\xe9\xc3\xebo\x97g\xafa(\x91:EA\x81c\xad\xd8l\xd0\"\xbc\x86\xfa\x8bqsɴ\xa3L\x00\xb3\xfa\x8cܟ\x82X[S\xa3\xf52\x17K|ZX\xd4z\xdb\xd1\xe9\x9f\xc5\xd97\x002#\xce\x02A\xa0\x841\xafR\xfd\xa0H\x96\x83)\xc1o\xa4\x03\x8b\xb5E\x87:\xc2\x14\xbdf:)8\xed\x88^\xa2%1Tۍ\x12\x84e;\xb4\x1e,r\xb3\xd6\xf2\xefG\xd9\x0e\xbcI\xc9\xec\xd1y\b\x15\xaa\x99\xa2dm\xf0g`Zt$W\xec\x00\x16iMhtK^\x98\xe0\xbaz|\xa4j\x90\xba4s\xd8x_\xbb\xf9l\xb6\x96>#47U\xd5h\xe9\x0f\xb3\x00\xb6r\xd5xc\xddL\xe0\x0e\xd5\xcc\xc9u\xc1,\xdfH\x8f\xdc7\x16g\xac\x96E0DGH\xadď6a\xba;[\xb6W\xd2\xf1\t\x90zGx\b^c\xcaDQ\xd1\xc4S\x14\xe8\x15\xb9\xee\xf9\x8f\xcb\x17Ț\xc4HŠ\x9c\x86\xf6\xfc\x92\xe3Cޔ\xbaD\x1b\xe7\x95\xd6TA&jQ\x1b\xa9}\xf8\xc1\x95D\xed\xc15\xabJzJ\x83\xbf5\xe8<\x85\xae+v\x11\xba\x18\xac\x10\x9a:\x80Dw\xc0\a\r\vV\xa1Z0\x87\xdf9V\x14\x15WP\x10n\x8aV\xbb7w\aG\xf7\xb6>\xe4\x9e:\x12\xdaA4X\xd6\xc8\xcf\xeaN\xa0\x93\x96*\xc33\x8f\xa1\xba:\x0eJP1ޔ\xf33\f\x12\xf40\xceѹ\x8fF`\xf7KG\xe5\x87\xe3\xc03\x1dk\xb4\x95t\xa1\xbdBil\xb7\xf3\xb0#\x92\xb7\x9f\x8cx݀\x03\xa0n\xaa\xbe\"\x05<#\x13\x9f\xb4:\x8c|\xfa\xab\x95\xbe\xbf\xd0H \xe9\x89*.\x0f\x9a?\xa1\x95F\\1\xfe]g\xf8\xd1\x05\x1b\xb3\x872\xe4\xbf\xf6\xea@\xd8\xe5\x0e\x9a\xf7Q;?\x0fO\x1f2\x82\xc7\xdaJ\x85\x99|5\x85\x87TԦ\x847 \xa4#\"\xe1\x82о\xb3t\xa3\x02ј\x83\xb7\xcd]\xe6s\xa3K\xb9\xee\x1b\xdd\xe6Fc\x19sEt\xc7s\x8b\xb0\x12\xa1\x16eGm\xcdN\n\xb4\x05Շ,%O\x9a46v\x90R\xa2\x12=l\x1a\xad\xb2`\x8aEAE\xcdԕ\x18.\x8e\x03\x03\x93fR\xc7\f>\t\bXc\xabԚ\xb5G-\xb0\xdbm\x826&\x00\x9aC\x01{\xe97\x11)\xd5P\xdd\xc1\xc5ڣg\x8b\x87\xa1\xd7\x1d\xdd_6H#c\xe3Ep\xc8-\xfa\x90m\xa8(}(\x95\xa6\x00\x1f\x1b\x17\xb0\xb6\x8b\x13\xf9\t\x84/\xcf\xde\xe2\xa1\xefh\xb8\x16\xdcD\x85FT\x0e$j\x0e?\xfcpݤ^w\xcb\x0fQ\xf7l\xa8\xc5\x12-\xea\x1e\x9b\xc8\xcfK\xe8Q\x944\x94aX\x96Ƚܡ:\x84\x9eD\xe0\xf93\xac\x1a\x0f\xa2\xc1\x105Ʒ{f\x85\x03n\xaa\x9ay\xb9\x92J\xfa\x03H7\"\x9f)e\xf6(Rı\xaa\xfda\n\x1f\xb4\xf3LstG\x1eD\x1e\x8b\xa9\xc0t\x1c\x95\xaa8\x10:f\x8700\x8a\xaf\x8c\xf3\xc0\xd1R:\xaa\x03\xec\xad\xd1\xeb1c\a\xda!\xed\x01\xadF\x8f\xa1#\n\xc3\x1d5C\x8e\xb5w3\xb3C\xbb\x93\xb8\x9f\xed\x8d\xddJ\xbd.H\xc1\"\x81\xcf,\xec\xecf?\x86\x7f\xbe&\vL\x1dq\xe2\x86\xe4]\x86Z?\x10\xbd\xf5\x1b\x8c-b\x19s\xd0X \x02A\xa9]\xa5܍\xc8:TvC\xbc\xbc\xfd\xe4\x90\x0f\xf5\x8f-\xf6[\xc7\x05P\x01\xf8R\x9c|[T\xac.\xe2h\xe6M%\xf9\xa4km\xcc\xfb\xcb\xf8\x937+R\vɉܞ\xe3F\xdeĉ\xb3=̀\x1b\xba\xbb\x9c1\xb4\x1cvS47q\x85+\x1a\x7fj\x8f=m}#t\xa7\xfe\xef\xd0\x13\xeft\xa0\x91\xf8\x01\xb3}?\a\xc0\xe4FkB*o\x80\x1d\xdb\xc0O\xae\xdb\xff\xeeD\xcfU÷8\xe0\xf8\x9e)\xef\xc2\xc0\xec\xe38\x8dti\x1c\x86\xc6tM\r\xb8^\x11\x9c-\xd0ޢ\xcb(\xf2-\x1eH\u0091[0X<\xc0\xaa\xd1BaVu\xbfAM\xdb1Y\x1e\x88\xec\xbf<.\xb3c\x03\x01K[\xa7\xec\xde1 yO\xbb\x00JA1\x87_\x1c\xa6u\x9f\xb1\x04\xa9\x9dG\xd6#\xe9\xf1\x89\xbdq\x0e\xab\xc3\x00\u05fa\xd9A\xcfX~\xbb\x8f\x82\xae\xe4\xa1\xd4 8\xc6\xc4J\xb0\x92\xfa{\xde\x0f-\x1e\x02\xc4\x12\xdf \"}\xe6ґe\xeetth\xd0i\xf1\fdR\xc7\x02a\xd5\xd8\":\x1fR\x01\x8by9H䇃q\xb9.\xe0\x12\xb3\xe89\xfb>v1*\x13\x80\xdd\xc80\xe0z\xb2\xc0E\xa6\x017\xb0\x8d\x9e\x99\xa39\x05w\xb2\x0e\xf8\x0e\xcc\x03\xfe\xfb\xec\x03\xeef \xf0\xddY\bܖ)\x97\xd9\b|\x13#\xb9\xe0\x8bK\\\x05\xae\xf2\x15\xb8\xc8Y`\x94\xb7\xc05\xee\x02w\xf2\x17\bx\x82\xa5\xfcr\x032?\x85\x81\xb9\x93\xd6\xcco\xa8kH\x81\xc0\x06\xfaj<\xa1\x18q\xd0q\xd3\xfb)\x85\xef+\xfa\xee%\xd2\x17չ\x87\xf7e@\xbfB\x8c\x9eҰ\xa3\x17\xf2\xef\x04 \xe7\a c\x04mТ\xdd\xf1\xb4\xfdO\xf1X\x81\x0f\xa0\xf8\x992\xaf\xfd\x19\x17\x8e'\xf2\x99\xff\x10K\xa3Ͱ\xb1\x16]m\xb4\xa0\xb6w\xdb\xe1\xc4I\xe5\xff\xdc\x11\xc5pX\x8bs\xfa\xda\xf9\x96\xa3p\xd3\xf9\\\xb8߸\xfb\x84.\xde\xfa\xb4Ͽ\xccʡݵ\x0e\xe9:6~\x97\xb3\xb9\xc1\xce\xd6:\xb0#\xaa\xa4\xa1\xd1\xe1\xc8\"4\xad\xe9d`F\x9b\x17\xfa\xd0<\xa4\x03m\xf64\xb9%-v=\x13\xe9M8\xb4dZ\xa4\xe3b\xfa4 y/\x95\xa2\x1ef\xb12\xe4,\xd4^Zj\x96,\xb4\xb1\xddo\xa6o\xfewg\x81\x8a9\xbf\xc6Qq\xff\x99\xa6\x00[\x99\xc6\x0f\xf4\xfeV\xc2\x0f\xd6t\xb8e\xbfG\xc7\xf0\xb7\x03\xd7\xe8\t\x8d\xc9\x11፵\xe1\xb2._\"ݱ\xd1\x1cC\xe0\x87Ο8\xb4\xbf\xf5\xff\x00\xe2\x06\xbb\x06\xbbt\xefe촭\xb8&'\xb7\xdf4\xab\xe3\x15\xec\x1c\xfe\xf1\xafɿ\x03\x00\x00\xff\xff%\xff\\)\x99#\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcVMo\x1b7\x10\xbd\xebW\f\xd0kwU\xa3hQ\xec\xadqr0\xda\x06\x82\x1d\xe4N\x91#-c.\xc9\xce\f\xe5\xba\x1f\xff\xbd \xb9+K\xab\x95\x93\\\xb27\x91Ù\xc7\xf7f\x1e\xd54\xcdJE\xfb\x11\x89m\xf0\x1d\xa8h\xf1/A\x9f\x7fq\xfb\xf8\v\xb76\xac\x0f7\xabG\xebM\a\xb7\x89%\f\xf7\xc8!\x91Ʒ\xb8\xb3ފ\r~5\xa0(\xa3Du+\x00\xe5}\x10\x95\x979\xff\x04\xd0\xc1\v\x05琚=\xfa\xf61mq\x9b\xac3H%\xf9T\xfa\xf0C{\xf3s\xfb\xd3\n\xc0\xab\x01;0\xe8Pp\xab\xf4c\x8a\x84\x7f&d\xe1\xf6\x80\x0e)\xb46\xac8\xa2\xce\xf9\xf7\x14R\xec\xe0e\xa3\x9e\x1fkW\xdcoK\xaa7%\xd5}MUv\x9de\xf9\xedZ\xc4\xefv\x8c\x8a.\x91rˀJ\x00[\xbfON\xd1b\xc8\n\x80u\x88\xd8\xc1\xfb\f+*\x8df\x050^\xbb\xc0l@\x19S\x88TnC\xd6\v\xd2mpi\x98\bl\xc0 k\xb2Q\nQ\x1fz,W\x84\xb0\x03\xe9\x11j9\x90\x00[\x1c\x11\x98r\x0e\xe0\x13\a\xbfQ\xd2w\xd0f\xbe\xda\x1a\x9a\x81\x8c\x01\x95\xea7\xf3ey\u0380Y\xc8\xfa\xfd5\b,J\x12O J]\x1b<\xd0\t\xbf\xe7\x00J|\x1b{\xc5\xe7\xd5\x1f\xcaƵ\xca5\xe6pS\x99\xd6=\x0e\xaa\x1bcCD\xff\xeb\xe6\xee\xe3\x8f\x0fg\xcbp\x8euAZ\xb0\fjB\x9a\x89\xab\xacA\xf0\b\x81`\b4\xb1\xca\xed1i\xa4\x10\x91\xc4N\xadU\xbf\x93\xe19Y\x9dA\xf8\xb79\xdb\x03Ȩ\xeb)0y\x8a\x90\v\x89cS\xa0\x19/Zɵ\f\x84\x91\x90\xd1\u05f9\xca\xcb\xcaC\xd8~B-\xed,\xf5\x03RN\x03܇\xe4L\x1e\xbe\x03\x92\x00\xa1\x0e{o\xff>\xe6\xe6|\xef\\\xd4))\x94\xe4\xb6\xf3\xca\xc1A\xb9\x84߃\xf2f\x96yP\xcf@\x98kB\xf2'\xf9\xca\x01\x9e\xe3\xf8#\x93h\xfd.tЋD\xee\xd6뽕\xc9Rt\x18\x86\xe4\xad<\xaf\x8b;\xd8m\x92@\xbc6x@\xb7f\xbbo\x14\xe9\xde\njI\x84k\x15mS.⋭\xb4\x83\xf9\x8eF\x13Ⳳ\x17\xddS\xbf\xe2\x02_!O\xf6\x84\xda#5U\xbd\xe2\x8b\ny)Sw\xff\xee\xe1\x03LH\xaaRU\x94\x97\xd0\v^&}2\x9b\xd6\xef\x90\xea\xb9\x1d\x85\xa1\xe4Dob\xb0^\xca\x0f\xed,z\x01N\xdb\xc1\nO\x1d\x9b\xa5\x9b\xa7\xbd-\xb6\x9b\x1d E\xa3\x04\xcd<\xe0\xceí\x1a\xd0\xdd*\xc6o\xacUV\x85\x9b,\xc2\x17\xa9u\xfa\x98̃+\xbd'\x1b\xd33pEڅ\xe1\x7f\x88\xa8\xb3\xb8\x99\xdf|\xda\ueb2ec\xb5\v\x04O\xbd\xd5\xfd4\xfc3\x9a\x8eFq\xce߲1\xe4\xef\xc5n\xe7;W/\x0fEdK8k\xd8\x06.\xbc\xfbu^\x8a\xa9~%3\xd5\xd1Gnt\"*\xcdw\xf4y\xb5t\xe8K\xb9@\xa2@\x17\xab3P\xefJP\xf9Ǡ\xacgP\xfey<\b\xd2+\x81'\xa4\r\x97\x95\x1ax\x8fO\v\xabw~CaO\xc8\xf3\x96ϛ\x9b\xca\x1e\xce߃WXZlʋE\xceVhNXd\t\xa4\xf6\xa7\xbcr\xda\x1e\x9d\xbe\x83\x7f\xfe[\xfd\x1f\x00\x00\xff\xff\xbeM\x1a\xea\xb1\n\x00\x00"), diff --git a/design/backup-repo-cache-volume.md b/design/Implemented/backup-repo-cache-volume.md similarity index 100% rename from design/backup-repo-cache-volume.md rename to design/Implemented/backup-repo-cache-volume.md diff --git a/design/bsl-certificate-support_design.md b/design/Implemented/bsl-certificate-support_design.md similarity index 100% rename from design/bsl-certificate-support_design.md rename to design/Implemented/bsl-certificate-support_design.md diff --git a/design/concurrent-backup-processing.md b/design/Implemented/concurrent-backup-processing.md similarity index 100% rename from design/concurrent-backup-processing.md rename to design/Implemented/concurrent-backup-processing.md diff --git a/design/wildcard-namespace-support-design.md b/design/Implemented/wildcard-namespace-support-design.md similarity index 100% rename from design/wildcard-namespace-support-design.md rename to design/Implemented/wildcard-namespace-support-design.md diff --git a/design/custom-volume-policy-action.md b/design/custom-volume-policy-action.md new file mode 100644 index 000000000..56bbf303c --- /dev/null +++ b/design/custom-volume-policy-action.md @@ -0,0 +1,93 @@ +# Add custom volume policy action + +## Abstract + +Currently, velero supports 3 different volume policy actions: +snapshot, fs-backup, and skip, which tell Velero how to handle backing +up PVC contents. Any other policy action is not allowed. This prevents +third party BackupItemAction (BIA) plugins which might want to perform +different actions on PVC via defined volume policies. + +## Background + +An external BIA plugin that wants to back up volumes via some custom +means (i.e. not CSI snapshots or fs-backup with kopia) is not able to +make use of the existing volume policy API. While the plugin could use +something like PVC annotations instead, this won't integrate with +existing volume policies, which is desirable in case the user wants to +specify some PVCs to use the custom plugin while leaving others using +CSI snapshots or fs-backup. + +## Goals + +- Add a fourth valid volume policy action "custom" +- Make use of the existing action parameters field to distinguish between multiple custom actions. + +## Non Goals + +- Implementing custom action logic in velero repo + +## High-Level Design + +A new VolumeActionType with the value "custom" will be added to +`internal/resourcepolicies`. When the action is "custom", velero will +not perform a snapshot or use fs-backup on the PVC. If there is no +registered plugin which implements the desired custom action, then it +will be equivalent to the "skip" action. Since there could be +different plugins that implement custom actions, when making the API +call (defined below) the plugin should also pass in a partial action +parameters map containing the portion of the map that identifies the +custom plugin as belonging to a particular external +implementation. For example, there might be a custom BIA that's +looking for a `custom` volume policy action with the parameter +`myCustomAction=true`. The volume policy action would be defined like +this: + +```yaml + action: + type: custom + parameters: + myCustomAction: true +``` + +In `internal/volumehelper/volume_policy_helper.go` a new interface +method will be added, similar to `ShouldPerformSnapshot` but it takes +a partial parameter map as an additional input, since for custom +actions to match, the action type must be `custom`, but also there may +be some parameter that needs to match (to distinguish between +different custom actions). We also want a way for the plugin to get +the parameter map for the action. This should probably just return the +map rather than the Action struct is under `internal`. + +```go +type VolumeHelper interface { + ShouldPerformSnapshot(obj runtime.Unstructured, groupResource schema.GroupResource) (bool, error) + ShouldPerformFSBackup(volume corev1api.Volume, pod corev1api.Pod) (bool, error) + ShouldPerformCustomAction(obj runtime.Unstructured, groupResource schema.GroupResource, map[string]any) (bool, error) + GetActionParameters(obj runtime.Unstructured, groupResource schema.GroupResource) (map[string]any, error) +} +``` + +In addition, since the VolumeHelper interface is expected to be called by external plugins, the interface (but not the implementation) should be moved from `internal/volumehelper` to `pkg/util/volumehelper`. + +In `pkg/plugin/utils/volumehelper/volume_policy_helper.go`, a new helper func will be added which delegates to the internal volumehelper.NewVolumeHelperImplWithNamespaces + +```go +func NewVolumeHelper( + volumePolicy *resourcepolicies.Policies, + snapshotVolumes *bool, + logger logrus.FieldLogger, + client crclient.Client, + defaultVolumesToFSBackup bool, + backupExcludePVC bool, + namespaces []string, +) (VolumeHelper, error) { +``` + + +## Alternative Considered + +An alternate approach was to create a new server arg to allow +user-defined parameters. That was rejected in favor of this approach, +as the explicitly-supported "custom" option integrates more easily +into a supportable plugin-callable API. diff --git a/go.mod b/go.mod index 65177f63d..e80bfbcf9 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/hashicorp/go-plugin v1.6.0 github.com/joho/godotenv v1.3.0 github.com/kopia/kopia v0.16.0 - github.com/kubernetes-csi/external-snapshotter/client/v8 v8.2.0 + github.com/kubernetes-csi/external-snapshotter/client/v8 v8.4.0 github.com/onsi/ginkgo/v2 v2.22.0 github.com/onsi/gomega v1.36.1 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 @@ -42,10 +42,11 @@ require ( github.com/vmware-tanzu/crash-diagnostics v0.3.7 go.uber.org/zap v1.27.1 golang.org/x/mod v0.30.0 - golang.org/x/oauth2 v0.33.0 - golang.org/x/text v0.31.0 + golang.org/x/oauth2 v0.34.0 + golang.org/x/sys v0.40.0 + golang.org/x/text v0.32.0 google.golang.org/api v0.256.0 - google.golang.org/grpc v1.77.0 + google.golang.org/grpc v1.79.3 google.golang.org/protobuf v1.36.10 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.33.3 @@ -63,7 +64,7 @@ require ( ) require ( - cel.dev/expr v0.24.0 // indirect + cel.dev/expr v0.25.1 // indirect cloud.google.com/go v0.121.6 // indirect cloud.google.com/go/auth v0.17.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect @@ -93,13 +94,13 @@ require ( github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chmduquesne/rollinghash v4.0.0+incompatible // indirect - github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect + github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/edsrzf/mmap-go v1.2.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect - github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect @@ -168,29 +169,28 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/zeebo/blake3 v0.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect - go.opentelemetry.io/otel v1.38.0 // indirect - go.opentelemetry.io/otel/metric v1.38.0 // indirect - go.opentelemetry.io/otel/sdk v1.38.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect - go.opentelemetry.io/otel/trace v1.38.0 // indirect + go.opentelemetry.io/otel v1.40.0 // indirect + go.opentelemetry.io/otel/metric v1.40.0 // indirect + go.opentelemetry.io/otel/sdk v1.40.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect + go.opentelemetry.io/otel/trace v1.40.0 // indirect go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect - golang.org/x/crypto v0.45.0 // indirect + golang.org/x/crypto v0.46.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/sync v0.18.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/term v0.37.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/term v0.38.0 // indirect golang.org/x/time v0.14.0 // indirect - golang.org/x/tools v0.38.0 // indirect + golang.org/x/tools v0.39.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto v0.0.0-20250603155806-513f23925822 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect diff --git a/go.sum b/go.sum index 4ff54e055..d8ee0cab0 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ al.essio.dev/pkg/shellescape v1.5.1 h1:86HrALUujYS/h+GtqoB26SBEdkWfmMI6FubjXlsXyho= al.essio.dev/pkg/shellescape v1.5.1/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890= -cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= -cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -189,8 +189,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0= -github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4= +github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/Tv1FZd4SCg8axKApyNyRsAt/w= +github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -227,15 +227,15 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM= -github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs= -github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo= -github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= +github.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3reDNEnfOboJ0QySbH736g= +github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= -github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= +github.com/envoyproxy/protoc-gen-validate v1.3.0 h1:TvGH1wof4H33rezVKWSpqKz5NXWg5VPuZ0uONDT6eb4= +github.com/envoyproxy/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -507,8 +507,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kubernetes-csi/external-snapshotter/client/v8 v8.2.0 h1:Q3jQ1NkFqv5o+F8dMmHd8SfEmlcwNeo1immFApntEwE= -github.com/kubernetes-csi/external-snapshotter/client/v8 v8.2.0/go.mod h1:E3vdYxHj2C2q6qo8/Da4g7P+IcwqRZyy3gJBzYybV9Y= +github.com/kubernetes-csi/external-snapshotter/client/v8 v8.4.0 h1:bMqrb3UHgHbP+PW9VwiejfDJU1R0PpXVZNMdeH8WYKI= +github.com/kubernetes-csi/external-snapshotter/client/v8 v8.4.0/go.mod h1:E3vdYxHj2C2q6qo8/Da4g7P+IcwqRZyy3gJBzYybV9Y= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= @@ -742,24 +742,24 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY6LiIY9I8cUfm+pJs= -go.opentelemetry.io/contrib/detectors/gcp v1.38.0/go.mod h1:SU+iU7nu5ud4oCb3LQOhIZ3nRLj6FNVrKgtflbaf2ts= +go.opentelemetry.io/contrib/detectors/gcp v1.39.0 h1:kWRNZMsfBHZ+uHjiH4y7Etn2FK26LAGkNFw7RHv1DhE= +go.opentelemetry.io/contrib/detectors/gcp v1.39.0/go.mod h1:t/OGqzHBa5v6RHZwrDBJ2OirWc+4q/w2fTbLZwAKjTk= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= -go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= +go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= -go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= -go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= -go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= -go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= -go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= -go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= -go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= -go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= +go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= +go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= +go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= +go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= +go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= +go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= +go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.starlark.net v0.0.0-20201006213952-227f4aabceb5/go.mod h1:f0znQkUKRrkk36XxWbGjMqQM8wGv/xHBVE2qc3B5oFU= go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY= @@ -790,8 +790,8 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -876,8 +876,8 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -891,8 +891,8 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo= -golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -904,8 +904,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -969,14 +969,14 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= -golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -986,8 +986,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1047,8 +1047,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= -golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1134,10 +1134,10 @@ google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaE google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4= google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s= -google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 h1:mepRgnBZa07I4TRuomDE4sTIYieg/osKmzIf4USdWS4= -google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101 h1:tRPGkdGHuewF4UisLzzHHr1spKw92qLM98nIzxbC0wY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls= +google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1159,8 +1159,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= -google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= +google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= +google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/hack/build-image/Dockerfile b/hack/build-image/Dockerfile index 89477d2fb..25a162a82 100644 --- a/hack/build-image/Dockerfile +++ b/hack/build-image/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM --platform=$TARGETPLATFORM golang:1.25-bookworm +FROM --platform=$TARGETPLATFORM golang:1.25-trixie ARG GOPROXY @@ -21,9 +21,11 @@ ENV GO111MODULE=on ENV GOPROXY=${GOPROXY} # kubebuilder test bundle is separated from kubebuilder. Need to setup it for CI test. -RUN curl -sSLo envtest-bins.tar.gz https://go.kubebuilder.io/test-tools/1.22.1/linux/$(go env GOARCH) && \ - mkdir /usr/local/kubebuilder && \ - tar -C /usr/local/kubebuilder --strip-components=1 -zvxf envtest-bins.tar.gz +# Using setup-envtest to download envtest binaries +RUN go install sigs.k8s.io/controller-runtime/tools/setup-envtest@v0.0.0-20260305094418-8122a6266696 && \ + mkdir -p /usr/local/kubebuilder/bin && \ + ENVTEST_ASSETS_DIR=$(setup-envtest use 1.33.0 --bin-dir /usr/local/kubebuilder/bin -p path) && \ + cp -r ${ENVTEST_ASSETS_DIR}/* /usr/local/kubebuilder/bin/ RUN wget --quiet https://github.com/kubernetes-sigs/kubebuilder/releases/download/v3.2.0/kubebuilder_linux_$(go env GOARCH) && \ mv kubebuilder_linux_$(go env GOARCH) /usr/local/kubebuilder/bin/kubebuilder && \ diff --git a/internal/delete/actions/csi/volumesnapshotcontent_action.go b/internal/delete/actions/csi/volumesnapshotcontent_action.go index d12c7c43a..98e0fc03b 100644 --- a/internal/delete/actions/csi/volumesnapshotcontent_action.go +++ b/internal/delete/actions/csi/volumesnapshotcontent_action.go @@ -18,7 +18,6 @@ package csi import ( "context" - "time" "github.com/google/uuid" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" @@ -27,14 +26,11 @@ import ( corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/wait" crclient "sigs.k8s.io/controller-runtime/pkg/client" - velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" "github.com/vmware-tanzu/velero/pkg/plugin/velero" - "github.com/vmware-tanzu/velero/pkg/util/boolptr" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" ) @@ -71,7 +67,7 @@ func (p *volumeSnapshotContentDeleteItemAction) Execute( // So skip deleting VolumeSnapshotContent not have the backup name // in its labels. if !kubeutil.HasBackupLabel(&snapCont.ObjectMeta, input.Backup.Name) { - p.log.Info( + p.log.Infof( "VolumeSnapshotContent %s was not taken by backup %s, skipping deletion", snapCont.Name, input.Backup.Name, @@ -81,6 +77,17 @@ func (p *volumeSnapshotContentDeleteItemAction) Execute( p.log.Infof("Deleting VolumeSnapshotContent %s", snapCont.Name) + // Try to delete the original VSC from the cluster first. + // This handles legacy (pre-1.15) backups where the original VSC + // with DeletionPolicy=Retain still exists in the cluster. + originalVSCName := snapCont.Name + if cleaned := p.tryDeleteOriginalVSC(context.TODO(), originalVSCName); cleaned { + p.log.Infof("Successfully deleted original VolumeSnapshotContent %s from cluster, skipping temp VSC creation", originalVSCName) + return nil + } + + // create a temp VSC to trigger cloud snapshot deletion + // (for backups where the original VSC no longer exists in cluster) uuid, err := uuid.NewRandom() if err != nil { p.log.WithError(err).Errorf("Fail to generate the UUID to create VSC %s", snapCont.Name) @@ -114,60 +121,62 @@ func (p *volumeSnapshotContentDeleteItemAction) Execute( if err := p.crClient.Create(context.TODO(), &snapCont); err != nil { return errors.Wrapf(err, "fail to create VolumeSnapshotContent %s", snapCont.Name) } + p.log.Infof("Created temp VolumeSnapshotContent %s with DeletionPolicy=Delete to trigger cloud snapshot cleanup", snapCont.Name) - // Read resource timeout from backup annotation, if not set, use default value. - timeout, err := time.ParseDuration( - input.Backup.Annotations[velerov1api.ResourceTimeoutAnnotation]) - if err != nil { - p.log.Warnf("fail to parse resource timeout annotation %s: %s", - input.Backup.Annotations[velerov1api.ResourceTimeoutAnnotation], err.Error()) - timeout = 10 * time.Minute - } - p.log.Debugf("resource timeout is set to %s", timeout.String()) - - interval := 5 * time.Second - - // Wait until VSC created and ReadyToUse is true. - if err := wait.PollUntilContextTimeout( - context.Background(), - interval, - timeout, - true, - func(ctx context.Context) (bool, error) { - return checkVSCReadiness(ctx, &snapCont, p.crClient) - }, - ); err != nil { - return errors.Wrapf(err, "fail to wait VolumeSnapshotContent %s becomes ready.", snapCont.Name) - } - + // Delete the temp VSC immediately to trigger cloud snapshot removal. + // The CSI driver will handle the actual cloud snapshot deletion. if err := p.crClient.Delete( context.TODO(), &snapCont, ); err != nil && !apierrors.IsNotFound(err) { - p.log.Infof("VolumeSnapshotContent %s not found", snapCont.Name) + p.log.WithError(err).Errorf("Failed to delete temp VolumeSnapshotContent %s", snapCont.Name) return err } + p.log.Infof("Successfully triggered deletion of VolumeSnapshotContent %s and its cloud snapshot", snapCont.Name) return nil } -var checkVSCReadiness = func( +// tryDeleteOriginalVSC attempts to find and delete the original VSC from +// the cluster (legacy pre-1.15 backups). It patches the DeletionPolicy to +// Delete so the CSI driver also removes the cloud snapshot, then deletes +// the VSC object itself. +// Returns true if the original VSC was found and deletion was initiated. +func (p *volumeSnapshotContentDeleteItemAction) tryDeleteOriginalVSC( ctx context.Context, - vsc *snapshotv1api.VolumeSnapshotContent, - client crclient.Client, -) (bool, error) { - tmpVSC := new(snapshotv1api.VolumeSnapshotContent) - if err := client.Get(ctx, crclient.ObjectKeyFromObject(vsc), tmpVSC); err != nil { - return false, errors.Wrapf( - err, "failed to get VolumeSnapshotContent %s", vsc.Name, - ) + vscName string, +) bool { + existing := new(snapshotv1api.VolumeSnapshotContent) + if err := p.crClient.Get(ctx, crclient.ObjectKey{Name: vscName}, existing); err != nil { + if apierrors.IsNotFound(err) { + p.log.Debugf("Original VolumeSnapshotContent %s not found in cluster, will use temp VSC flow", vscName) + } else { + p.log.WithError(err).Warnf("Error looking up original VolumeSnapshotContent %s, will use temp VSC flow", vscName) + } + return false } - if tmpVSC.Status != nil && boolptr.IsSetToTrue(tmpVSC.Status.ReadyToUse) { - return true, nil + p.log.Debugf("Found original VolumeSnapshotContent %s in cluster (legacy backup), cleaning up directly", vscName) + + // Patch DeletionPolicy to Delete so the CSI driver removes the cloud snapshot + if existing.Spec.DeletionPolicy != snapshotv1api.VolumeSnapshotContentDelete { + original := existing.DeepCopy() + existing.Spec.DeletionPolicy = snapshotv1api.VolumeSnapshotContentDelete + if err := p.crClient.Patch(ctx, existing, crclient.MergeFrom(original)); err != nil { + p.log.WithError(err).Warnf("Failed to patch DeletionPolicy on original VSC %s, will use temp VSC flow", vscName) + return false + } + p.log.Debugf("Patched DeletionPolicy to Delete on original VolumeSnapshotContent %s", vscName) } - return false, nil + // Delete the original VSC — the CSI driver will clean up the cloud snapshot + if err := p.crClient.Delete(ctx, existing); err != nil && !apierrors.IsNotFound(err) { + p.log.WithError(err).Warnf("Failed to delete original VolumeSnapshotContent %s, will use temp VSC flow", vscName) + return false + } + + p.log.Infof("Deleted original VolumeSnapshotContent %s with DeletionPolicy=Delete, CSI driver will remove cloud snapshot", vscName) + return true } func NewVolumeSnapshotContentDeleteItemAction( diff --git a/internal/delete/actions/csi/volumesnapshotcontent_action_test.go b/internal/delete/actions/csi/volumesnapshotcontent_action_test.go index 24baccdb2..114bd752f 100644 --- a/internal/delete/actions/csi/volumesnapshotcontent_action_test.go +++ b/internal/delete/actions/csi/volumesnapshotcontent_action_test.go @@ -22,9 +22,10 @@ import ( "testing" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" + corev1api "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -37,19 +38,44 @@ import ( velerotest "github.com/vmware-tanzu/velero/pkg/test" ) +// fakeClientWithErrors wraps a real client and injects errors for specific operations. +type fakeClientWithErrors struct { + crclient.Client + getError error + patchError error + deleteError error +} + +func (c *fakeClientWithErrors) Get(ctx context.Context, key crclient.ObjectKey, obj crclient.Object, opts ...crclient.GetOption) error { + if c.getError != nil { + return c.getError + } + return c.Client.Get(ctx, key, obj, opts...) +} + +func (c *fakeClientWithErrors) Patch(ctx context.Context, obj crclient.Object, patch crclient.Patch, opts ...crclient.PatchOption) error { + if c.patchError != nil { + return c.patchError + } + return c.Client.Patch(ctx, obj, patch, opts...) +} + +func (c *fakeClientWithErrors) Delete(ctx context.Context, obj crclient.Object, opts ...crclient.DeleteOption) error { + if c.deleteError != nil { + return c.deleteError + } + return c.Client.Delete(ctx, obj, opts...) +} + func TestVSCExecute(t *testing.T) { snapshotHandleStr := "test" tests := []struct { - name string - item runtime.Unstructured - vsc *snapshotv1api.VolumeSnapshotContent - backup *velerov1api.Backup - function func( - ctx context.Context, - vsc *snapshotv1api.VolumeSnapshotContent, - client crclient.Client, - ) (bool, error) - expectErr bool + name string + item runtime.Unstructured + vsc *snapshotv1api.VolumeSnapshotContent + backup *velerov1api.Backup + preExistingVSC *snapshotv1api.VolumeSnapshotContent + expectErr bool }{ { name: "VolumeSnapshotContent doesn't have backup label", @@ -71,27 +97,22 @@ func TestVSCExecute(t *testing.T) { { name: "Normal case, VolumeSnapshot should be deleted", vsc: builder.ForVolumeSnapshotContent("bar").ObjectMeta(builder.WithLabelsMap(map[string]string{velerov1api.BackupNameLabel: "backup"})).VolumeSnapshotClassName("volumesnapshotclass").Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: &snapshotHandleStr}).Result(), - backup: builder.ForBackup("velero", "backup").ObjectMeta(builder.WithAnnotationsMap(map[string]string{velerov1api.ResourceTimeoutAnnotation: "5s"})).Result(), + backup: builder.ForBackup("velero", "backup").Result(), expectErr: false, - function: func( - ctx context.Context, - vsc *snapshotv1api.VolumeSnapshotContent, - client crclient.Client, - ) (bool, error) { - return true, nil - }, }, { - name: "Error case, deletion fails", + name: "Original VSC exists in cluster, cleaned up directly", vsc: builder.ForVolumeSnapshotContent("bar").ObjectMeta(builder.WithLabelsMap(map[string]string{velerov1api.BackupNameLabel: "backup"})).Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: &snapshotHandleStr}).Result(), - backup: builder.ForBackup("velero", "backup").ObjectMeta(builder.WithAnnotationsMap(map[string]string{velerov1api.ResourceTimeoutAnnotation: "5s"})).Result(), - expectErr: true, - function: func( - ctx context.Context, - vsc *snapshotv1api.VolumeSnapshotContent, - client crclient.Client, - ) (bool, error) { - return false, errors.Errorf("test error case") + backup: builder.ForBackup("velero", "backup").Result(), + expectErr: false, + preExistingVSC: &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{Name: "bar"}, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, + Driver: "disk.csi.azure.com", + Source: snapshotv1api.VolumeSnapshotContentSource{SnapshotHandle: stringPtr("snap-123")}, + VolumeSnapshotRef: corev1api.ObjectReference{Name: "vs-1", Namespace: "default"}, + }, }, }, } @@ -100,7 +121,10 @@ func TestVSCExecute(t *testing.T) { t.Run(test.name, func(t *testing.T) { crClient := velerotest.NewFakeControllerRuntimeClient(t) logger := logrus.StandardLogger() - checkVSCReadiness = test.function + + if test.preExistingVSC != nil { + require.NoError(t, crClient.Create(t.Context(), test.preExistingVSC)) + } p := volumeSnapshotContentDeleteItemAction{log: logger, crClient: crClient} @@ -158,52 +182,153 @@ func TestNewVolumeSnapshotContentDeleteItemAction(t *testing.T) { require.NoError(t, err1) } -func TestCheckVSCReadiness(t *testing.T) { +func TestTryDeleteOriginalVSC(t *testing.T) { tests := []struct { name string - vsc *snapshotv1api.VolumeSnapshotContent - createVSC bool - expectErr bool - ready bool + vscName string + existing *snapshotv1api.VolumeSnapshotContent + createIt bool + expectRet bool }{ { - name: "VSC not exist", - vsc: &snapshotv1api.VolumeSnapshotContent{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vsc-1", - Namespace: "velero", - }, - }, - createVSC: false, - expectErr: true, - ready: false, + name: "VSC not found in cluster, returns false", + vscName: "not-found", + expectRet: false, }, { - name: "VSC not ready", - vsc: &snapshotv1api.VolumeSnapshotContent{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vsc-1", - Namespace: "velero", + name: "VSC found with Retain policy, patches and deletes", + vscName: "legacy-vsc", + existing: &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{Name: "legacy-vsc"}, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, + Driver: "disk.csi.azure.com", + Source: snapshotv1api.VolumeSnapshotContentSource{ + SnapshotHandle: stringPtr("snap-123"), + }, + VolumeSnapshotRef: corev1api.ObjectReference{ + Name: "vs-1", + Namespace: "default", + }, }, }, - createVSC: true, - expectErr: false, - ready: false, + createIt: true, + expectRet: true, + }, + { + name: "VSC found with Delete policy already, just deletes", + vscName: "already-delete-vsc", + existing: &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{Name: "already-delete-vsc"}, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + DeletionPolicy: snapshotv1api.VolumeSnapshotContentDelete, + Driver: "disk.csi.azure.com", + Source: snapshotv1api.VolumeSnapshotContentSource{ + SnapshotHandle: stringPtr("snap-456"), + }, + VolumeSnapshotRef: corev1api.ObjectReference{ + Name: "vs-2", + Namespace: "default", + }, + }, + }, + createIt: true, + expectRet: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { crClient := velerotest.NewFakeControllerRuntimeClient(t) - if test.createVSC { - require.NoError(t, crClient.Create(t.Context(), test.vsc)) + logger := logrus.StandardLogger() + p := &volumeSnapshotContentDeleteItemAction{ + log: logger, + crClient: crClient, } - ready, err := checkVSCReadiness(t.Context(), test.vsc, crClient) - require.Equal(t, test.ready, ready) - if test.expectErr { - require.Error(t, err) + if test.createIt && test.existing != nil { + require.NoError(t, crClient.Create(t.Context(), test.existing)) + } + + result := p.tryDeleteOriginalVSC(t.Context(), test.vscName) + require.Equal(t, test.expectRet, result) + + // If cleanup succeeded, verify the VSC is gone + if test.expectRet { + err := crClient.Get(t.Context(), crclient.ObjectKey{Name: test.vscName}, + &snapshotv1api.VolumeSnapshotContent{}) + require.True(t, apierrors.IsNotFound(err), + "VSC should have been deleted from cluster") } }) } + + // Error injection tests for tryDeleteOriginalVSC + t.Run("Get returns non-NotFound error, returns false", func(t *testing.T) { + errClient := &fakeClientWithErrors{ + Client: velerotest.NewFakeControllerRuntimeClient(t), + getError: fmt.Errorf("connection refused"), + } + p := &volumeSnapshotContentDeleteItemAction{ + log: logrus.StandardLogger(), + crClient: errClient, + } + require.False(t, p.tryDeleteOriginalVSC(t.Context(), "some-vsc")) + }) + + t.Run("Patch fails, returns false", func(t *testing.T) { + realClient := velerotest.NewFakeControllerRuntimeClient(t) + vsc := &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{Name: "patch-fail-vsc"}, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, + Driver: "disk.csi.azure.com", + Source: snapshotv1api.VolumeSnapshotContentSource{SnapshotHandle: stringPtr("snap-789")}, + VolumeSnapshotRef: corev1api.ObjectReference{Name: "vs-3", Namespace: "default"}, + }, + } + require.NoError(t, realClient.Create(t.Context(), vsc)) + + errClient := &fakeClientWithErrors{ + Client: realClient, + patchError: fmt.Errorf("patch forbidden"), + } + p := &volumeSnapshotContentDeleteItemAction{ + log: logrus.StandardLogger(), + crClient: errClient, + } + require.False(t, p.tryDeleteOriginalVSC(t.Context(), "patch-fail-vsc")) + }) + + t.Run("Delete fails, returns false", func(t *testing.T) { + realClient := velerotest.NewFakeControllerRuntimeClient(t) + vsc := &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{Name: "delete-fail-vsc"}, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + DeletionPolicy: snapshotv1api.VolumeSnapshotContentDelete, + Driver: "disk.csi.azure.com", + Source: snapshotv1api.VolumeSnapshotContentSource{SnapshotHandle: stringPtr("snap-999")}, + VolumeSnapshotRef: corev1api.ObjectReference{Name: "vs-4", Namespace: "default"}, + }, + } + require.NoError(t, realClient.Create(t.Context(), vsc)) + + errClient := &fakeClientWithErrors{ + Client: realClient, + deleteError: fmt.Errorf("delete forbidden"), + } + p := &volumeSnapshotContentDeleteItemAction{ + log: logrus.StandardLogger(), + crClient: errClient, + } + require.False(t, p.tryDeleteOriginalVSC(t.Context(), "delete-fail-vsc")) + }) +} + +func boolPtr(b bool) *bool { + return &b +} + +func stringPtr(s string) *string { + return &s } diff --git a/internal/resourcepolicies/resource_policies.go b/internal/resourcepolicies/resource_policies.go index d484cabce..6b5046e57 100644 --- a/internal/resourcepolicies/resource_policies.go +++ b/internal/resourcepolicies/resource_policies.go @@ -42,6 +42,8 @@ const ( FSBackup VolumeActionType = "fs-backup" // snapshot action can have 3 different meaning based on velero configuration and backup spec - cloud provider based snapshots, local csi snapshots and datamover snapshots Snapshot VolumeActionType = "snapshot" + // custom action is used to identify a volume that will be handled by an external plugin. Velero will not snapshot or use fs-backup if action=="custom" + Custom VolumeActionType = "custom" ) // Action defined as one action for a specific way of backup diff --git a/internal/resourcepolicies/volume_resources_validator.go b/internal/resourcepolicies/volume_resources_validator.go index b6031eec2..652c41d30 100644 --- a/internal/resourcepolicies/volume_resources_validator.go +++ b/internal/resourcepolicies/volume_resources_validator.go @@ -90,7 +90,7 @@ func decodeStruct(r io.Reader, s any) error { func (a *Action) validate() error { // validate Type valid := false - if a.Type == Skip || a.Type == Snapshot || a.Type == FSBackup { + if a.Type == Skip || a.Type == Snapshot || a.Type == FSBackup || a.Type == Custom { valid = true } if !valid { diff --git a/internal/volume/volumes_information.go b/internal/volume/volumes_information.go index 463b81f46..4d5961bdb 100644 --- a/internal/volume/volumes_information.go +++ b/internal/volume/volumes_information.go @@ -146,6 +146,10 @@ type CSISnapshotInfo struct { // The VolumeSnapshot's Status.ReadyToUse value ReadyToUse *bool + + // The VolumeGroupSnapshotHandle from VSC status, used to create stub VGSC during restore + // for CSI drivers that populate this field (e.g., Ceph RBD). + VolumeGroupSnapshotHandle string `json:"volumeGroupSnapshotHandle,omitempty"` } // SnapshotDataMovementInfo is used for displaying the snapshot data mover status. @@ -456,6 +460,10 @@ func (v *BackupVolumesInformation) generateVolumeInfoForCSIVolumeSnapshot() { if volumeSnapshotContent.Status.SnapshotHandle != nil { snapshotHandle = *volumeSnapshotContent.Status.SnapshotHandle } + volumeGroupSnapshotHandle := "" + if volumeSnapshotContent.Status != nil && volumeSnapshotContent.Status.VolumeGroupSnapshotHandle != nil { + volumeGroupSnapshotHandle = *volumeSnapshotContent.Status.VolumeGroupSnapshotHandle + } if pvcPVInfo := v.pvMap.retrieve("", *volumeSnapshot.Spec.Source.PersistentVolumeClaimName, volumeSnapshot.Namespace); pvcPVInfo != nil { volumeInfo := &BackupVolumeInfo{ BackupMethod: CSISnapshot, @@ -466,12 +474,13 @@ func (v *BackupVolumesInformation) generateVolumeInfoForCSIVolumeSnapshot() { SnapshotDataMoved: false, PreserveLocalSnapshot: true, CSISnapshotInfo: &CSISnapshotInfo{ - VSCName: *volumeSnapshot.Status.BoundVolumeSnapshotContentName, - Size: size, - Driver: volumeSnapshotContent.Spec.Driver, - SnapshotHandle: snapshotHandle, - OperationID: operation.Spec.OperationID, - ReadyToUse: volumeSnapshot.Status.ReadyToUse, + VSCName: *volumeSnapshot.Status.BoundVolumeSnapshotContentName, + Size: size, + Driver: volumeSnapshotContent.Spec.Driver, + SnapshotHandle: snapshotHandle, + OperationID: operation.Spec.OperationID, + ReadyToUse: volumeSnapshot.Status.ReadyToUse, + VolumeGroupSnapshotHandle: volumeGroupSnapshotHandle, }, PVInfo: &PVInfo{ ReclaimPolicy: string(pvcPVInfo.PV.Spec.PersistentVolumeReclaimPolicy), diff --git a/internal/volumehelper/volume_policy_helper.go b/internal/volumehelper/volume_policy_helper.go index 160a2005e..339b80011 100644 --- a/internal/volumehelper/volume_policy_helper.go +++ b/internal/volumehelper/volume_policy_helper.go @@ -18,13 +18,9 @@ import ( "github.com/vmware-tanzu/velero/pkg/util/boolptr" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" podvolumeutil "github.com/vmware-tanzu/velero/pkg/util/podvolume" + vhutil "github.com/vmware-tanzu/velero/pkg/util/volumehelper" ) -type VolumeHelper interface { - ShouldPerformSnapshot(obj runtime.Unstructured, groupResource schema.GroupResource) (bool, error) - ShouldPerformFSBackup(volume corev1api.Volume, pod corev1api.Pod) (bool, error) -} - type volumeHelperImpl struct { volumePolicy *resourcepolicies.Policies snapshotVolumes *bool @@ -53,7 +49,7 @@ func NewVolumeHelperImpl( client crclient.Client, defaultVolumesToFSBackup bool, backupExcludePVC bool, -) VolumeHelper { +) vhutil.VolumeHelper { // Pass nil namespaces - no cache will be built, so this never fails. // This is used by plugins that don't need the cache optimization. vh, _ := NewVolumeHelperImplWithNamespaces( @@ -81,7 +77,7 @@ func NewVolumeHelperImplWithNamespaces( defaultVolumesToFSBackup bool, backupExcludePVC bool, namespaces []string, -) (VolumeHelper, error) { +) (vhutil.VolumeHelper, error) { var pvcPodCache *podvolumeutil.PVCPodCache if len(namespaces) > 0 { pvcPodCache = podvolumeutil.NewPVCPodCache() @@ -110,7 +106,7 @@ func NewVolumeHelperImplWithCache( client crclient.Client, logger logrus.FieldLogger, pvcPodCache *podvolumeutil.PVCPodCache, -) (VolumeHelper, error) { +) (vhutil.VolumeHelper, error) { resourcePolicies, err := resourcepolicies.GetResourcePoliciesFromBackup(backup, client, logger) if err != nil { return nil, errors.Wrap(err, "failed to get volume policies from backup") @@ -134,6 +130,7 @@ func (v *volumeHelperImpl) ShouldPerformSnapshot(obj runtime.Unstructured, group pv := new(corev1api.PersistentVolume) var err error + var pvNotFoundErr error if groupResource == kuberesource.PersistentVolumeClaims { if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &pvc); err != nil { v.logger.WithError(err).Error("fail to convert unstructured into PVC") @@ -142,8 +139,10 @@ func (v *volumeHelperImpl) ShouldPerformSnapshot(obj runtime.Unstructured, group pv, err = kubeutil.GetPVForPVC(pvc, v.client) if err != nil { - v.logger.WithError(err).Errorf("fail to get PV for PVC %s", pvc.Namespace+"/"+pvc.Name) - return false, err + // Any error means PV not available - save to return later if no policy matches + v.logger.Debugf("PV not found for PVC %s: %v", pvc.Namespace+"/"+pvc.Name, err) + pvNotFoundErr = err + pv = nil } } @@ -158,7 +157,7 @@ func (v *volumeHelperImpl) ShouldPerformSnapshot(obj runtime.Unstructured, group vfd := resourcepolicies.NewVolumeFilterData(pv, nil, pvc) action, err := v.volumePolicy.GetMatchAction(vfd) if err != nil { - v.logger.WithError(err).Errorf("fail to get VolumePolicy match action for PV %s", pv.Name) + v.logger.WithError(err).Errorf("fail to get VolumePolicy match action for %+v", vfd) return false, err } @@ -167,15 +166,21 @@ func (v *volumeHelperImpl) ShouldPerformSnapshot(obj runtime.Unstructured, group // If there is no match action, go on to the next check. if action != nil { if action.Type == resourcepolicies.Snapshot { - v.logger.Infof(fmt.Sprintf("performing snapshot action for pv %s", pv.Name)) + v.logger.Infof("performing snapshot action for %+v", vfd) return true, nil } else { - v.logger.Infof("Skip snapshot action for pv %s as the action type is %s", pv.Name, action.Type) + v.logger.Infof("Skip snapshot action for %+v as the action type is %s", vfd, action.Type) return false, nil } } } + // If resource is PVC, and PV is nil (e.g., Pending/Lost PVC with no matching policy), return the original error + if groupResource == kuberesource.PersistentVolumeClaims && pv == nil && pvNotFoundErr != nil { + v.logger.WithError(pvNotFoundErr).Errorf("fail to get PV for PVC %s", pvc.Namespace+"/"+pvc.Name) + return false, pvNotFoundErr + } + // If this PV is claimed, see if we've already taken a (pod volume backup) // snapshot of the contents of this PV. If so, don't take a snapshot. if pv.Spec.ClaimRef != nil { @@ -209,7 +214,7 @@ func (v *volumeHelperImpl) ShouldPerformSnapshot(obj runtime.Unstructured, group return true, nil } - v.logger.Infof(fmt.Sprintf("skipping snapshot action for pv %s possibly due to no volume policy setting or snapshotVolumes is false", pv.Name)) + v.logger.Infof("skipping snapshot action for pv %s possibly due to no volume policy setting or snapshotVolumes is false", pv.Name) return false, nil } @@ -219,6 +224,7 @@ func (v volumeHelperImpl) ShouldPerformFSBackup(volume corev1api.Volume, pod cor return false, nil } + var pvNotFoundErr error if v.volumePolicy != nil { var resource any var err error @@ -230,10 +236,13 @@ func (v volumeHelperImpl) ShouldPerformFSBackup(volume corev1api.Volume, pod cor v.logger.WithError(err).Errorf("fail to get PVC for pod %s", pod.Namespace+"/"+pod.Name) return false, err } - resource, err = kubeutil.GetPVForPVC(pvc, v.client) + pvResource, err := kubeutil.GetPVForPVC(pvc, v.client) if err != nil { - v.logger.WithError(err).Errorf("fail to get PV for PVC %s", pvc.Namespace+"/"+pvc.Name) - return false, err + // Any error means PV not available - save to return later if no policy matches + v.logger.Debugf("PV not found for PVC %s: %v", pvc.Namespace+"/"+pvc.Name, err) + pvNotFoundErr = err + } else { + resource = pvResource } } @@ -260,6 +269,12 @@ func (v volumeHelperImpl) ShouldPerformFSBackup(volume corev1api.Volume, pod cor return false, nil } } + + // If no policy matched and PV was not found, return the original error + if pvNotFoundErr != nil { + v.logger.WithError(pvNotFoundErr).Errorf("fail to get PV for PVC %s", pvc.Namespace+"/"+pvc.Name) + return false, pvNotFoundErr + } } if v.shouldPerformFSBackupLegacy(volume, pod) { @@ -300,6 +315,121 @@ func (v volumeHelperImpl) shouldPerformFSBackupLegacy( } } +func (v *volumeHelperImpl) ShouldPerformCustomAction(obj runtime.Unstructured, groupResource schema.GroupResource, matchParams map[string]any) (bool, error) { + // check if volume policy exists and also check if the object(pv/pvc) fits a volume policy criteria and see if the associated action is custom with the provided param values + pvc := new(corev1api.PersistentVolumeClaim) + pv := new(corev1api.PersistentVolume) + var err error + + var pvNotFoundErr error + if groupResource == kuberesource.PersistentVolumeClaims { + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &pvc); err != nil { + v.logger.WithError(err).Error("fail to convert unstructured into PVC") + return false, err + } + + pv, err = kubeutil.GetPVForPVC(pvc, v.client) + if err != nil { + // Any error means PV not available - save to return later if no policy matches + v.logger.Debugf("PV not found for PVC %s: %v", pvc.Namespace+"/"+pvc.Name, err) + pvNotFoundErr = err + pv = nil + } + } + + if groupResource == kuberesource.PersistentVolumes { + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &pv); err != nil { + v.logger.WithError(err).Error("fail to convert unstructured into PV") + return false, err + } + } + + if v.volumePolicy != nil { + vfd := resourcepolicies.NewVolumeFilterData(pv, nil, pvc) + action, err := v.volumePolicy.GetMatchAction(vfd) + if err != nil { + v.logger.WithError(err).Errorf("fail to get VolumePolicy match action for %+v", vfd) + return false, err + } + + // If there is a match action, and the action type is custom, return true + // if the provided parameters match as well, else return false. + // If there is no match action, also return false + if action != nil { + if action.Type == resourcepolicies.Custom { + for k, requiredValue := range matchParams { + if actionValue, ok := action.Parameters[k]; !ok || actionValue != requiredValue { + v.logger.Infof("Skipping custom action for %+v as value for parameter %s is %s rather than the required %s", vfd, k, actionValue, requiredValue) + return false, nil + } + } + v.logger.Infof("performing custom action for %+v", vfd) + return true, nil + } else { + v.logger.Infof("Skipping custom action for %+v as the action type is %s", vfd, action.Type) + return false, nil + } + } + } + // If resource is PVC, and PV is nil (e.g., Pending/Lost PVC with no matching policy), return the original error + // Don't error out on no PV, just return false + if groupResource == kuberesource.PersistentVolumeClaims && pv == nil && pvNotFoundErr != nil { + v.logger.WithError(pvNotFoundErr).Warnf("fail to get PV for PVC %s", pvc.Namespace+"/"+pvc.Name) + return false, nil + } + + v.logger.Infof("skipping custom action for pv %s due to no matching volume policy", pv.Name) + return false, nil +} + +// returns false if no matching action found. Returns true with the action name and Parameters map if there is a matching policy +func (v *volumeHelperImpl) GetActionParameters(obj runtime.Unstructured, groupResource schema.GroupResource) (bool, string, map[string]any, error) { + // if volume policy exists, return action parameters. + pvc := new(corev1api.PersistentVolumeClaim) + pv := new(corev1api.PersistentVolume) + var err error + + if groupResource == kuberesource.PersistentVolumeClaims { + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &pvc); err != nil { + v.logger.WithError(err).Error("fail to convert unstructured into PVC") + return false, "", nil, err + } + + pv, err = kubeutil.GetPVForPVC(pvc, v.client) + if err != nil { + v.logger.WithError(err).Warnf("failed to get PV for PVC %s", pvc.Namespace+"/"+pvc.Name) + return false, "", nil, nil + } + } + + if groupResource == kuberesource.PersistentVolumes { + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &pv); err != nil { + v.logger.WithError(err).Error("fail to convert unstructured into PV") + return false, "", nil, err + } + } + + if v.volumePolicy != nil { + vfd := resourcepolicies.NewVolumeFilterData(pv, nil, pvc) + action, err := v.volumePolicy.GetMatchAction(vfd) + if err != nil { + v.logger.WithError(err).Errorf("fail to get VolumePolicy match action for PV %s", pv.Name) + return false, "", nil, err + } + + // If there is a match action, and the action type is custom, return true + // if the provided parameters match as well, else return false. + // If there is no match action, also return false + if action != nil { + v.logger.Infof("found matching action for pv %s, returning parameters", pv.Name) + return true, string(action.Type), action.Parameters, nil + } + } + + v.logger.Infof("no matching volume policy found for pv %s, no parameters to return", pv.Name) + return false, "", nil, nil +} + func (v *volumeHelperImpl) shouldIncludeVolumeInBackup(vol corev1api.Volume) bool { includeVolumeInBackup := true // cannot backup hostpath volumes as they are not mounted into /var/lib/kubelet/pods diff --git a/internal/volumehelper/volume_policy_helper_test.go b/internal/volumehelper/volume_policy_helper_test.go index 8d6073c2b..5e52ae73b 100644 --- a/internal/volumehelper/volume_policy_helper_test.go +++ b/internal/volumehelper/volume_policy_helper_test.go @@ -286,7 +286,7 @@ func TestVolumeHelperImpl_ShouldPerformSnapshot(t *testing.T) { expectedErr: false, }, { - name: "PVC not having PV, return false and error case PV not found", + name: "PVC not having PV, return false and error when no matching policy", inputObj: builder.ForPersistentVolumeClaim("default", "example-pvc").StorageClass("gp2-csi").Result(), groupResource: kuberesource.PersistentVolumeClaims, resourcePolicies: &resourcepolicies.ResourcePolicies{ @@ -1234,3 +1234,312 @@ func TestNewVolumeHelperImplWithCache_UsesCache(t *testing.T) { require.NoError(t, err) require.False(t, shouldSnapshot, "Expected snapshot to be skipped due to fs-backup selection via cache") } + +// TestVolumeHelperImpl_ShouldPerformSnapshot_UnboundPVC tests that Pending and Lost PVCs with +// phase-based skip policies don't cause errors when GetPVForPVC would fail. +func TestVolumeHelperImpl_ShouldPerformSnapshot_UnboundPVC(t *testing.T) { + testCases := []struct { + name string + inputPVC *corev1api.PersistentVolumeClaim + resourcePolicies *resourcepolicies.ResourcePolicies + shouldSnapshot bool + expectedErr bool + }{ + { + name: "Pending PVC with phase-based skip policy should not error and return false", + inputPVC: builder.ForPersistentVolumeClaim("ns", "pvc-pending"). + StorageClass("non-existent-class"). + Phase(corev1api.ClaimPending). + Result(), + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "pvcPhase": []string{"Pending"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + }, + shouldSnapshot: false, + expectedErr: false, + }, + { + name: "Pending PVC without matching skip policy should error (no PV)", + inputPVC: builder.ForPersistentVolumeClaim("ns", "pvc-pending-no-policy"). + StorageClass("non-existent-class"). + Phase(corev1api.ClaimPending). + Result(), + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + }, + shouldSnapshot: false, + expectedErr: true, + }, + { + name: "Lost PVC with phase-based skip policy should not error and return false", + inputPVC: builder.ForPersistentVolumeClaim("ns", "pvc-lost"). + StorageClass("some-class"). + Phase(corev1api.ClaimLost). + Result(), + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "pvcPhase": []string{"Lost"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + }, + shouldSnapshot: false, + expectedErr: false, + }, + { + name: "Lost PVC with policy for Pending and Lost should not error and return false", + inputPVC: builder.ForPersistentVolumeClaim("ns", "pvc-lost"). + StorageClass("some-class"). + Phase(corev1api.ClaimLost). + Result(), + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "pvcPhase": []string{"Pending", "Lost"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + }, + shouldSnapshot: false, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClient := velerotest.NewFakeControllerRuntimeClient(t) + + var p *resourcepolicies.Policies + if tc.resourcePolicies != nil { + p = &resourcepolicies.Policies{} + err := p.BuildPolicy(tc.resourcePolicies) + require.NoError(t, err) + } + + vh := NewVolumeHelperImpl( + p, + ptr.To(true), + logrus.StandardLogger(), + fakeClient, + false, + false, + ) + + obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.inputPVC) + require.NoError(t, err) + + actualShouldSnapshot, actualError := vh.ShouldPerformSnapshot(&unstructured.Unstructured{Object: obj}, kuberesource.PersistentVolumeClaims) + if tc.expectedErr { + require.Error(t, actualError, "Want error; Got nil error") + return + } + + require.NoError(t, actualError) + require.Equalf(t, tc.shouldSnapshot, actualShouldSnapshot, "Want shouldSnapshot as %t; Got shouldSnapshot as %t", tc.shouldSnapshot, actualShouldSnapshot) + }) + } +} + +// TestVolumeHelperImpl_ShouldPerformFSBackup_UnboundPVC tests that Pending and Lost PVCs with +// phase-based skip policies don't cause errors when GetPVForPVC would fail. +func TestVolumeHelperImpl_ShouldPerformFSBackup_UnboundPVC(t *testing.T) { + testCases := []struct { + name string + pod *corev1api.Pod + pvc *corev1api.PersistentVolumeClaim + resourcePolicies *resourcepolicies.ResourcePolicies + shouldFSBackup bool + expectedErr bool + }{ + { + name: "Pending PVC with phase-based skip policy should not error and return false", + pod: builder.ForPod("ns", "pod-1"). + Volumes( + &corev1api.Volume{ + Name: "vol-pending", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-pending", + }, + }, + }).Result(), + pvc: builder.ForPersistentVolumeClaim("ns", "pvc-pending"). + StorageClass("non-existent-class"). + Phase(corev1api.ClaimPending). + Result(), + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "pvcPhase": []string{"Pending"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + }, + shouldFSBackup: false, + expectedErr: false, + }, + { + name: "Pending PVC without matching skip policy should error (no PV)", + pod: builder.ForPod("ns", "pod-1"). + Volumes( + &corev1api.Volume{ + Name: "vol-pending", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-pending-no-policy", + }, + }, + }).Result(), + pvc: builder.ForPersistentVolumeClaim("ns", "pvc-pending-no-policy"). + StorageClass("non-existent-class"). + Phase(corev1api.ClaimPending). + Result(), + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "storageClass": []string{"gp2-csi"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + }, + shouldFSBackup: false, + expectedErr: true, + }, + { + name: "Lost PVC with phase-based skip policy should not error and return false", + pod: builder.ForPod("ns", "pod-1"). + Volumes( + &corev1api.Volume{ + Name: "vol-lost", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-lost", + }, + }, + }).Result(), + pvc: builder.ForPersistentVolumeClaim("ns", "pvc-lost"). + StorageClass("some-class"). + Phase(corev1api.ClaimLost). + Result(), + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "pvcPhase": []string{"Lost"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + }, + shouldFSBackup: false, + expectedErr: false, + }, + { + name: "Lost PVC with policy for Pending and Lost should not error and return false", + pod: builder.ForPod("ns", "pod-1"). + Volumes( + &corev1api.Volume{ + Name: "vol-lost", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-lost", + }, + }, + }).Result(), + pvc: builder.ForPersistentVolumeClaim("ns", "pvc-lost"). + StorageClass("some-class"). + Phase(corev1api.ClaimLost). + Result(), + resourcePolicies: &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "pvcPhase": []string{"Pending", "Lost"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + }, + shouldFSBackup: false, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, tc.pvc) + require.NoError(t, fakeClient.Create(t.Context(), tc.pod)) + + var p *resourcepolicies.Policies + if tc.resourcePolicies != nil { + p = &resourcepolicies.Policies{} + err := p.BuildPolicy(tc.resourcePolicies) + require.NoError(t, err) + } + + vh := NewVolumeHelperImpl( + p, + ptr.To(true), + logrus.StandardLogger(), + fakeClient, + false, + false, + ) + + actualShouldFSBackup, actualError := vh.ShouldPerformFSBackup(tc.pod.Spec.Volumes[0], *tc.pod) + if tc.expectedErr { + require.Error(t, actualError, "Want error; Got nil error") + return + } + + require.NoError(t, actualError) + require.Equalf(t, tc.shouldFSBackup, actualShouldFSBackup, "Want shouldFSBackup as %t; Got shouldFSBackup as %t", tc.shouldFSBackup, actualShouldFSBackup) + }) + } +} diff --git a/pkg/apis/velero/v1/backup_repository_types.go b/pkg/apis/velero/v1/backup_repository_types.go index 621ffcfcc..8e2b3b715 100644 --- a/pkg/apis/velero/v1/backup_repository_types.go +++ b/pkg/apis/velero/v1/backup_repository_types.go @@ -35,8 +35,7 @@ type BackupRepositorySpec struct { // +optional RepositoryType string `json:"repositoryType"` - // ResticIdentifier is the full restic-compatible string for identifying - // this repository. This field is only used when RepositoryType is "restic". + // Deprecated // +optional ResticIdentifier string `json:"resticIdentifier,omitempty"` @@ -58,8 +57,7 @@ const ( BackupRepositoryPhaseReady BackupRepositoryPhase = "Ready" BackupRepositoryPhaseNotReady BackupRepositoryPhase = "NotReady" - BackupRepositoryTypeRestic string = "restic" - BackupRepositoryTypeKopia string = "kopia" + BackupRepositoryTypeKopia string = "kopia" ) // BackupRepositoryStatus is the current status of a BackupRepository. diff --git a/pkg/apis/velero/v1/labels_annotations.go b/pkg/apis/velero/v1/labels_annotations.go index c1431d3cc..921af498e 100644 --- a/pkg/apis/velero/v1/labels_annotations.go +++ b/pkg/apis/velero/v1/labels_annotations.go @@ -102,6 +102,15 @@ const ( // even if the resource contains a matching selector label. ExcludeFromBackupLabel = "velero.io/exclude-from-backup" + // SkipFromBackupAnnotation is the annotation used by internal BackupItemActions + // to indicate that a resource should be skipped from backup, + // even if it doesn't have the ExcludeFromBackupLabel. + // This is used in cases where we want to skip backup of a resource based on some logic in a plugin. + // + // Notice: SkipFromBackupAnnotation's priority is higher than MustIncludeAdditionalItemAnnotation. + // If SkipFromBackupAnnotation is set, the resource will be skipped even if MustIncludeAdditionalItemAnnotation is set. + SkipFromBackupAnnotation = "velero.io/skip-from-backup" + // defaultVGSLabelKey is the default label key used to group PVCs under a VolumeGroupSnapshot DefaultVGSLabelKey = "velero.io/volume-group" @@ -132,6 +141,7 @@ const ( VolumeSnapshotRestoreSize = "velero.io/csi-volumesnapshot-restore-size" DriverNameAnnotation = "velero.io/csi-driver-name" VSCDeletionPolicyAnnotation = "velero.io/csi-vsc-deletion-policy" + VolumeGroupSnapshotHandleAnnotation = "velero.io/csi-volumegroupsnapshot-handle" VolumeSnapshotClassSelectorLabel = "velero.io/csi-volumesnapshot-class" VolumeSnapshotClassDriverBackupAnnotationPrefix = "velero.io/csi-volumesnapshot-class" VolumeSnapshotClassDriverPVCAnnotation = "velero.io/csi-volumesnapshot-class" diff --git a/pkg/archive/extractor.go b/pkg/archive/extractor.go index 87d9ab413..aae9a7a85 100644 --- a/pkg/archive/extractor.go +++ b/pkg/archive/extractor.go @@ -19,8 +19,10 @@ package archive import ( "archive/tar" "compress/gzip" + "fmt" "io" "path/filepath" + "strings" "github.com/sirupsen/logrus" @@ -66,6 +68,16 @@ func (e *Extractor) writeFile(target string, tarRdr *tar.Reader) error { return nil } +// sanitizeArchivePath sanitizes archive file path from "G305: Zip Slip vulnerability" +func sanitizeArchivePath(destDir, sourcePath string) (targetPath string, err error) { + targetPath = filepath.Join(destDir, sourcePath) + if strings.HasPrefix(targetPath, filepath.Clean(destDir)) { + return targetPath, nil + } + + return "", fmt.Errorf("invalid archive path %q: escapes target directory", sourcePath) +} + func (e *Extractor) readBackup(tarRdr *tar.Reader) (string, error) { dir, err := e.fs.TempDir("", "") if err != nil { @@ -84,7 +96,11 @@ func (e *Extractor) readBackup(tarRdr *tar.Reader) (string, error) { return "", err } - target := filepath.Join(dir, header.Name) //nolint:gosec // Internal usage. No need to check. + target, err := sanitizeArchivePath(dir, header.Name) + if err != nil { + e.log.Infof("error sanitizing archive path: %s", err.Error()) + return "", err + } switch header.Typeflag { case tar.TypeDir: diff --git a/pkg/archive/extractor_test.go b/pkg/archive/extractor_test.go index 47cea734c..a4daf02ca 100644 --- a/pkg/archive/extractor_test.go +++ b/pkg/archive/extractor_test.go @@ -18,6 +18,7 @@ package archive import ( "archive/tar" + "bytes" "compress/gzip" "io" "os" @@ -87,6 +88,31 @@ func TestUnzipAndExtractBackup(t *testing.T) { } } +func TestUnzipAndExtractBackupRejectsPathTraversal(t *testing.T) { + ext := NewExtractor(test.NewLogger(), test.NewFakeFileSystem()) + + var buf bytes.Buffer + gzw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gzw) + + err := tw.WriteHeader(&tar.Header{ + Name: "../escape.txt", + Mode: 0600, + Typeflag: tar.TypeReg, + Size: int64(len("data")), + }) + require.NoError(t, err) + + _, err = tw.Write([]byte("data")) + require.NoError(t, err) + require.NoError(t, tw.Close()) + require.NoError(t, gzw.Close()) + + _, err = ext.UnzipAndExtractBackup(&buf) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid archive path") +} + func createArchive(files []string, fs filesystem.Interface) (string, error) { outName := "output.tar.gz" out, err := fs.Create(outName) diff --git a/pkg/backup/actions/csi/pvc_action.go b/pkg/backup/actions/csi/pvc_action.go index 8e2e77316..7f5fd2afa 100644 --- a/pkg/backup/actions/csi/pvc_action.go +++ b/pkg/backup/actions/csi/pvc_action.go @@ -24,7 +24,7 @@ import ( "k8s.io/client-go/util/retry" - volumegroupsnapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta1" + volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -44,7 +44,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" - internalvolumehelper "github.com/vmware-tanzu/velero/internal/volumehelper" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" veleroclient "github.com/vmware-tanzu/velero/pkg/client" @@ -59,6 +58,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/util/csi" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" podvolumeutil "github.com/vmware-tanzu/velero/pkg/util/podvolume" + vhutil "github.com/vmware-tanzu/velero/pkg/util/volumehelper" ) // TODO: Replace hardcoded VolumeSnapshot finalizer strings with constants from @@ -128,9 +128,9 @@ func (p *pvcBackupItemAction) ensurePVCPodCacheForNamespace(ctx context.Context, // getVolumeHelperWithCache creates a VolumeHelper using the pre-built PVC-to-Pod cache. // The cache should be ensured for the relevant namespace(s) before calling this. -func (p *pvcBackupItemAction) getVolumeHelperWithCache(backup *velerov1api.Backup) (internalvolumehelper.VolumeHelper, error) { +func (p *pvcBackupItemAction) getVolumeHelperWithCache(backup *velerov1api.Backup) (vhutil.VolumeHelper, error) { // Create VolumeHelper with our lazy-built cache - vh, err := internalvolumehelper.NewVolumeHelperImplWithCache( + vh, err := volumehelper.NewVolumeHelperWithCache( *backup, p.crClient, p.log, @@ -149,7 +149,7 @@ func (p *pvcBackupItemAction) getVolumeHelperWithCache(backup *velerov1api.Backu // Since plugin instances are unique per backup (created via newPluginManager and // cleaned up via CleanupClients at backup completion), we can safely cache this. // See issue #9179 and PR #9226 for details. -func (p *pvcBackupItemAction) getOrCreateVolumeHelper(backup *velerov1api.Backup) (internalvolumehelper.VolumeHelper, error) { +func (p *pvcBackupItemAction) getOrCreateVolumeHelper(backup *velerov1api.Backup) (vhutil.VolumeHelper, error) { // Initialize the PVC-to-Pod cache if needed if p.pvcPodCache == nil { p.pvcPodCache = podvolumeutil.NewPVCPodCache() @@ -322,13 +322,9 @@ func (p *pvcBackupItemAction) Execute( return nil, nil, "", nil, err } - shouldSnapshot, err := volumehelper.ShouldPerformSnapshotWithVolumeHelper( + shouldSnapshot, err := vh.ShouldPerformSnapshot( item, kuberesource.PersistentVolumeClaims, - *backup, - p.crClient, - p.log, - vh, ) if err != nil { return nil, nil, "", nil, err @@ -471,7 +467,7 @@ func (p *pvcBackupItemAction) Progress( return progress, biav2.InvalidOperationIDError(operationID) } - dataUpload, err := getDataUpload(context.Background(), p.crClient, operationID) + dataUpload, err := getDataUpload(context.Background(), p.crClient, backup.Namespace, operationID) if err != nil { p.log.Errorf( "fail to get DataUpload for backup %s/%s by operation ID %s: %s", @@ -516,7 +512,7 @@ func (p *pvcBackupItemAction) Cancel(operationID string, backup *velerov1api.Bac return biav2.InvalidOperationIDError(operationID) } - dataUpload, err := getDataUpload(context.Background(), p.crClient, operationID) + dataUpload, err := getDataUpload(context.Background(), p.crClient, backup.Namespace, operationID) if err != nil { p.log.Errorf( "fail to get DataUpload for backup %s/%s: %s", @@ -609,10 +605,12 @@ func createDataUpload( func getDataUpload( ctx context.Context, crClient crclient.Client, + namespace string, operationID string, ) (*velerov2alpha1.DataUpload, error) { dataUploadList := new(velerov2alpha1.DataUploadList) err := crClient.List(ctx, dataUploadList, &crclient.ListOptions{ + Namespace: namespace, LabelSelector: labels.SelectorFromSet( map[string]string{velerov1api.AsyncOperationIDLabel: operationID}, ), @@ -708,7 +706,7 @@ func (p *pvcBackupItemAction) getVolumeSnapshotReference( } // Filter PVCs by volume policy - filteredPVCs, err := p.filterPVCsByVolumePolicy(groupedPVCs, backup, vh) + filteredPVCs, err := p.filterPVCsByVolumePolicy(groupedPVCs, vh) if err != nil { return nil, errors.Wrapf(err, "failed to filter PVCs by volume policy for VolumeGroupSnapshot group %q", group) } @@ -769,7 +767,7 @@ func (p *pvcBackupItemAction) getVolumeSnapshotReference( } // Re-fetch latest VGS to ensure status is populated after VGSC binding - latestVGS := &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{} + latestVGS := &volumegroupsnapshotv1beta2.VolumeGroupSnapshot{} if err := p.crClient.Get(ctx, crclient.ObjectKeyFromObject(newVGS), latestVGS); err != nil { return nil, errors.Wrapf(err, "failed to re-fetch VolumeGroupSnapshot %s after VGSC binding wait", newVGS.Name) } @@ -844,8 +842,7 @@ func (p *pvcBackupItemAction) listGroupedPVCs(ctx context.Context, namespace, la func (p *pvcBackupItemAction) filterPVCsByVolumePolicy( pvcs []corev1api.PersistentVolumeClaim, - backup *velerov1api.Backup, - vh internalvolumehelper.VolumeHelper, + vh vhutil.VolumeHelper, ) ([]corev1api.PersistentVolumeClaim, error) { var filteredPVCs []corev1api.PersistentVolumeClaim @@ -859,13 +856,9 @@ func (p *pvcBackupItemAction) filterPVCsByVolumePolicy( // Check if this PVC should be snapshotted according to volume policies // Uses the cached VolumeHelper for better performance with many PVCs/pods - shouldSnapshot, err := volumehelper.ShouldPerformSnapshotWithVolumeHelper( + shouldSnapshot, err := vh.ShouldPerformSnapshot( unstructuredPVC, kuberesource.PersistentVolumeClaims, - *backup, - p.crClient, - p.log, - vh, ) if err != nil { return nil, errors.Wrapf(err, "failed to check volume policy for PVC %s/%s", pvc.Namespace, pvc.Name) @@ -922,7 +915,7 @@ func (p *pvcBackupItemAction) determineVGSClass( } // 3. Fallback to label-based default - vgsClassList := &volumegroupsnapshotv1beta1.VolumeGroupSnapshotClassList{} + vgsClassList := &volumegroupsnapshotv1beta2.VolumeGroupSnapshotClassList{} if err := p.crClient.List(ctx, vgsClassList); err != nil { return "", errors.Wrap(err, "failed to list VolumeGroupSnapshotClasses") } @@ -951,22 +944,22 @@ func (p *pvcBackupItemAction) createVolumeGroupSnapshot( backup *velerov1api.Backup, pvc corev1api.PersistentVolumeClaim, vgsLabelKey, vgsLabelValue, vgsClassName string, -) (*volumegroupsnapshotv1beta1.VolumeGroupSnapshot, error) { +) (*volumegroupsnapshotv1beta2.VolumeGroupSnapshot, error) { vgsLabels := map[string]string{ velerov1api.BackupNameLabel: label.GetValidName(backup.Name), velerov1api.BackupUIDLabel: string(backup.UID), vgsLabelKey: vgsLabelValue, } - vgs := &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{ + vgs := &volumegroupsnapshotv1beta2.VolumeGroupSnapshot{ ObjectMeta: metav1.ObjectMeta{ GenerateName: fmt.Sprintf("velero-%s-", vgsLabelValue), Namespace: pvc.Namespace, Labels: vgsLabels, }, - Spec: volumegroupsnapshotv1beta1.VolumeGroupSnapshotSpec{ + Spec: volumegroupsnapshotv1beta2.VolumeGroupSnapshotSpec{ VolumeGroupSnapshotClassName: &vgsClassName, - Source: volumegroupsnapshotv1beta1.VolumeGroupSnapshotSource{ + Source: volumegroupsnapshotv1beta2.VolumeGroupSnapshotSource{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ vgsLabelKey: vgsLabelValue, @@ -994,7 +987,7 @@ func (p *pvcBackupItemAction) createVolumeGroupSnapshot( func (p *pvcBackupItemAction) waitForVGSAssociatedVS( ctx context.Context, groupedPVCs []corev1api.PersistentVolumeClaim, - vgs *volumegroupsnapshotv1beta1.VolumeGroupSnapshot, + vgs *volumegroupsnapshotv1beta2.VolumeGroupSnapshot, timeout time.Duration, ) (map[string]*snapshotv1api.VolumeSnapshot, error) { expected := len(groupedPVCs) @@ -1037,10 +1030,10 @@ func (p *pvcBackupItemAction) waitForVGSAssociatedVS( return vsMap, nil } -func hasOwnerReference(obj metav1.Object, vgs *volumegroupsnapshotv1beta1.VolumeGroupSnapshot) bool { +func hasOwnerReference(obj metav1.Object, vgs *volumegroupsnapshotv1beta2.VolumeGroupSnapshot) bool { for _, ref := range obj.GetOwnerReferences() { if ref.Kind == kuberesource.VGSKind && - ref.APIVersion == volumegroupsnapshotv1beta1.GroupName+"/"+volumegroupsnapshotv1beta1.SchemeGroupVersion.Version && + ref.APIVersion == volumegroupsnapshotv1beta2.GroupName+"/"+volumegroupsnapshotv1beta2.SchemeGroupVersion.Version && ref.UID == vgs.UID { return true } @@ -1051,7 +1044,7 @@ func hasOwnerReference(obj metav1.Object, vgs *volumegroupsnapshotv1beta1.Volume func (p *pvcBackupItemAction) updateVGSCreatedVS( ctx context.Context, vsMap map[string]*snapshotv1api.VolumeSnapshot, - vgs *volumegroupsnapshotv1beta1.VolumeGroupSnapshot, + vgs *volumegroupsnapshotv1beta2.VolumeGroupSnapshot, backup *velerov1api.Backup, ) error { for pvcName, vs := range vsMap { @@ -1094,7 +1087,7 @@ func (p *pvcBackupItemAction) updateVGSCreatedVS( return nil } -func (p *pvcBackupItemAction) patchVGSCDeletionPolicy(ctx context.Context, vgs *volumegroupsnapshotv1beta1.VolumeGroupSnapshot) error { +func (p *pvcBackupItemAction) patchVGSCDeletionPolicy(ctx context.Context, vgs *volumegroupsnapshotv1beta2.VolumeGroupSnapshot) error { if vgs == nil || vgs.Status == nil || vgs.Status.BoundVolumeGroupSnapshotContentName == nil { return errors.New("VolumeGroupSnapshotContent name not found in VGS status") } @@ -1102,7 +1095,7 @@ func (p *pvcBackupItemAction) patchVGSCDeletionPolicy(ctx context.Context, vgs * vgscName := vgs.Status.BoundVolumeGroupSnapshotContentName return retry.RetryOnConflict(retry.DefaultBackoff, func() error { - vgsc := &volumegroupsnapshotv1beta1.VolumeGroupSnapshotContent{} + vgsc := &volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent{} if err := p.crClient.Get(ctx, crclient.ObjectKey{Name: *vgscName}, vgsc); err != nil { return errors.Wrapf(err, "failed to get VolumeGroupSnapshotContent %s for VolumeGroupSnapshot %s/%s", *vgscName, vgs.Namespace, vgs.Name) } @@ -1121,9 +1114,9 @@ func (p *pvcBackupItemAction) patchVGSCDeletionPolicy(ctx context.Context, vgs * }) } -func (p *pvcBackupItemAction) deleteVGSAndVGSC(ctx context.Context, vgs *volumegroupsnapshotv1beta1.VolumeGroupSnapshot) error { +func (p *pvcBackupItemAction) deleteVGSAndVGSC(ctx context.Context, vgs *volumegroupsnapshotv1beta2.VolumeGroupSnapshot) error { if vgs.Status != nil && vgs.Status.BoundVolumeGroupSnapshotContentName != nil { - vgsc := &volumegroupsnapshotv1beta1.VolumeGroupSnapshotContent{ + vgsc := &volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: *vgs.Status.BoundVolumeGroupSnapshotContentName, }, @@ -1148,11 +1141,11 @@ func (p *pvcBackupItemAction) deleteVGSAndVGSC(ctx context.Context, vgs *volumeg func (p *pvcBackupItemAction) waitForVGSCBinding( ctx context.Context, - vgs *volumegroupsnapshotv1beta1.VolumeGroupSnapshot, + vgs *volumegroupsnapshotv1beta2.VolumeGroupSnapshot, timeout time.Duration, ) error { return wait.PollUntilContextTimeout(ctx, time.Second, timeout, true, func(ctx context.Context) (bool, error) { - vgsRef := &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{} + vgsRef := &volumegroupsnapshotv1beta2.VolumeGroupSnapshot{} if err := p.crClient.Get(ctx, crclient.ObjectKeyFromObject(vgs), vgsRef); err != nil { return false, err } @@ -1165,8 +1158,8 @@ func (p *pvcBackupItemAction) waitForVGSCBinding( }) } -func (p *pvcBackupItemAction) getVGSByLabels(ctx context.Context, namespace string, labels map[string]string) (*volumegroupsnapshotv1beta1.VolumeGroupSnapshot, error) { - vgsList := &volumegroupsnapshotv1beta1.VolumeGroupSnapshotList{} +func (p *pvcBackupItemAction) getVGSByLabels(ctx context.Context, namespace string, labels map[string]string) (*volumegroupsnapshotv1beta2.VolumeGroupSnapshot, error) { + vgsList := &volumegroupsnapshotv1beta2.VolumeGroupSnapshotList{} if err := p.crClient.List(ctx, vgsList, crclient.InNamespace(namespace), crclient.MatchingLabels(labels), diff --git a/pkg/backup/actions/csi/pvc_action_test.go b/pkg/backup/actions/csi/pvc_action_test.go index b94d63701..9ffe20be5 100644 --- a/pkg/backup/actions/csi/pvc_action_test.go +++ b/pkg/backup/actions/csi/pvc_action_test.go @@ -25,7 +25,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/kuberesource" - volumegroupsnapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta1" + volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2" "github.com/stretchr/testify/assert" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" @@ -307,6 +307,28 @@ func TestProgress(t *testing.T) { operationID: "testing", expectedErr: "not found DataUpload for operationID testing", }, + { + name: "DataUpload in different namespace is not found", + backup: builder.ForBackup("velero", "test").Result(), + dataUpload: &velerov2alpha1.DataUpload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataUpload", + APIVersion: "v2alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "other-namespace", + Name: "testing", + Labels: map[string]string{ + velerov1api.AsyncOperationIDLabel: "testing", + }, + }, + Status: velerov2alpha1.DataUploadStatus{ + Phase: velerov2alpha1.DataUploadPhaseFailed, + }, + }, + operationID: "testing", + expectedErr: "not found DataUpload for operationID testing", + }, { name: "DataUpload is found", backup: builder.ForBackup("velero", "test").Result(), @@ -375,15 +397,15 @@ func TestCancel(t *testing.T) { tests := []struct { name string backup *velerov1api.Backup - dataUpload velerov2alpha1.DataUpload + dataUpload *velerov2alpha1.DataUpload operationID string - expectedErr error + expectedErr string expectedDataUpload velerov2alpha1.DataUpload }{ { name: "Cancel DataUpload", backup: builder.ForBackup("velero", "test").Result(), - dataUpload: velerov2alpha1.DataUpload{ + dataUpload: &velerov2alpha1.DataUpload{ TypeMeta: metav1.TypeMeta{ Kind: "DataUpload", APIVersion: velerov2alpha1.SchemeGroupVersion.String(), @@ -414,6 +436,31 @@ func TestCancel(t *testing.T) { }, }, }, + { + name: "DataUpload cannot be found", + backup: builder.ForBackup("velero", "test").Result(), + operationID: "testing", + expectedErr: "not found DataUpload for operationID testing", + }, + { + name: "DataUpload in different namespace is not found", + backup: builder.ForBackup("velero", "test").Result(), + dataUpload: &velerov2alpha1.DataUpload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataUpload", + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "other-namespace", + Name: "testing", + Labels: map[string]string{ + velerov1api.AsyncOperationIDLabel: "testing", + }, + }, + }, + operationID: "testing", + expectedErr: "not found DataUpload for operationID testing", + }, } for _, tc := range tests { @@ -426,17 +473,23 @@ func TestCancel(t *testing.T) { crClient: crClient, } - err := crClient.Create(t.Context(), &tc.dataUpload) - require.NoError(t, err) + if tc.dataUpload != nil { + err := crClient.Create(t.Context(), tc.dataUpload) + require.NoError(t, err) + } - err = pvcBIA.Cancel(tc.operationID, tc.backup) - require.NoError(t, err) + err := pvcBIA.Cancel(tc.operationID, tc.backup) + if tc.expectedErr != "" { + require.EqualError(t, err, tc.expectedErr) + } else { + require.NoError(t, err) - du := new(velerov2alpha1.DataUpload) - err = crClient.Get(t.Context(), crclient.ObjectKey{Namespace: tc.dataUpload.Namespace, Name: tc.dataUpload.Name}, du) - require.NoError(t, err) + du := new(velerov2alpha1.DataUpload) + err = crClient.Get(t.Context(), crclient.ObjectKey{Namespace: tc.dataUpload.Namespace, Name: tc.dataUpload.Name}, du) + require.NoError(t, err) - require.True(t, cmp.Equal(tc.expectedDataUpload, *du, cmpopts.IgnoreFields(velerov2alpha1.DataUpload{}, "ResourceVersion"))) + require.True(t, cmp.Equal(tc.expectedDataUpload, *du, cmpopts.IgnoreFields(velerov2alpha1.DataUpload{}, "ResourceVersion"))) + } }) } } @@ -842,9 +895,13 @@ volumePolicies: crClient: client, } - // Pass nil for VolumeHelper in tests - it will fall back to creating a new one per call - // This is the expected behavior for testing and third-party plugins - result, err := action.filterPVCsByVolumePolicy(tt.pvcs, backup, nil) + // Create a VolumeHelper using the same method the plugin would use + vh, err := action.getOrCreateVolumeHelper(backup) + require.NoError(t, err) + require.NotNil(t, vh) + + // Test with the pre-created VolumeHelper + result, err := action.filterPVCsByVolumePolicy(tt.pvcs, vh) if tt.expectError { require.Error(t, err) } else { @@ -959,7 +1016,7 @@ volumePolicies: require.NotNil(t, vh) // Test with the pre-created VolumeHelper (non-nil path) - result, err := action.filterPVCsByVolumePolicy(pvcs, backup, vh) + result, err := action.filterPVCsByVolumePolicy(pvcs, vh) require.NoError(t, err) // Should filter out the NFS PVC, leaving only the CSI PVC @@ -1117,7 +1174,7 @@ func TestDetermineVGSClass(t *testing.T) { name string backup *velerov1api.Backup pvc *corev1api.PersistentVolumeClaim - existingVGSClass []volumegroupsnapshotv1beta1.VolumeGroupSnapshotClass + existingVGSClass []volumegroupsnapshotv1beta2.VolumeGroupSnapshotClass expectError bool expectResult string }{ @@ -1149,7 +1206,7 @@ func TestDetermineVGSClass(t *testing.T) { name: "Default label-based match", pvc: &corev1api.PersistentVolumeClaim{}, backup: &velerov1api.Backup{}, - existingVGSClass: []volumegroupsnapshotv1beta1.VolumeGroupSnapshotClass{ + existingVGSClass: []volumegroupsnapshotv1beta2.VolumeGroupSnapshotClass{ { ObjectMeta: metav1.ObjectMeta{ Name: "default-class", @@ -1170,7 +1227,7 @@ func TestDetermineVGSClass(t *testing.T) { name: "Multiple matching VGS classes", pvc: &corev1api.PersistentVolumeClaim{}, backup: &velerov1api.Backup{}, - existingVGSClass: []volumegroupsnapshotv1beta1.VolumeGroupSnapshotClass{ + existingVGSClass: []volumegroupsnapshotv1beta2.VolumeGroupSnapshotClass{ { ObjectMeta: metav1.ObjectMeta{ Name: "class1", @@ -1200,7 +1257,7 @@ func TestDetermineVGSClass(t *testing.T) { client := velerotest.NewFakeControllerRuntimeClient(t, initObjs...) logger := logrus.New() - require.NoError(t, volumegroupsnapshotv1beta1.AddToScheme(client.Scheme())) + require.NoError(t, volumegroupsnapshotv1beta2.AddToScheme(client.Scheme())) action := &pvcBackupItemAction{crClient: client, log: logger} @@ -1259,13 +1316,13 @@ func TestCreateVolumeGroupSnapshot(t *testing.T) { assert.Equal(t, string(testBackup.UID), vgs.Labels[velerov1api.BackupUIDLabel]) // Check that it exists in fake client - retrieved := &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{} + retrieved := &volumegroupsnapshotv1beta2.VolumeGroupSnapshot{} err = crClient.Get(t.Context(), crclient.ObjectKey{Name: vgs.Name, Namespace: vgs.Namespace}, retrieved) require.NoError(t, err) } func TestWaitForVGSAssociatedVS(t *testing.T) { - vgs := &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{ + vgs := &volumegroupsnapshotv1beta2.VolumeGroupSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "test-vgs", Namespace: "test-ns", @@ -1278,7 +1335,7 @@ func TestWaitForVGSAssociatedVS(t *testing.T) { if owned { refs = []metav1.OwnerReference{ { - APIVersion: "groupsnapshot.storage.k8s.io/v1beta1", + APIVersion: "groupsnapshot.storage.k8s.io/v1beta2", Kind: "VolumeGroupSnapshot", Name: vgs.Name, UID: vgs.UID, @@ -1425,7 +1482,7 @@ func TestUpdateVGSCreatedVS(t *testing.T) { }, } - vgs := &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{ + vgs := &volumegroupsnapshotv1beta2.VolumeGroupSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "test-vgs", Namespace: "ns", @@ -1438,7 +1495,7 @@ func TestUpdateVGSCreatedVS(t *testing.T) { if withVGSOwner { refs = []metav1.OwnerReference{ { - APIVersion: "groupsnapshot.storage.k8s.io/v1beta1", + APIVersion: "groupsnapshot.storage.k8s.io/v1beta2", Kind: "VolumeGroupSnapshot", Name: vgs.Name, UID: vgs.UID, @@ -1557,18 +1614,18 @@ func TestPatchVGSCDeletionPolicy(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - vgsc := &volumegroupsnapshotv1beta1.VolumeGroupSnapshotContent{ + vgsc := &volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent{ ObjectMeta: metav1.ObjectMeta{Name: "test-vgsc"}, - Spec: volumegroupsnapshotv1beta1.VolumeGroupSnapshotContentSpec{ + Spec: volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentSpec{ DeletionPolicy: tt.initialPolicy, }, } - vgs := &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{ + vgs := &volumegroupsnapshotv1beta2.VolumeGroupSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "test-vgs", Namespace: "ns", }, - Status: &volumegroupsnapshotv1beta1.VolumeGroupSnapshotStatus{ + Status: &volumegroupsnapshotv1beta2.VolumeGroupSnapshotStatus{ BoundVolumeGroupSnapshotContentName: pointer.String("test-vgsc"), }, } @@ -1586,7 +1643,7 @@ func TestPatchVGSCDeletionPolicy(t *testing.T) { } require.NoError(t, err) - updated := &volumegroupsnapshotv1beta1.VolumeGroupSnapshotContent{} + updated := &volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent{} err = client.Get(t.Context(), crclient.ObjectKey{Name: "test-vgsc"}, updated) require.NoError(t, err) require.Equal(t, tt.expectedPolicy, updated.Spec.DeletionPolicy) @@ -1595,20 +1652,20 @@ func TestPatchVGSCDeletionPolicy(t *testing.T) { } func TestDeleteVGSAndVGSC(t *testing.T) { - makeVGS := func(name, namespace string, boundVGSCName *string) *volumegroupsnapshotv1beta1.VolumeGroupSnapshot { - return &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{ + makeVGS := func(name, namespace string, boundVGSCName *string) *volumegroupsnapshotv1beta2.VolumeGroupSnapshot { + return &volumegroupsnapshotv1beta2.VolumeGroupSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, }, - Status: &volumegroupsnapshotv1beta1.VolumeGroupSnapshotStatus{ + Status: &volumegroupsnapshotv1beta2.VolumeGroupSnapshotStatus{ BoundVolumeGroupSnapshotContentName: boundVGSCName, }, } } - makeVGSC := func(name string) *volumegroupsnapshotv1beta1.VolumeGroupSnapshotContent { - return &volumegroupsnapshotv1beta1.VolumeGroupSnapshotContent{ + makeVGSC := func(name string) *volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent { + return &volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, @@ -1617,8 +1674,8 @@ func TestDeleteVGSAndVGSC(t *testing.T) { tests := []struct { name string - vgs *volumegroupsnapshotv1beta1.VolumeGroupSnapshot - existingVGSC *volumegroupsnapshotv1beta1.VolumeGroupSnapshotContent + vgs *volumegroupsnapshotv1beta2.VolumeGroupSnapshot + existingVGSC *volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent expectVGSCDelete bool expectVGSDelete bool }{ @@ -1664,13 +1721,13 @@ func TestDeleteVGSAndVGSC(t *testing.T) { // Check VGSC is deleted if tt.expectVGSCDelete { - got := &volumegroupsnapshotv1beta1.VolumeGroupSnapshotContent{} + got := &volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent{} err = client.Get(t.Context(), crclient.ObjectKey{Name: "test-vgsc"}, got) assert.True(t, apierrors.IsNotFound(err), "expected VGSC to be deleted") } // Check VGS is deleted - gotVGS := &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{} + gotVGS := &volumegroupsnapshotv1beta2.VolumeGroupSnapshot{} err = client.Get(t.Context(), crclient.ObjectKey{Name: "test-vgs", Namespace: "ns"}, gotVGS) assert.True(t, apierrors.IsNotFound(err), "expected VGS to be deleted") }) @@ -1765,8 +1822,8 @@ func TestFindExistingVSForBackup(t *testing.T) { } func TestWaitForVGSCBinding(t *testing.T) { - makeVGS := func(name string, withStatus bool) *volumegroupsnapshotv1beta1.VolumeGroupSnapshot { - vgs := &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{ + makeVGS := func(name string, withStatus bool) *volumegroupsnapshotv1beta2.VolumeGroupSnapshot { + vgs := &volumegroupsnapshotv1beta2.VolumeGroupSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: "ns", @@ -1774,7 +1831,7 @@ func TestWaitForVGSCBinding(t *testing.T) { } if withStatus { contentName := "vgsc-123" - vgs.Status = &volumegroupsnapshotv1beta1.VolumeGroupSnapshotStatus{ + vgs.Status = &volumegroupsnapshotv1beta2.VolumeGroupSnapshotStatus{ BoundVolumeGroupSnapshotContentName: &contentName, } } @@ -1783,7 +1840,7 @@ func TestWaitForVGSCBinding(t *testing.T) { tests := []struct { name string - vgs *volumegroupsnapshotv1beta1.VolumeGroupSnapshot + vgs *volumegroupsnapshotv1beta2.VolumeGroupSnapshot expectErr bool }{ { @@ -1826,8 +1883,8 @@ func TestGetVGSByLabels(t *testing.T) { labelVal := "backup-123" testLabels := map[string]string{labelKey: labelVal} - makeVGS := func(name string, labels map[string]string) *volumegroupsnapshotv1beta1.VolumeGroupSnapshot { - return &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{ + makeVGS := func(name string, labels map[string]string) *volumegroupsnapshotv1beta2.VolumeGroupSnapshot { + return &volumegroupsnapshotv1beta2.VolumeGroupSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: "test-ns", @@ -1912,7 +1969,7 @@ func (f *failingClient) List(ctx context.Context, list crclient.ObjectList, opts } func TestHasOwnerReference(t *testing.T) { - vgs := &volumegroupsnapshotv1beta1.VolumeGroupSnapshot{ + vgs := &volumegroupsnapshotv1beta2.VolumeGroupSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "test-vgs", Namespace: "test-ns", @@ -1929,7 +1986,7 @@ func TestHasOwnerReference(t *testing.T) { name: "match kind, apiversion, uid", ownerRef: metav1.OwnerReference{ Kind: kuberesource.VGSKind, - APIVersion: volumegroupsnapshotv1beta1.GroupName + "/" + volumegroupsnapshotv1beta1.SchemeGroupVersion.Version, + APIVersion: volumegroupsnapshotv1beta2.GroupName + "/" + volumegroupsnapshotv1beta2.SchemeGroupVersion.Version, UID: vgs.UID, }, expect: true, @@ -1938,7 +1995,7 @@ func TestHasOwnerReference(t *testing.T) { name: "mismatch kind", ownerRef: metav1.OwnerReference{ Kind: "other-kind", - APIVersion: volumegroupsnapshotv1beta1.GroupName + "/" + volumegroupsnapshotv1beta1.SchemeGroupVersion.Version, + APIVersion: volumegroupsnapshotv1beta2.GroupName + "/" + volumegroupsnapshotv1beta2.SchemeGroupVersion.Version, UID: vgs.UID, }, expect: false, @@ -1956,7 +2013,7 @@ func TestHasOwnerReference(t *testing.T) { name: "mismatch uid", ownerRef: metav1.OwnerReference{ Kind: kuberesource.VGSKind, - APIVersion: volumegroupsnapshotv1beta1.GroupName + "/" + volumegroupsnapshotv1beta1.SchemeGroupVersion.Version, + APIVersion: volumegroupsnapshotv1beta2.GroupName + "/" + volumegroupsnapshotv1beta2.SchemeGroupVersion.Version, UID: "wrong-uid", }, expect: false, diff --git a/pkg/backup/actions/csi/volumesnapshot_action.go b/pkg/backup/actions/csi/volumesnapshot_action.go index b7283b628..0e0e9a840 100644 --- a/pkg/backup/actions/csi/volumesnapshot_action.go +++ b/pkg/backup/actions/csi/volumesnapshot_action.go @@ -151,6 +151,12 @@ func (p *volumeSnapshotBackupItemAction) Execute( annotations[velerov1api.VolumeSnapshotRestoreSize] = resource.NewQuantity( *vsc.Status.RestoreSize, resource.BinarySI).String() } + + // Capture VolumeGroupSnapshotHandle to create stub VGSC during restore + // for CSI drivers that populate this field (e.g., Ceph RBD). + if vsc.Status.VolumeGroupSnapshotHandle != nil { + annotations[velerov1api.VolumeGroupSnapshotHandleAnnotation] = *vsc.Status.VolumeGroupSnapshotHandle + } } p.log.Infof("Patching VolumeSnapshotContent %s with velero BackupNameLabel", diff --git a/pkg/backup/backed_up_items_map.go b/pkg/backup/backed_up_items_map.go index f5764cd8e..174a50e1e 100644 --- a/pkg/backup/backed_up_items_map.go +++ b/pkg/backup/backed_up_items_map.go @@ -98,6 +98,14 @@ func (m *backedUpItemsMap) AddItem(key itemKey) { m.totalItems[key] = struct{}{} } +func (m *backedUpItemsMap) DeleteItem(key itemKey) { + m.Lock() + defer m.Unlock() + + delete(m.backedUpItems, key) + delete(m.totalItems, key) +} + func (m *backedUpItemsMap) AddItemToTotal(key itemKey) { m.Lock() defer m.Unlock() diff --git a/pkg/backup/item_backupper.go b/pkg/backup/item_backupper.go index 7b1ea69bd..2ca266e91 100644 --- a/pkg/backup/item_backupper.go +++ b/pkg/backup/item_backupper.go @@ -40,7 +40,6 @@ import ( "github.com/vmware-tanzu/velero/internal/hook" "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/volume" - "github.com/vmware-tanzu/velero/internal/volumehelper" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/archive" "github.com/vmware-tanzu/velero/pkg/client" @@ -54,6 +53,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/podvolume" "github.com/vmware-tanzu/velero/pkg/util/boolptr" csiutil "github.com/vmware-tanzu/velero/pkg/util/csi" + "github.com/vmware-tanzu/velero/pkg/util/volumehelper" ) const ( @@ -244,6 +244,14 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti return false, itemFiles, kubeerrs.NewAggregate(backupErrs) } + // If err is nil and updatedObj is nil, it means the item is skipped by plugin action, + // we should return here to avoid backing up the item, and avoid potential NPE in the following code. + if updatedObj == nil { + log.Infof("Remove item from the backup's backupItems list and totalItems list because it's skipped by plugin action.") + ib.backupRequest.BackedUpItems.DeleteItem(key) + return false, itemFiles, nil + } + itemFiles = append(itemFiles, additionalItemFiles...) obj = updatedObj if metadata, err = meta.Accessor(obj); err != nil { @@ -398,6 +406,13 @@ func (ib *itemBackupper) executeActions( } u := &unstructured.Unstructured{Object: updatedItem.UnstructuredContent()} + + if _, ok := u.GetAnnotations()[velerov1api.SkipFromBackupAnnotation]; ok { + log.Infof("Resource (groupResource=%s, namespace=%s, name=%s) is skipped from backup by action %s.", + groupResource.String(), namespace, name, actionName) + return nil, itemFiles, nil + } + if actionName == csiBIAPluginName { if additionalItemIdentifiers == nil && u.GetAnnotations()[velerov1api.SkippedNoCSIPVAnnotation] == "true" { // snapshot was skipped by CSI plugin @@ -687,15 +702,14 @@ func (ib *itemBackupper) getMatchAction(obj runtime.Unstructured, groupResource return nil, errors.WithStack(err) } - pvName := pvc.Spec.VolumeName - if pvName == "" { - return nil, errors.Errorf("PVC has no volume backing this claim") - } - - pv := &corev1api.PersistentVolume{} - if err := ib.kbClient.Get(context.Background(), kbClient.ObjectKey{Name: pvName}, pv); err != nil { - return nil, errors.WithStack(err) + var pv *corev1api.PersistentVolume + if pvName := pvc.Spec.VolumeName; pvName != "" { + pv = &corev1api.PersistentVolume{} + if err := ib.kbClient.Get(context.Background(), kbClient.ObjectKey{Name: pvName}, pv); err != nil { + return nil, errors.WithStack(err) + } } + // If pv is nil for unbound PVCs - policy matching will use PVC-only conditions vfd := resourcepolicies.NewVolumeFilterData(pv, nil, pvc) return ib.backupRequest.ResPolicies.GetMatchAction(vfd) } @@ -709,7 +723,10 @@ func (ib *itemBackupper) trackSkippedPV(obj runtime.Unstructured, groupResource if name, err := getPVName(obj, groupResource); len(name) > 0 && err == nil { ib.backupRequest.SkippedPVTracker.Track(name, approach, reason) } else if err != nil { - log.WithError(err).Warnf("unable to get PV name, skip tracking.") + // Log at info level for tracking purposes. This is not an error because + // it's expected for some resources (e.g., PVCs in Pending or Lost phase) + // to not have a PV name. This occurs when volume policy skips unbound PVCs. + log.WithError(err).Infof("unable to get PV name, skip tracking.") } } @@ -719,6 +736,17 @@ func (ib *itemBackupper) unTrackSkippedPV(obj runtime.Unstructured, groupResourc if name, err := getPVName(obj, groupResource); len(name) > 0 && err == nil { ib.backupRequest.SkippedPVTracker.Untrack(name) } else if err != nil { + // For PVCs in Pending or Lost phase, it's expected that there's no PV name. + // Log at debug level instead of warning to reduce noise. + if groupResource == kuberesource.PersistentVolumeClaims { + pvc := new(corev1api.PersistentVolumeClaim) + if convErr := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), pvc); convErr == nil { + if pvc.Status.Phase == corev1api.ClaimPending || pvc.Status.Phase == corev1api.ClaimLost { + log.WithError(err).Debugf("unable to get PV name for %s PVC, skip untracking.", pvc.Status.Phase) + return + } + } + } log.WithError(err).Warnf("unable to get PV name, skip untracking.") } } diff --git a/pkg/backup/item_backupper_test.go b/pkg/backup/item_backupper_test.go index b76536baa..be91b6d34 100644 --- a/pkg/backup/item_backupper_test.go +++ b/pkg/backup/item_backupper_test.go @@ -17,12 +17,15 @@ limitations under the License. package backup import ( + "bytes" "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/runtime/schema" + ctrlfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/stretchr/testify/assert" @@ -269,3 +272,225 @@ func TestAddVolumeInfo(t *testing.T) { }) } } + +func TestGetMatchAction_PendingLostPVC(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, corev1api.AddToScheme(scheme)) + + // Create resource policies that skip Pending/Lost PVCs + resPolicies := &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "pvcPhase": []string{"Pending", "Lost"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + } + policies := &resourcepolicies.Policies{} + err := policies.BuildPolicy(resPolicies) + require.NoError(t, err) + + testCases := []struct { + name string + pvc *corev1api.PersistentVolumeClaim + pv *corev1api.PersistentVolume + expectedAction *resourcepolicies.Action + expectError bool + }{ + { + name: "Pending PVC with no VolumeName should match pvcPhase policy", + pvc: builder.ForPersistentVolumeClaim("ns", "pending-pvc"). + StorageClass("test-sc"). + Phase(corev1api.ClaimPending). + Result(), + pv: nil, + expectedAction: &resourcepolicies.Action{Type: resourcepolicies.Skip}, + expectError: false, + }, + { + name: "Lost PVC with no VolumeName should match pvcPhase policy", + pvc: builder.ForPersistentVolumeClaim("ns", "lost-pvc"). + StorageClass("test-sc"). + Phase(corev1api.ClaimLost). + Result(), + pv: nil, + expectedAction: &resourcepolicies.Action{Type: resourcepolicies.Skip}, + expectError: false, + }, + { + name: "Bound PVC with VolumeName and matching PV should not match pvcPhase policy", + pvc: builder.ForPersistentVolumeClaim("ns", "bound-pvc"). + StorageClass("test-sc"). + VolumeName("test-pv"). + Phase(corev1api.ClaimBound). + Result(), + pv: builder.ForPersistentVolume("test-pv").StorageClass("test-sc").Result(), + expectedAction: nil, + expectError: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Build fake client with PV if present + clientBuilder := ctrlfake.NewClientBuilder().WithScheme(scheme) + if tc.pv != nil { + clientBuilder = clientBuilder.WithObjects(tc.pv) + } + fakeClient := clientBuilder.Build() + + ib := &itemBackupper{ + kbClient: fakeClient, + backupRequest: &Request{ + ResPolicies: policies, + }, + } + + // Convert PVC to unstructured + pvcData, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pvc) + require.NoError(t, err) + obj := &unstructured.Unstructured{Object: pvcData} + + action, err := ib.getMatchAction(obj, kuberesource.PersistentVolumeClaims, csiBIAPluginName) + if tc.expectError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + if tc.expectedAction == nil { + assert.Nil(t, action) + } else { + require.NotNil(t, action) + assert.Equal(t, tc.expectedAction.Type, action.Type) + } + }) + } +} + +func TestTrackSkippedPV_PendingLostPVC(t *testing.T) { + testCases := []struct { + name string + pvc *corev1api.PersistentVolumeClaim + }{ + { + name: "Pending PVC should log at info level", + pvc: builder.ForPersistentVolumeClaim("ns", "pending-pvc"). + Phase(corev1api.ClaimPending). + Result(), + }, + { + name: "Lost PVC should log at info level", + pvc: builder.ForPersistentVolumeClaim("ns", "lost-pvc"). + Phase(corev1api.ClaimLost). + Result(), + }, + { + name: "Bound PVC without VolumeName should log at info level", + pvc: builder.ForPersistentVolumeClaim("ns", "bound-pvc"). + Phase(corev1api.ClaimBound). + Result(), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ib := &itemBackupper{ + backupRequest: &Request{ + SkippedPVTracker: NewSkipPVTracker(), + }, + } + + // Set up log capture + logOutput := &bytes.Buffer{} + logger := logrus.New() + logger.SetOutput(logOutput) + logger.SetLevel(logrus.DebugLevel) + + // Convert PVC to unstructured + pvcData, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pvc) + require.NoError(t, err) + obj := &unstructured.Unstructured{Object: pvcData} + + ib.trackSkippedPV(obj, kuberesource.PersistentVolumeClaims, "", "test reason", logger) + + logStr := logOutput.String() + assert.Contains(t, logStr, "level=info") + assert.Contains(t, logStr, "unable to get PV name, skip tracking.") + }) + } +} + +func TestUnTrackSkippedPV_PendingLostPVC(t *testing.T) { + testCases := []struct { + name string + pvc *corev1api.PersistentVolumeClaim + expectWarningLog bool + expectDebugMessage string + }{ + { + name: "Pending PVC should log at debug level, not warning", + pvc: builder.ForPersistentVolumeClaim("ns", "pending-pvc"). + Phase(corev1api.ClaimPending). + Result(), + expectWarningLog: false, + expectDebugMessage: "unable to get PV name for Pending PVC, skip untracking.", + }, + { + name: "Lost PVC should log at debug level, not warning", + pvc: builder.ForPersistentVolumeClaim("ns", "lost-pvc"). + Phase(corev1api.ClaimLost). + Result(), + expectWarningLog: false, + expectDebugMessage: "unable to get PV name for Lost PVC, skip untracking.", + }, + { + name: "Bound PVC without VolumeName should log warning", + pvc: builder.ForPersistentVolumeClaim("ns", "bound-pvc"). + Phase(corev1api.ClaimBound). + Result(), + expectWarningLog: true, + expectDebugMessage: "", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ib := &itemBackupper{ + backupRequest: &Request{ + SkippedPVTracker: NewSkipPVTracker(), + }, + } + + // Set up log capture + logOutput := &bytes.Buffer{} + logger := logrus.New() + logger.SetOutput(logOutput) + logger.SetLevel(logrus.DebugLevel) + + // Convert PVC to unstructured + pvcData, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pvc) + require.NoError(t, err) + obj := &unstructured.Unstructured{Object: pvcData} + + ib.unTrackSkippedPV(obj, kuberesource.PersistentVolumeClaims, logger) + + logStr := logOutput.String() + if tc.expectWarningLog { + assert.Contains(t, logStr, "level=warning") + assert.Contains(t, logStr, "unable to get PV name, skip untracking.") + } else { + assert.NotContains(t, logStr, "level=warning") + if tc.expectDebugMessage != "" { + assert.Contains(t, logStr, "level=debug") + assert.Contains(t, logStr, tc.expectDebugMessage) + } + } + }) + } +} diff --git a/pkg/client/factory.go b/pkg/client/factory.go index 4b3c8941e..51dfb62c3 100644 --- a/pkg/client/factory.go +++ b/pkg/client/factory.go @@ -19,7 +19,7 @@ package client import ( "os" - volumegroupsnapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta1" + volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" @@ -168,7 +168,7 @@ func (f *factory) KubebuilderClient() (kbclient.Client, error) { if err := snapshotv1api.AddToScheme(scheme); err != nil { return nil, err } - if err := volumegroupsnapshotv1beta1.AddToScheme(scheme); err != nil { + if err := volumegroupsnapshotv1beta2.AddToScheme(scheme); err != nil { return nil, err } kubebuilderClient, err := kbclient.New(clientConfig, kbclient.Options{ @@ -207,7 +207,7 @@ func (f *factory) KubebuilderWatchClient() (kbclient.WithWatch, error) { if err := snapshotv1api.AddToScheme(scheme); err != nil { return nil, err } - if err := volumegroupsnapshotv1beta1.AddToScheme(scheme); err != nil { + if err := volumegroupsnapshotv1beta2.AddToScheme(scheme); err != nil { return nil, err } kubebuilderWatchClient, err := kbclient.NewWithWatch(clientConfig, kbclient.Options{ diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 13de4d552..8115e1353 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -275,11 +275,21 @@ func (o *Options) AsVeleroOptions() (*install.VeleroOptions, error) { return nil, err } } - veleroPodResources, err := kubeutil.ParseResourceRequirements(o.VeleroPodCPURequest, o.VeleroPodMemRequest, o.VeleroPodCPULimit, o.VeleroPodMemLimit) + veleroPodResources, err := kubeutil.ParseCPUAndMemoryResources( + o.VeleroPodCPURequest, + o.VeleroPodMemRequest, + o.VeleroPodCPULimit, + o.VeleroPodMemLimit, + ) if err != nil { return nil, err } - nodeAgentPodResources, err := kubeutil.ParseResourceRequirements(o.NodeAgentPodCPURequest, o.NodeAgentPodMemRequest, o.NodeAgentPodCPULimit, o.NodeAgentPodMemLimit) + nodeAgentPodResources, err := kubeutil.ParseCPUAndMemoryResources( + o.NodeAgentPodCPURequest, + o.NodeAgentPodMemRequest, + o.NodeAgentPodCPULimit, + o.NodeAgentPodMemLimit, + ) if err != nil { return nil, err } @@ -371,8 +381,8 @@ This is useful as a starting point for more customized installations. # velero install --provider azure --plugins velero/velero-plugin-for-microsoft-azure:v1.0.0 --bucket $BLOB_CONTAINER --secret-file ./credentials-velero --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID]`, Run: func(c *cobra.Command, args []string) { - cmd.CheckError(o.Validate(c, args, f)) cmd.CheckError(o.Complete(args, f)) + cmd.CheckError(o.Validate(c, args, f)) cmd.CheckError(o.Run(c, f)) }, } diff --git a/pkg/cmd/cli/install/install_test.go b/pkg/cmd/cli/install/install_test.go index c5d147646..1e0e7a4cf 100644 --- a/pkg/cmd/cli/install/install_test.go +++ b/pkg/cmd/cli/install/install_test.go @@ -17,11 +17,18 @@ limitations under the License. package install import ( + "context" "testing" + "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + corev1api "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks" + velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestPriorityClassNameFlag(t *testing.T) { @@ -91,3 +98,168 @@ func TestPriorityClassNameFlag(t *testing.T) { }) } } + +// makeValidateCmd returns a minimal *cobra.Command that satisfies output.ValidateFlags. +func makeValidateCmd() *cobra.Command { + c := &cobra.Command{} + // output.ValidateFlags only inspects the "output" flag; add it so validation passes. + c.Flags().StringP("output", "o", "", "output format") + return c +} + +// configMapInNamespace builds a ConfigMap with a single JSON data entry in the given namespace. +func configMapInNamespace(namespace, name, jsonValue string) *corev1api.ConfigMap { + return &corev1api.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Data: map[string]string{ + "config": jsonValue, + }, + } +} + +// TestValidateConfigMapsUseFactoryNamespace verifies that Validate resolves the target +// namespace correctly for all three ConfigMap flags. +// +// The fix (Option B) calls Complete before Validate in NewCommand so that o.Namespace is +// populated from f.Namespace() before VerifyJSONConfigs runs. Tests mirror that order by +// calling Complete before Validate. +func TestValidateConfigMapsUseFactoryNamespace(t *testing.T) { + const targetNS = "tenant-b" + const defaultNS = "default" + + // Shared options that satisfy every other validation gate: + // - NoDefaultBackupLocation=true + UseVolumeSnapshots=false skips provider/bucket/plugins checks + // - NoSecret=true satisfies the secret-file check + baseOptions := func() *Options { + o := NewInstallOptions() + o.NoDefaultBackupLocation = true + o.UseVolumeSnapshots = false + o.NoSecret = true + return o + } + + tests := []struct { + name string + setupOpts func(o *Options, cmName string) + cmJSON string + wantErrMsg string // substring expected in error; empty means success + }{ + { + name: "NodeAgentConfigMap found in factory namespace", + setupOpts: func(o *Options, cmName string) { + o.NodeAgentConfigMap = cmName + }, + cmJSON: `{}`, + }, + { + name: "NodeAgentConfigMap not found when only in default namespace", + setupOpts: func(o *Options, cmName string) { + o.NodeAgentConfigMap = cmName + }, + cmJSON: `{}`, + wantErrMsg: "--node-agent-configmap specified ConfigMap", + }, + { + name: "RepoMaintenanceJobConfigMap found in factory namespace", + setupOpts: func(o *Options, cmName string) { + o.RepoMaintenanceJobConfigMap = cmName + }, + cmJSON: `{}`, + }, + { + name: "RepoMaintenanceJobConfigMap not found when only in default namespace", + setupOpts: func(o *Options, cmName string) { + o.RepoMaintenanceJobConfigMap = cmName + }, + cmJSON: `{}`, + wantErrMsg: "--repo-maintenance-job-configmap specified ConfigMap", + }, + { + name: "BackupRepoConfigMap found in factory namespace", + setupOpts: func(o *Options, cmName string) { + o.BackupRepoConfigMap = cmName + }, + cmJSON: `{}`, + }, + { + name: "BackupRepoConfigMap not found when only in default namespace", + setupOpts: func(o *Options, cmName string) { + o.BackupRepoConfigMap = cmName + }, + cmJSON: `{}`, + wantErrMsg: "--backup-repository-configmap specified ConfigMap", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + const cmName = "my-config" + + // Decide where to place the ConfigMap: + // "not found" cases put it in "default", so the factory namespace lookup misses it. + cmNamespace := targetNS + if tc.wantErrMsg != "" { + cmNamespace = defaultNS + } + + cm := configMapInNamespace(cmNamespace, cmName, tc.cmJSON) + kbClient := velerotest.NewFakeControllerRuntimeClient(t, cm) + + f := &factorymocks.Factory{} + f.On("Namespace").Return(targetNS) + f.On("KubebuilderClient").Return(kbClient, nil) + + o := baseOptions() + tc.setupOpts(o, cmName) + + // Mirror the NewCommand call order: Complete populates o.Namespace before Validate runs. + require.NoError(t, o.Complete([]string{}, f)) + + c := makeValidateCmd() + c.SetContext(context.Background()) + + err := o.Validate(c, []string{}, f) + + if tc.wantErrMsg == "" { + require.NoError(t, err) + } else { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErrMsg) + } + }) + } +} + +// TestNewCommandRunClosureOrder covers the Run closure in NewCommand (the lines that were +// reordered by the fix: Complete → Validate → Run). +// +// The closure uses CheckError which calls os.Exit on any error, so the only safe path is one +// where all three steps return nil. DryRun=true causes o.Run to return after PrintWithFormat +// (which is a no-op when no --output flag is set) without touching any cluster clients. +func TestNewCommandRunClosureOrder(t *testing.T) { + const targetNS = "tenant-b" + const cmName = "my-config" + + cm := configMapInNamespace(targetNS, cmName, `{}`) + kbClient := velerotest.NewFakeControllerRuntimeClient(t, cm) + + f := &factorymocks.Factory{} + f.On("Namespace").Return(targetNS) + f.On("KubebuilderClient").Return(kbClient, nil) + + c := NewCommand(f) + c.SetArgs([]string{ + "--no-default-backup-location", + "--use-volume-snapshots=false", + "--no-secret", + "--dry-run", + "--node-agent-configmap", cmName, + }) + + // Execute drives the full Run closure: Complete populates o.Namespace, Validate + // looks up the ConfigMap in targetNS (succeeds), Run returns early via DryRun. + require.NoError(t, c.Execute()) +} diff --git a/pkg/cmd/cli/nodeagent/server.go b/pkg/cmd/cli/nodeagent/server.go index 74e003572..7e7c86e6c 100644 --- a/pkg/cmd/cli/nodeagent/server.go +++ b/pkg/cmd/cli/nodeagent/server.go @@ -323,7 +323,25 @@ func (s *nodeAgentServer) run() { podResources := corev1api.ResourceRequirements{} if s.dataPathConfigs != nil && s.dataPathConfigs.PodResources != nil { - if res, err := kube.ParseResourceRequirements(s.dataPathConfigs.PodResources.CPURequest, s.dataPathConfigs.PodResources.MemoryRequest, s.dataPathConfigs.PodResources.CPULimit, s.dataPathConfigs.PodResources.MemoryLimit); err != nil { + // To make the PodResources ConfigMap without ephemeral storage request/limit backward compatible, + // need to avoid set value as empty, because empty string will cause parsing error. + ephemeralStorageRequest := constant.DefaultEphemeralStorageRequest + if s.dataPathConfigs.PodResources.EphemeralStorageRequest != "" { + ephemeralStorageRequest = s.dataPathConfigs.PodResources.EphemeralStorageRequest + } + ephemeralStorageLimit := constant.DefaultEphemeralStorageLimit + if s.dataPathConfigs.PodResources.EphemeralStorageLimit != "" { + ephemeralStorageLimit = s.dataPathConfigs.PodResources.EphemeralStorageLimit + } + + if res, err := kube.ParseResourceRequirements( + s.dataPathConfigs.PodResources.CPURequest, + s.dataPathConfigs.PodResources.MemoryRequest, + ephemeralStorageRequest, + s.dataPathConfigs.PodResources.CPULimit, + s.dataPathConfigs.PodResources.MemoryLimit, + ephemeralStorageLimit, + ); err != nil { s.logger.WithError(err).Warn("Pod resource requirements are invalid, ignore") } else { podResources = res diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index e744013e8..cb38a40d0 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -27,7 +27,7 @@ import ( "time" logrusr "github.com/bombsimon/logrusr/v3" - volumegroupsnapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta1" + volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -247,7 +247,7 @@ func newServer(f client.Factory, config *config.Config, logger *logrus.Logger) ( cancelFunc() return nil, err } - if err := volumegroupsnapshotv1beta1.AddToScheme(scheme); err != nil { + if err := volumegroupsnapshotv1beta2.AddToScheme(scheme); err != nil { cancelFunc() return nil, err } diff --git a/pkg/cmd/server/server_test.go b/pkg/cmd/server/server_test.go index 1ea9d0022..c602f7c9e 100644 --- a/pkg/cmd/server/server_test.go +++ b/pkg/cmd/server/server_test.go @@ -204,9 +204,9 @@ func Test_newServer(t *testing.T) { }, logger) require.Error(t, err) - // invalid clientQPS Restic uploader + // invalid clientQPS Kopia uploader _, err = newServer(factory, &config.Config{ - UploaderType: uploader.ResticType, + UploaderType: uploader.KopiaType, ClientQPS: -1, }, logger) require.Error(t, err) diff --git a/pkg/constant/constant.go b/pkg/constant/constant.go index 8eac99573..f6ff09ae5 100644 --- a/pkg/constant/constant.go +++ b/pkg/constant/constant.go @@ -23,4 +23,7 @@ const ( PluginCSIPVCRestoreRIA = "velero.io/csi-pvc-restorer" PluginCsiVolumeSnapshotRestoreRIA = "velero.io/csi-volumesnapshot-restorer" + + DefaultEphemeralStorageRequest = "0" + DefaultEphemeralStorageLimit = "0" ) diff --git a/pkg/controller/backup_deletion_controller_test.go b/pkg/controller/backup_deletion_controller_test.go index ab3687438..58d9b0420 100644 --- a/pkg/controller/backup_deletion_controller_test.go +++ b/pkg/controller/backup_deletion_controller_test.go @@ -821,12 +821,12 @@ func TestGetSnapshotsInBackup(t *testing.T) { { VolumeNamespace: "ns-1", SnapshotID: "snap-3", - RepositoryType: "restic", + RepositoryType: "kopia", }, { VolumeNamespace: "ns-1", SnapshotID: "snap-4", - RepositoryType: "restic", + RepositoryType: "kopia", }, }, }, @@ -876,7 +876,7 @@ func TestGetSnapshotsInBackup(t *testing.T) { { VolumeNamespace: "ns-1", SnapshotID: "snap-3", - RepositoryType: "restic", + RepositoryType: "kopia", }, }, }, diff --git a/pkg/controller/backup_repository_controller.go b/pkg/controller/backup_repository_controller.go index 16e0b740a..11ebb5aec 100644 --- a/pkg/controller/backup_repository_controller.go +++ b/pkg/controller/backup_repository_controller.go @@ -43,7 +43,6 @@ import ( "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/metrics" - repoconfig "github.com/vmware-tanzu/velero/pkg/repository/config" "github.com/vmware-tanzu/velero/pkg/repository/maintenance" repomanager "github.com/vmware-tanzu/velero/pkg/repository/manager" "github.com/vmware-tanzu/velero/pkg/util/kube" @@ -53,9 +52,11 @@ import ( const ( repoSyncPeriod = 5 * time.Minute defaultMaintainFrequency = 7 * 24 * time.Hour - defaultMaintenanceStatusQueueLength = 3 + defaultMaintenanceStatusQueueLength = 25 ) +var maintenanceStatusQueueLength = defaultMaintenanceStatusQueueLength + type BackupRepoReconciler struct { client.Client namespace string @@ -238,6 +239,10 @@ func (r *BackupRepoReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } + if backupRepo.Spec.RepositoryType != velerov1api.BackupRepositoryTypeKopia { + return ctrl.Result{}, nil + } + bsl, bslErr := r.getBSL(ctx, backupRepo) if bslErr != nil { log.WithError(bslErr).Error("Fail to get BSL for BackupRepository. Skip reconciling.") @@ -245,7 +250,7 @@ func (r *BackupRepoReconciler) Reconcile(ctx context.Context, req ctrl.Request) } if backupRepo.Status.Phase == "" || backupRepo.Status.Phase == velerov1api.BackupRepositoryPhaseNew { - if err := r.initializeRepo(ctx, backupRepo, bsl, log); err != nil { + if err := r.initializeRepo(ctx, backupRepo, log); err != nil { log.WithError(err).Error("error initialize repository") return ctrl.Result{}, errors.WithStack(err) } @@ -263,7 +268,7 @@ func (r *BackupRepoReconciler) Reconcile(ctx context.Context, req ctrl.Request) switch backupRepo.Status.Phase { case velerov1api.BackupRepositoryPhaseNotReady: - ready, err := r.checkNotReadyRepo(ctx, backupRepo, bsl, log) + ready, err := r.checkNotReadyRepo(ctx, backupRepo, log) if err != nil { return ctrl.Result{}, err } else if !ready { @@ -311,35 +316,9 @@ func (r *BackupRepoReconciler) getBSL(ctx context.Context, req *velerov1api.Back return loc, nil } -func (r *BackupRepoReconciler) getIdentifierByBSL(bsl *velerov1api.BackupStorageLocation, req *velerov1api.BackupRepository) (string, error) { - repoIdentifier, err := repoconfig.GetRepoIdentifier(bsl, req.Spec.VolumeNamespace) - if err != nil { - return "", errors.Wrapf(err, "error to get identifier for repo %s", req.Name) - } - - return repoIdentifier, nil -} - -func (r *BackupRepoReconciler) initializeRepo(ctx context.Context, req *velerov1api.BackupRepository, bsl *velerov1api.BackupStorageLocation, log logrus.FieldLogger) error { +func (r *BackupRepoReconciler) initializeRepo(ctx context.Context, req *velerov1api.BackupRepository, log logrus.FieldLogger) error { log.WithField("repoConfig", r.backupRepoConfig).Info("Initializing backup repository") - var repoIdentifier string - // Only get restic identifier for restic repositories - if req.Spec.RepositoryType == "" || req.Spec.RepositoryType == velerov1api.BackupRepositoryTypeRestic { - var err error - repoIdentifier, err = r.getIdentifierByBSL(bsl, req) - if err != nil { - return r.patchBackupRepository(ctx, req, func(rr *velerov1api.BackupRepository) { - rr.Status.Message = err.Error() - rr.Status.Phase = velerov1api.BackupRepositoryPhaseNotReady - - if rr.Spec.MaintenanceFrequency.Duration <= 0 { - rr.Spec.MaintenanceFrequency = metav1.Duration{Duration: r.getRepositoryMaintenanceFrequency(req)} - } - }) - } - } - config, err := getBackupRepositoryConfig(ctx, r, r.backupRepoConfig, r.namespace, req.Name, req.Spec.RepositoryType, log) if err != nil { log.WithError(err).Warn("Failed to get repo config, repo config is ignored") @@ -349,11 +328,6 @@ func (r *BackupRepoReconciler) initializeRepo(ctx context.Context, req *velerov1 // defaulting - if the patch fails, return an error so the item is returned to the queue if err := r.patchBackupRepository(ctx, req, func(rr *velerov1api.BackupRepository) { - // Only set ResticIdentifier for restic repositories - if rr.Spec.RepositoryType == "" || rr.Spec.RepositoryType == velerov1api.BackupRepositoryTypeRestic { - rr.Spec.ResticIdentifier = repoIdentifier - } - if rr.Spec.MaintenanceFrequency.Duration <= 0 { rr.Spec.MaintenanceFrequency = metav1.Duration{Duration: r.getRepositoryMaintenanceFrequency(req)} } @@ -397,7 +371,7 @@ func ensureRepo(repo *velerov1api.BackupRepository, repoManager repomanager.Mana } func (r *BackupRepoReconciler) recallMaintenance(ctx context.Context, req *velerov1api.BackupRepository, log logrus.FieldLogger) error { - history, err := maintenance.WaitAllJobsComplete(ctx, r.Client, req, defaultMaintenanceStatusQueueLength, log) + history, err := maintenance.WaitAllJobsComplete(ctx, r.Client, req, maintenanceStatusQueueLength, log) if err != nil { return errors.Wrapf(err, "error waiting incomplete repo maintenance job for repo %s", req.Name) } @@ -455,7 +429,7 @@ func consolidateHistory(coming, cur []velerov1api.BackupRepositoryMaintenanceSta truncated := []velerov1api.BackupRepositoryMaintenanceStatus{} for consolidator.Len() > 0 { - if len(truncated) == defaultMaintenanceStatusQueueLength { + if len(truncated) == maintenanceStatusQueueLength { break } @@ -565,8 +539,8 @@ func updateRepoMaintenanceHistory(repo *velerov1api.BackupRepository, result vel } startingPos := 0 - if len(repo.Status.RecentMaintenance) >= defaultMaintenanceStatusQueueLength { - startingPos = len(repo.Status.RecentMaintenance) - defaultMaintenanceStatusQueueLength + 1 + if len(repo.Status.RecentMaintenance) >= maintenanceStatusQueueLength { + startingPos = len(repo.Status.RecentMaintenance) - maintenanceStatusQueueLength + 1 } repo.Status.RecentMaintenance = append(repo.Status.RecentMaintenance[startingPos:], latest) @@ -576,25 +550,9 @@ func dueForMaintenance(req *velerov1api.BackupRepository, now time.Time) bool { return req.Status.LastMaintenanceTime == nil || req.Status.LastMaintenanceTime.Add(req.Spec.MaintenanceFrequency.Duration).Before(now) } -func (r *BackupRepoReconciler) checkNotReadyRepo(ctx context.Context, req *velerov1api.BackupRepository, bsl *velerov1api.BackupStorageLocation, log logrus.FieldLogger) (bool, error) { +func (r *BackupRepoReconciler) checkNotReadyRepo(ctx context.Context, req *velerov1api.BackupRepository, log logrus.FieldLogger) (bool, error) { log.Info("Checking backup repository for readiness") - // Only check and update restic identifier for restic repositories - if req.Spec.RepositoryType == "" || req.Spec.RepositoryType == velerov1api.BackupRepositoryTypeRestic { - repoIdentifier, err := r.getIdentifierByBSL(bsl, req) - if err != nil { - return false, r.patchBackupRepository(ctx, req, repoNotReady(err.Error())) - } - - if repoIdentifier != req.Spec.ResticIdentifier { - if err := r.patchBackupRepository(ctx, req, func(rr *velerov1api.BackupRepository) { - rr.Spec.ResticIdentifier = repoIdentifier - }); err != nil { - return false, err - } - } - } - // we need to ensure it (first check, if check fails, attempt to init) // because we don't know if it's been successfully initialized yet. if err := ensureRepo(req, r.repositoryManager); err != nil { diff --git a/pkg/controller/backup_repository_controller_test.go b/pkg/controller/backup_repository_controller_test.go index 81a973200..ed952f4ae 100644 --- a/pkg/controller/backup_repository_controller_test.go +++ b/pkg/controller/backup_repository_controller_test.go @@ -98,32 +98,6 @@ func TestPatchBackupRepository(t *testing.T) { } func TestCheckNotReadyRepo(t *testing.T) { - // Test for restic repository - t.Run("restic repository", func(t *testing.T) { - rr := mockBackupRepositoryCR() - rr.Spec.BackupStorageLocation = "default" - rr.Spec.ResticIdentifier = "fake-identifier" - rr.Spec.VolumeNamespace = "volume-ns-1" - rr.Spec.RepositoryType = velerov1api.BackupRepositoryTypeRestic - reconciler := mockBackupRepoReconciler(t, "PrepareRepo", rr, nil) - err := reconciler.Client.Create(t.Context(), rr) - require.NoError(t, err) - location := velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Config: map[string]string{"resticRepoPrefix": "s3:test.amazonaws.com/bucket/restic"}, - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: velerov1api.DefaultNamespace, - Name: rr.Spec.BackupStorageLocation, - }, - } - - _, err = reconciler.checkNotReadyRepo(t.Context(), rr, &location, reconciler.logger) - require.NoError(t, err) - assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase) - assert.Equal(t, "s3:test.amazonaws.com/bucket/restic/volume-ns-1", rr.Spec.ResticIdentifier) - }) - // Test for kopia repository t.Run("kopia repository", func(t *testing.T) { rr := mockBackupRepositoryCR() @@ -133,48 +107,13 @@ func TestCheckNotReadyRepo(t *testing.T) { reconciler := mockBackupRepoReconciler(t, "PrepareRepo", rr, nil) err := reconciler.Client.Create(t.Context(), rr) require.NoError(t, err) - location := velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Config: map[string]string{"resticRepoPrefix": "s3:test.amazonaws.com/bucket/restic"}, - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: velerov1api.DefaultNamespace, - Name: rr.Spec.BackupStorageLocation, - }, - } - _, err = reconciler.checkNotReadyRepo(t.Context(), rr, &location, reconciler.logger) + _, err = reconciler.checkNotReadyRepo(t.Context(), rr, reconciler.logger) require.NoError(t, err) assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase) // ResticIdentifier should remain empty for kopia assert.Empty(t, rr.Spec.ResticIdentifier) }) - - // Test for empty repository type (defaults to restic) - t.Run("empty repository type", func(t *testing.T) { - rr := mockBackupRepositoryCR() - rr.Spec.BackupStorageLocation = "default" - rr.Spec.ResticIdentifier = "fake-identifier" - rr.Spec.VolumeNamespace = "volume-ns-1" - // Deliberately leave RepositoryType empty - reconciler := mockBackupRepoReconciler(t, "PrepareRepo", rr, nil) - err := reconciler.Client.Create(t.Context(), rr) - require.NoError(t, err) - location := velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Config: map[string]string{"resticRepoPrefix": "s3:test.amazonaws.com/bucket/restic"}, - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: velerov1api.DefaultNamespace, - Name: rr.Spec.BackupStorageLocation, - }, - } - - _, err = reconciler.checkNotReadyRepo(t.Context(), rr, &location, reconciler.logger) - require.NoError(t, err) - assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase) - assert.Equal(t, "s3:test.amazonaws.com/bucket/restic/volume-ns-1", rr.Spec.ResticIdentifier) - }) } func startMaintenanceJobFail(client.Client, context.Context, *velerov1api.BackupRepository, string, logrus.Level, *logging.FormatFlag, logrus.FieldLogger) (string, error) { @@ -463,17 +402,8 @@ func TestInitializeRepo(t *testing.T) { reconciler := mockBackupRepoReconciler(t, "PrepareRepo", rr, nil) err := reconciler.Client.Create(t.Context(), rr) require.NoError(t, err) - location := velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Config: map[string]string{"resticRepoPrefix": "s3:test.amazonaws.com/bucket/restic"}, - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: velerov1api.DefaultNamespace, - Name: rr.Spec.BackupStorageLocation, - }, - } - err = reconciler.initializeRepo(t.Context(), rr, &location, reconciler.logger) + err = reconciler.initializeRepo(t.Context(), rr, reconciler.logger) require.NoError(t, err) assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase) } @@ -999,6 +929,8 @@ func TestUpdateRepoMaintenanceHistory(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + maintenanceStatusQueueLength = 3 + updateRepoMaintenanceHistory(test.backupRepo, test.result, &metav1.Time{Time: standardTime}, &metav1.Time{Time: standardTime.Add(time.Hour)}, "fake-message-0") for at := range test.backupRepo.Status.RecentMaintenance { @@ -1494,7 +1426,7 @@ func TestDeleteOldMaintenanceJobWithConfigMap(t *testing.T) { MaintenanceFrequency: metav1.Duration{Duration: testMaintenanceFrequency}, BackupStorageLocation: "default", VolumeNamespace: "test-ns", - RepositoryType: "restic", + RepositoryType: "kopia", }, Status: velerov1api.BackupRepositoryStatus{ Phase: velerov1api.BackupRepositoryPhaseReady, @@ -1531,7 +1463,7 @@ func TestDeleteOldMaintenanceJobWithConfigMap(t *testing.T) { MaintenanceFrequency: metav1.Duration{Duration: testMaintenanceFrequency}, BackupStorageLocation: "default", VolumeNamespace: "test-ns", - RepositoryType: "restic", + RepositoryType: "kopia", }, Status: velerov1api.BackupRepositoryStatus{ Phase: velerov1api.BackupRepositoryPhaseReady, @@ -1550,8 +1482,8 @@ func TestDeleteOldMaintenanceJobWithConfigMap(t *testing.T) { Name: "repo-maintenance-job-config", }, Data: map[string]string{ - "global": `{"keepLatestMaintenanceJobs": 5}`, - "test-ns-default-restic": `{"keepLatestMaintenanceJobs": 2}`, + "global": `{"keepLatestMaintenanceJobs": 5}`, + "test-ns-default-kopia": `{"keepLatestMaintenanceJobs": 2}`, }, }, }, @@ -1605,58 +1537,6 @@ func TestInitializeRepoWithRepositoryTypes(t *testing.T) { corev1api.AddToScheme(scheme) velerov1api.AddToScheme(scheme) - // Test for restic repository - t.Run("restic repository", func(t *testing.T) { - rr := mockBackupRepositoryCR() - rr.Spec.BackupStorageLocation = "default" - rr.Spec.VolumeNamespace = "volume-ns-1" - rr.Spec.RepositoryType = velerov1api.BackupRepositoryTypeRestic - - location := &velerov1api.BackupStorageLocation{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: velerov1api.DefaultNamespace, - Name: "default", - }, - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "aws", - StorageType: velerov1api.StorageType{ - ObjectStorage: &velerov1api.ObjectStorageLocation{ - Bucket: "test-bucket", - Prefix: "test-prefix", - }, - }, - Config: map[string]string{ - "region": "us-east-1", - }, - }, - } - - fakeClient := clientFake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(rr, location).Build() - mgr := &repomokes.Manager{} - mgr.On("PrepareRepo", rr).Return(nil) - - reconciler := NewBackupRepoReconciler( - velerov1api.DefaultNamespace, - velerotest.NewLogger(), - fakeClient, - mgr, - testMaintenanceFrequency, - "", - "", - logrus.InfoLevel, - nil, - nil, - ) - - err := reconciler.initializeRepo(t.Context(), rr, location, reconciler.logger) - require.NoError(t, err) - - // Verify ResticIdentifier is set for restic - assert.NotEmpty(t, rr.Spec.ResticIdentifier) - assert.Contains(t, rr.Spec.ResticIdentifier, "volume-ns-1") - assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase) - }) - // Test for kopia repository t.Run("kopia repository", func(t *testing.T) { rr := mockBackupRepositoryCR() @@ -1700,65 +1580,13 @@ func TestInitializeRepoWithRepositoryTypes(t *testing.T) { nil, ) - err := reconciler.initializeRepo(t.Context(), rr, location, reconciler.logger) + err := reconciler.initializeRepo(t.Context(), rr, reconciler.logger) require.NoError(t, err) // Verify ResticIdentifier is NOT set for kopia assert.Empty(t, rr.Spec.ResticIdentifier) assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase) }) - - // Test for empty repository type (defaults to restic) - t.Run("empty repository type", func(t *testing.T) { - rr := mockBackupRepositoryCR() - rr.Spec.BackupStorageLocation = "default" - rr.Spec.VolumeNamespace = "volume-ns-1" - // Leave RepositoryType empty - - location := &velerov1api.BackupStorageLocation{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: velerov1api.DefaultNamespace, - Name: "default", - }, - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "aws", - StorageType: velerov1api.StorageType{ - ObjectStorage: &velerov1api.ObjectStorageLocation{ - Bucket: "test-bucket", - Prefix: "test-prefix", - }, - }, - Config: map[string]string{ - "region": "us-east-1", - }, - }, - } - - fakeClient := clientFake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(rr, location).Build() - mgr := &repomokes.Manager{} - mgr.On("PrepareRepo", rr).Return(nil) - - reconciler := NewBackupRepoReconciler( - velerov1api.DefaultNamespace, - velerotest.NewLogger(), - fakeClient, - mgr, - testMaintenanceFrequency, - "", - "", - logrus.InfoLevel, - nil, - nil, - ) - - err := reconciler.initializeRepo(t.Context(), rr, location, reconciler.logger) - require.NoError(t, err) - - // Verify ResticIdentifier is set when type is empty (defaults to restic) - assert.NotEmpty(t, rr.Spec.ResticIdentifier) - assert.Contains(t, rr.Spec.ResticIdentifier, "volume-ns-1") - assert.Equal(t, velerov1api.BackupRepositoryPhaseReady, rr.Status.Phase) - }) } func TestRepoMaintenanceMetricsRecording(t *testing.T) { diff --git a/pkg/controller/pod_volume_restore_controller_legacy.go b/pkg/controller/pod_volume_restore_controller_legacy.go index 731b70db9..9ddececf5 100644 --- a/pkg/controller/pod_volume_restore_controller_legacy.go +++ b/pkg/controller/pod_volume_restore_controller_legacy.go @@ -360,5 +360,5 @@ func (c *PodVolumeRestoreReconcilerLegacy) closeDataPath(ctx context.Context, pv } func IsLegacyPVR(pvr *velerov1api.PodVolumeRestore) bool { - return pvr.Spec.UploaderType == uploader.ResticType + return pvr.Spec.UploaderType == "restic" } diff --git a/pkg/controller/restore_controller.go b/pkg/controller/restore_controller.go index 208a3ddca..469951f27 100644 --- a/pkg/controller/restore_controller.go +++ b/pkg/controller/restore_controller.go @@ -529,6 +529,7 @@ func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backu LabelSelector: labels.Set(map[string]string{ api.BackupNameLabel: label.GetValidName(restore.Spec.BackupName), }).AsSelector(), + Namespace: restore.Namespace, } podVolumeBackupList := &api.PodVolumeBackupList{} diff --git a/pkg/controller/restore_controller_test.go b/pkg/controller/restore_controller_test.go index 3acc03d2a..b013ee64d 100644 --- a/pkg/controller/restore_controller_test.go +++ b/pkg/controller/restore_controller_test.go @@ -238,6 +238,8 @@ func TestRestoreReconcile(t *testing.T) { expectedFinalPhase string addValidFinalizer bool emptyVolumeInfo bool + podVolumeBackups []*velerov1api.PodVolumeBackup + expectedPVBCount int }{ { name: "restore with both namespace in both includedNamespaces and excludedNamespaces fails validation", @@ -357,6 +359,22 @@ func TestRestoreReconcile(t *testing.T) { expectedCompletedTime: ×tamp, expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseInProgress).Result(), }, + { + name: "valid restore gets executed and only includes pod volume backups from restore namespace", + location: defaultStorageLocation, + restore: NewRestore("foo", "bar2", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(), + backup: defaultBackup().StorageLocation("default").Result(), + podVolumeBackups: []*velerov1api.PodVolumeBackup{ + builder.ForPodVolumeBackup("foo", "pvb-1").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), + builder.ForPodVolumeBackup("other-ns", "pvb-2").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), + }, + expectedPVBCount: 1, + expectedErr: false, + expectedPhase: string(velerov1api.RestorePhaseInProgress), + expectedStartTime: ×tamp, + expectedCompletedTime: ×tamp, + expectedRestorerCall: NewRestore("foo", "bar2", "backup-1", "ns-1", "", velerov1api.RestorePhaseInProgress).Result(), + }, { name: "restoration of nodes is not supported", location: defaultStorageLocation, @@ -501,6 +519,13 @@ func TestRestoreReconcile(t *testing.T) { defaultStorageLocation.ObjectMeta.ResourceVersion = "" }() + if test.podVolumeBackups != nil { + for _, pvb := range test.podVolumeBackups { + err := fakeClient.Create(t.Context(), pvb) + require.NoError(t, err) + } + } + r := NewRestoreReconciler( t.Context(), velerov1api.DefaultNamespace, @@ -670,6 +695,10 @@ func TestRestoreReconcile(t *testing.T) { // the mock stores the pointer, which gets modified after assert.Equal(t, test.expectedRestorerCall.Spec, restorer.calledWithArg.Spec) assert.Equal(t, test.expectedRestorerCall.Status.Phase, restorer.calledWithArg.Status.Phase) + + if test.podVolumeBackups != nil { + assert.Len(t, restorer.calledWithPVBs, test.expectedPVBCount) + } }) } } @@ -1021,8 +1050,9 @@ func NewRestore(ns, name, backup, includeNS, includeResource string, phase veler type fakeRestorer struct { mock.Mock - calledWithArg velerov1api.Restore - kbClient client.Client + calledWithArg velerov1api.Restore + calledWithPVBs []*velerov1api.PodVolumeBackup + kbClient client.Client } func (r *fakeRestorer) Restore( @@ -1045,6 +1075,7 @@ func (r *fakeRestorer) RestoreWithResolvers(req *pkgrestore.Request, r.kbClient, volumeSnapshotterGetter) r.calledWithArg = *req.Restore + r.calledWithPVBs = req.PodVolumeBackups return res.Get(0).(results.Result), res.Get(1).(results.Result) } diff --git a/pkg/controller/restore_finalizer_controller.go b/pkg/controller/restore_finalizer_controller.go index 0061bdf45..f82216bc3 100644 --- a/pkg/controller/restore_finalizer_controller.go +++ b/pkg/controller/restore_finalizer_controller.go @@ -22,6 +22,8 @@ import ( "sync" "time" + volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" @@ -43,6 +45,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" "github.com/vmware-tanzu/velero/pkg/plugin/velero" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/results" ) @@ -291,13 +294,16 @@ type finalizerContext struct { resourceTimeout time.Duration } -func (ctx *finalizerContext) execute() (results.Result, results.Result) { //nolint:unparam //temporarily ignore the lint report: result 0 is always nil (unparam) +func (ctx *finalizerContext) execute() (results.Result, results.Result) { warnings, errs := results.Result{}, results.Result{} // implement finalization tasks pdpErrs := ctx.patchDynamicPVWithVolumeInfo() errs.Merge(&pdpErrs) + vgscWarnings := ctx.cleanupStubVGSC() + warnings.Merge(&vgscWarnings) + rehErrs := ctx.WaitRestoreExecHook() errs.Merge(&rehErrs) @@ -443,6 +449,93 @@ func (ctx *finalizerContext) patchDynamicPVWithVolumeInfo() (errs results.Result return errs } +// cleanupStubVGSC deletes stub VolumeGroupSnapshotContent objects that were +// created during restore to satisfy CSI controller validation. These stubs are +// labeled with velero.io/restore-name for identification. +// Before deleting each VGSC, it waits for all related VolumeSnapshotContents +// to become ReadyToUse, since the CSI controller needs the VGSC during VSC reconciliation. +func (ctx *finalizerContext) cleanupStubVGSC() (warnings results.Result) { + ctx.logger.Info("cleaning up stub VolumeGroupSnapshotContents") + + vgscList := &volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentList{} + err := ctx.crClient.List( + context.Background(), + vgscList, + client.MatchingLabels{velerov1api.RestoreNameLabel: ctx.restore.Name}, + ) + if err != nil { + // If the CRD is not installed, listing will fail. This is expected + // on clusters without VolumeGroupSnapshot support, so treat as warning. + ctx.logger.WithError(err).Warn("failed to list stub VolumeGroupSnapshotContents, skipping cleanup") + warnings.Add("cluster", errors.Wrap(err, "failed to list stub VolumeGroupSnapshotContents")) + return warnings + } + + if len(vgscList.Items) == 0 { + ctx.logger.Info("no stub VolumeGroupSnapshotContents to clean up") + return warnings + } + + for i := range vgscList.Items { + vgsc := &vgscList.Items[i] + log := ctx.logger.WithField("vgsc", vgsc.Name) + + // Collect the snapshot handles associated with this VGSC + snapshotHandles := map[string]bool{} + if vgsc.Spec.Source.GroupSnapshotHandles != nil { + for _, h := range vgsc.Spec.Source.GroupSnapshotHandles.VolumeSnapshotHandles { + snapshotHandles[h] = true + } + } + + if len(snapshotHandles) > 0 { + // Wait for related VSCs to become ReadyToUse before deleting the VGSC + log.Infof("waiting for %d related VolumeSnapshotContents to become ReadyToUse", len(snapshotHandles)) + err := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, ctx.resourceTimeout, true, func(context.Context) (bool, error) { + vscList := &snapshotv1api.VolumeSnapshotContentList{} + if err := ctx.crClient.List(context.Background(), vscList, client.MatchingLabels{velerov1api.RestoreNameLabel: ctx.restore.Name}); err != nil { + log.WithError(err).Warn("failed to list VolumeSnapshotContents") + return false, nil + } + + for j := range vscList.Items { + vsc := &vscList.Items[j] + if vsc.Spec.Source.SnapshotHandle == nil { + continue + } + if !snapshotHandles[*vsc.Spec.Source.SnapshotHandle] { + continue + } + // This VSC is related to our VGSC + if vsc.Status == nil || !boolptr.IsSetToTrue(vsc.Status.ReadyToUse) { + log.Debugf("VolumeSnapshotContent %s not yet ReadyToUse", vsc.Name) + return false, nil + } + } + return true, nil + }) + if err != nil { + log.WithError(err).Warn("timed out waiting for related VolumeSnapshotContents to become ReadyToUse, proceeding with VGSC deletion") + warnings.Add("cluster", errors.Wrapf(err, "timed out waiting for VSCs related to VGSC %s", vgsc.Name)) + } + } + + log.Info("deleting stub VolumeGroupSnapshotContent") + if err := ctx.crClient.Delete(context.Background(), vgsc); err != nil { + if apierrors.IsNotFound(err) { + log.Info("stub VolumeGroupSnapshotContent already deleted") + continue + } + log.WithError(err).Warn("failed to delete stub VolumeGroupSnapshotContent") + warnings.Add("cluster", errors.Wrapf(err, "failed to delete stub VolumeGroupSnapshotContent %s", vgsc.Name)) + } else { + log.Info("deleted stub VolumeGroupSnapshotContent") + } + } + + return warnings +} + func needPatch(newPV *corev1api.PersistentVolume, pvInfo *volume.PVInfo) bool { if newPV.Spec.PersistentVolumeReclaimPolicy != corev1api.PersistentVolumeReclaimPolicy(pvInfo.ReclaimPolicy) { return true diff --git a/pkg/controller/restore_finalizer_controller_test.go b/pkg/controller/restore_finalizer_controller_test.go index 56366a360..6fb5ba303 100644 --- a/pkg/controller/restore_finalizer_controller_test.go +++ b/pkg/controller/restore_finalizer_controller_test.go @@ -22,6 +22,8 @@ import ( "testing" "time" + volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -45,6 +47,7 @@ import ( pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" pkgUtilKubeMocks "github.com/vmware-tanzu/velero/pkg/util/kube/mocks" "github.com/vmware-tanzu/velero/pkg/util/results" ) @@ -739,3 +742,253 @@ func TestRestoreOperationList(t *testing.T) { }) } } + +func TestCleanupStubVGSC(t *testing.T) { + snapshotHandle1 := "snap-handle-1" + snapshotHandle2 := "snap-handle-2" + + tests := []struct { + name string + restore *velerov1api.Restore + existingVGSCs []*volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent + existingVSCs []*snapshotv1api.VolumeSnapshotContent + expectedRemaining int + expectedWarnings bool + }{ + { + name: "no stub VGSCs to clean up", + restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore-1").Result(), + existingVGSCs: nil, + expectedRemaining: 0, + expectedWarnings: false, + }, + { + name: "single stub VGSC deleted after VSCs are ready", + restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore-1").Result(), + existingVGSCs: []*volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "vgsc-stub-1", + Labels: map[string]string{ + velerov1api.RestoreNameLabel: "restore-1", + }, + }, + Spec: volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentSpec{ + Driver: "rbd.csi.ceph.com", + Source: volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentSource{ + GroupSnapshotHandles: &volumegroupsnapshotv1beta2.GroupSnapshotHandles{ + VolumeGroupSnapshotHandle: "vgs-handle-1", + VolumeSnapshotHandles: []string{snapshotHandle1}, + }, + }, + }, + }, + }, + existingVSCs: []*snapshotv1api.VolumeSnapshotContent{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-1", + Labels: map[string]string{ + velerov1api.RestoreNameLabel: "restore-1", + }, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + Driver: "rbd.csi.ceph.com", + DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, + Source: snapshotv1api.VolumeSnapshotContentSource{ + SnapshotHandle: &snapshotHandle1, + }, + VolumeSnapshotRef: corev1api.ObjectReference{ + Name: "vs-1", + Namespace: "ns-1", + }, + }, + Status: &snapshotv1api.VolumeSnapshotContentStatus{ + ReadyToUse: boolptr.True(), + }, + }, + }, + expectedRemaining: 0, + expectedWarnings: false, + }, + { + name: "multiple stub VGSCs deleted", + restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore-1").Result(), + existingVGSCs: []*volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "vgsc-stub-1", + Labels: map[string]string{ + velerov1api.RestoreNameLabel: "restore-1", + }, + }, + Spec: volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentSpec{ + Driver: "rbd.csi.ceph.com", + Source: volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentSource{ + GroupSnapshotHandles: &volumegroupsnapshotv1beta2.GroupSnapshotHandles{ + VolumeGroupSnapshotHandle: "vgs-handle-1", + VolumeSnapshotHandles: []string{snapshotHandle1}, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "vgsc-stub-2", + Labels: map[string]string{ + velerov1api.RestoreNameLabel: "restore-1", + }, + }, + Spec: volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentSpec{ + Driver: "rbd.csi.ceph.com", + Source: volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentSource{ + GroupSnapshotHandles: &volumegroupsnapshotv1beta2.GroupSnapshotHandles{ + VolumeGroupSnapshotHandle: "vgs-handle-2", + VolumeSnapshotHandles: []string{snapshotHandle2}, + }, + }, + }, + }, + }, + existingVSCs: []*snapshotv1api.VolumeSnapshotContent{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-1", + Labels: map[string]string{ + velerov1api.RestoreNameLabel: "restore-1", + }, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + Driver: "rbd.csi.ceph.com", + DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, + Source: snapshotv1api.VolumeSnapshotContentSource{ + SnapshotHandle: &snapshotHandle1, + }, + VolumeSnapshotRef: corev1api.ObjectReference{ + Name: "vs-1", + Namespace: "ns-1", + }, + }, + Status: &snapshotv1api.VolumeSnapshotContentStatus{ + ReadyToUse: boolptr.True(), + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-2", + Labels: map[string]string{ + velerov1api.RestoreNameLabel: "restore-1", + }, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + Driver: "rbd.csi.ceph.com", + DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, + Source: snapshotv1api.VolumeSnapshotContentSource{ + SnapshotHandle: &snapshotHandle2, + }, + VolumeSnapshotRef: corev1api.ObjectReference{ + Name: "vs-2", + Namespace: "ns-1", + }, + }, + Status: &snapshotv1api.VolumeSnapshotContentStatus{ + ReadyToUse: boolptr.True(), + }, + }, + }, + expectedRemaining: 0, + expectedWarnings: false, + }, + { + name: "VGSCs from different restore are not deleted", + restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore-1").Result(), + existingVGSCs: []*volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "vgsc-stub-mine", + Labels: map[string]string{ + velerov1api.RestoreNameLabel: "restore-1", + }, + }, + Spec: volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentSpec{ + Driver: "rbd.csi.ceph.com", + Source: volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentSource{}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "vgsc-stub-other", + Labels: map[string]string{ + velerov1api.RestoreNameLabel: "restore-2", + }, + }, + Spec: volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentSpec{ + Driver: "rbd.csi.ceph.com", + Source: volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentSource{}, + }, + }, + }, + expectedRemaining: 1, + expectedWarnings: false, + }, + { + name: "VGSC deleted even when no snapshot handles in spec", + restore: builder.ForRestore(velerov1api.DefaultNamespace, "restore-1").Result(), + existingVGSCs: []*volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "vgsc-stub-empty", + Labels: map[string]string{ + velerov1api.RestoreNameLabel: "restore-1", + }, + }, + Spec: volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentSpec{ + Driver: "rbd.csi.ceph.com", + Source: volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentSource{}, + }, + }, + }, + expectedRemaining: 0, + expectedWarnings: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fakeClient := velerotest.NewFakeControllerRuntimeClientBuilder(t).Build() + logger := velerotest.NewLogger() + + ctx := &finalizerContext{ + logger: logger, + crClient: fakeClient, + restore: tc.restore, + resourceTimeout: 10 * time.Second, + } + + for _, vgsc := range tc.existingVGSCs { + require.NoError(t, fakeClient.Create(t.Context(), vgsc)) + } + for _, vsc := range tc.existingVSCs { + require.NoError(t, fakeClient.Create(t.Context(), vsc)) + } + + warnings := ctx.cleanupStubVGSC() + + if tc.expectedWarnings { + assert.False(t, warnings.IsEmpty()) + } else { + assert.True(t, warnings.IsEmpty(), "expected no warnings") + } + + remainingList := &volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentList{} + require.NoError(t, fakeClient.List(t.Context(), remainingList)) + assert.Len(t, remainingList.Items, tc.expectedRemaining) + + // Verify remaining VGSCs don't belong to this restore + for _, remaining := range remainingList.Items { + assert.NotEqual(t, tc.restore.Name, remaining.Labels[velerov1api.RestoreNameLabel], + "VGSC %s should have been deleted", remaining.Name) + } + }) + } +} diff --git a/pkg/controller/schedule_controller.go b/pkg/controller/schedule_controller.go index ec8894571..443b3c08b 100644 --- a/pkg/controller/schedule_controller.go +++ b/pkg/controller/schedule_controller.go @@ -129,6 +129,13 @@ func (c *scheduleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c } else { schedule.Status.Phase = velerov1.SchedulePhaseEnabled schedule.Status.ValidationErrors = nil + + // Compute expected interval between consecutive scheduled backup runs. + // Only meaningful when the cron expression is valid. + now := c.clock.Now() + nextRun := cronSchedule.Next(now) + nextNextRun := cronSchedule.Next(nextRun) + c.metrics.SetScheduleExpectedIntervalSeconds(schedule.Name, nextNextRun.Sub(nextRun).Seconds()) } scheduleNeedsPatch := false diff --git a/pkg/datamover/backup_micro_service.go b/pkg/datamover/backup_micro_service.go index 3c6601cbf..61c308f15 100644 --- a/pkg/datamover/backup_micro_service.go +++ b/pkg/datamover/backup_micro_service.go @@ -310,6 +310,7 @@ func (r *BackupMicroService) cancelDataUpload(du *velerov2alpha1api.DataUpload) fsBackup := r.dataPathMgr.GetAsyncBR(du.Name) if fsBackup == nil { r.OnDataUploadCancelled(r.ctx, du.GetNamespace(), du.GetName()) + r.eventRecorder.EndingEvent(du, false, datapath.EventReasonStopped, "Data path for %s exited without start", du.Name) } else { fsBackup.Cancel() } diff --git a/pkg/datamover/backup_micro_service_test.go b/pkg/datamover/backup_micro_service_test.go index 160fcaac7..c5a9a4487 100644 --- a/pkg/datamover/backup_micro_service_test.go +++ b/pkg/datamover/backup_micro_service_test.go @@ -259,8 +259,8 @@ func TestCancelDataUpload(t *testing.T) { }{ { name: "no fs backup", - expectedEventReason: datapath.EventReasonCancelled, - expectedEventMsg: "Data path for data upload fake-data-upload canceled", + expectedEventReason: datapath.EventReasonStopped, + expectedEventMsg: "Data path for fake-data-upload exited without start", expectedErr: datapath.ErrCancelled, }, } diff --git a/pkg/datamover/restore_micro_service.go b/pkg/datamover/restore_micro_service.go index 1b7ebcc09..0cb5f18ec 100644 --- a/pkg/datamover/restore_micro_service.go +++ b/pkg/datamover/restore_micro_service.go @@ -288,6 +288,7 @@ func (r *RestoreMicroService) cancelDataDownload(dd *velerov2alpha1api.DataDownl fsBackup := r.dataPathMgr.GetAsyncBR(dd.Name) if fsBackup == nil { r.OnDataDownloadCancelled(r.ctx, dd.GetNamespace(), dd.GetName()) + r.eventRecorder.EndingEvent(dd, false, datapath.EventReasonStopped, "Data path for %s exited without start", dd.Name) } else { fsBackup.Cancel() } diff --git a/pkg/datamover/restore_micro_service_test.go b/pkg/datamover/restore_micro_service_test.go index f315ef639..a53e4d6d5 100644 --- a/pkg/datamover/restore_micro_service_test.go +++ b/pkg/datamover/restore_micro_service_test.go @@ -203,8 +203,8 @@ func TestCancelDataDownload(t *testing.T) { }{ { name: "no fs restore", - expectedEventReason: datapath.EventReasonCancelled, - expectedEventMsg: "Data path for data download fake-data-download canceled", + expectedEventReason: datapath.EventReasonStopped, + expectedEventMsg: "Data path for fake-data-download exited without start", expectedErr: datapath.ErrCancelled, }, } diff --git a/pkg/datapath/file_system.go b/pkg/datapath/file_system.go index f0f84acdb..61fac1e47 100644 --- a/pkg/datapath/file_system.go +++ b/pkg/datapath/file_system.go @@ -251,11 +251,9 @@ func (fs *fileSystemBR) boostRepoConnect(ctx context.Context, repositoryType str if err := repoProvider.NewUnifiedRepoProvider(*credentialGetter, repositoryType, fs.log).BoostRepoConnect(ctx, repoProvider.RepoParam{BackupLocation: fs.backupLocation, BackupRepo: fs.backupRepo, CacheDir: cacheDir}); err != nil { return err } - } else { - if err := repoProvider.NewResticRepositoryProvider(*credentialGetter, filesystem.NewFileSystem(), fs.log).BoostRepoConnect(ctx, repoProvider.RepoParam{BackupLocation: fs.backupLocation, BackupRepo: fs.backupRepo}); err != nil { - return err - } + + return nil } - return nil + return errors.Errorf("error getting provider for repo %s", repositoryType) } diff --git a/pkg/exposer/csi_snapshot.go b/pkg/exposer/csi_snapshot.go index 5acb229d2..079a4b527 100644 --- a/pkg/exposer/csi_snapshot.go +++ b/pkg/exposer/csi_snapshot.go @@ -124,6 +124,15 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O "owner": ownerObject.Name, }) + volumeTopology, err := kube.GetVolumeTopology(ctx, e.kubeClient.CoreV1(), e.kubeClient.StorageV1(), csiExposeParam.SourcePVName, csiExposeParam.StorageClass) + if err != nil { + return errors.Wrapf(err, "error getting volume topology for PV %s, storage class %s", csiExposeParam.SourcePVName, csiExposeParam.StorageClass) + } + + if volumeTopology != nil { + curLog.Infof("Using volume topology %v", volumeTopology) + } + curLog.Info("Exposing CSI snapshot") volumeSnapshot, err := csi.WaitVolumeSnapshotReady(ctx, e.csiSnapshotClient, csiExposeParam.SnapshotName, csiExposeParam.SourceNamespace, csiExposeParam.ExposeTimeout, curLog) @@ -254,6 +263,7 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O csiExposeParam.NodeOS, csiExposeParam.PriorityClassName, intoleratableNodes, + volumeTopology, ) if err != nil { return errors.Wrap(err, "error to create backup pod") @@ -320,7 +330,8 @@ func (e *csiSnapshotExposer) GetExposed(ctx context.Context, ownerObject corev1a curLog.WithField("pod", pod.Name).Infof("Backup volume is found in pod at index %v", i) var nodeOS *string - if os, found := pod.Spec.NodeSelector[kube.NodeOSLabel]; found { + if pod.Spec.OS != nil { + os := string(pod.Spec.OS.Name) nodeOS = &os } @@ -588,6 +599,7 @@ func (e *csiSnapshotExposer) createBackupPod( nodeOS string, priorityClassName string, intoleratableNodes []string, + volumeTopology *corev1api.NodeSelector, ) (*corev1api.Pod, error) { podName := ownerObject.Name @@ -643,6 +655,10 @@ func (e *csiSnapshotExposer) createBackupPod( args = append(args, podInfo.logFormatArgs...) args = append(args, podInfo.logLevelArgs...) + if affinity == nil { + affinity = &kube.LoadAffinity{} + } + var securityCtx *corev1api.PodSecurityContext nodeSelector := map[string]string{} podOS := corev1api.PodOS{} @@ -654,9 +670,14 @@ func (e *csiSnapshotExposer) createBackupPod( }, } - nodeSelector[kube.NodeOSLabel] = kube.NodeOSWindows podOS.Name = kube.NodeOSWindows + affinity.NodeSelector.MatchExpressions = append(affinity.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: kube.NodeOSLabel, + Values: []string{kube.NodeOSWindows}, + Operator: metav1.LabelSelectorOpIn, + }) + toleration = append(toleration, []corev1api.Toleration{ { Key: "os", @@ -683,11 +704,15 @@ func (e *csiSnapshotExposer) createBackupPod( } } - nodeSelector[kube.NodeOSLabel] = kube.NodeOSLinux podOS.Name = kube.NodeOSLinux + + affinity.NodeSelector.MatchExpressions = append(affinity.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: kube.NodeOSLabel, + Values: []string{kube.NodeOSWindows}, + Operator: metav1.LabelSelectorOpNotIn, + }) } - var podAffinity *corev1api.Affinity if len(intoleratableNodes) > 0 { if affinity == nil { affinity = &kube.LoadAffinity{} @@ -700,9 +725,7 @@ func (e *csiSnapshotExposer) createBackupPod( }) } - if affinity != nil { - podAffinity = kube.ToSystemAffinity([]*kube.LoadAffinity{affinity}) - } + podAffinity := kube.ToSystemAffinity(affinity, volumeTopology) pod := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/exposer/csi_snapshot_priority_test.go b/pkg/exposer/csi_snapshot_priority_test.go index 345d5b327..d1ffa4700 100644 --- a/pkg/exposer/csi_snapshot_priority_test.go +++ b/pkg/exposer/csi_snapshot_priority_test.go @@ -154,6 +154,7 @@ func TestCreateBackupPodWithPriorityClass(t *testing.T) { kube.NodeOSLinux, tc.expectedPriorityClass, nil, + nil, ) require.NoError(t, err, tc.description) @@ -239,6 +240,7 @@ func TestCreateBackupPodWithMissingConfigMap(t *testing.T) { kube.NodeOSLinux, "", // empty priority class since config map is missing nil, + nil, ) // Should succeed even when config map is missing diff --git a/pkg/exposer/csi_snapshot_test.go b/pkg/exposer/csi_snapshot_test.go index b4dd92c3f..e1a9860eb 100644 --- a/pkg/exposer/csi_snapshot_test.go +++ b/pkg/exposer/csi_snapshot_test.go @@ -68,6 +68,12 @@ func TestExpose(t *testing.T) { var restoreSize int64 = 123456 + scObj := &storagev1api.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-sc", + }, + } + snapshotClass := "fake-snapshot-class" vsObject := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ @@ -199,6 +205,18 @@ func TestExpose(t *testing.T) { expectedAffinity *corev1api.Affinity expectedPVCAnnotation map[string]string }{ + { + name: "get volume topology fail", + ownerBackup: backup, + exposeParam: CSISnapshotExposeParam{ + SnapshotName: "fake-vs", + OperationTimeout: time.Millisecond, + ExposeTimeout: time.Millisecond, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", + }, + err: "error getting volume topology for PV fake-pv, storage class fake-sc: error getting storage class fake-sc: storageclasses.storage.k8s.io \"fake-sc\" not found", + }, { name: "wait vs ready fail", ownerBackup: backup, @@ -206,6 +224,11 @@ func TestExpose(t *testing.T) { SnapshotName: "fake-vs", OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", + }, + kubeClientObj: []runtime.Object{ + scObj, }, err: "error wait volume snapshot ready: error to get VolumeSnapshot /fake-vs: volumesnapshots.snapshot.storage.k8s.io \"fake-vs\" not found", }, @@ -217,10 +240,15 @@ func TestExpose(t *testing.T) { SourceNamespace: "fake-ns", OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, }, + kubeClientObj: []runtime.Object{ + scObj, + }, err: "error to get volume snapshot content: error getting volume snapshot content from API: volumesnapshotcontents.snapshot.storage.k8s.io \"fake-vsc\" not found", }, { @@ -231,6 +259,8 @@ func TestExpose(t *testing.T) { SourceNamespace: "fake-ns", OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, @@ -245,6 +275,9 @@ func TestExpose(t *testing.T) { }, }, }, + kubeClientObj: []runtime.Object{ + scObj, + }, err: "error to delete volume snapshot: error to delete volume snapshot: fake-delete-error", }, { @@ -255,6 +288,8 @@ func TestExpose(t *testing.T) { SourceNamespace: "fake-ns", OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, @@ -269,6 +304,9 @@ func TestExpose(t *testing.T) { }, }, }, + kubeClientObj: []runtime.Object{ + scObj, + }, err: "error to delete volume snapshot content: error to delete volume snapshot content: fake-delete-error", }, { @@ -279,6 +317,8 @@ func TestExpose(t *testing.T) { SourceNamespace: "fake-ns", OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, @@ -293,6 +333,9 @@ func TestExpose(t *testing.T) { }, }, }, + kubeClientObj: []runtime.Object{ + scObj, + }, err: "error to create backup volume snapshot: fake-create-error", }, { @@ -303,6 +346,8 @@ func TestExpose(t *testing.T) { SourceNamespace: "fake-ns", OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, @@ -317,6 +362,9 @@ func TestExpose(t *testing.T) { }, }, }, + kubeClientObj: []runtime.Object{ + scObj, + }, err: "error to create backup volume snapshot content: fake-create-error", }, { @@ -326,11 +374,16 @@ func TestExpose(t *testing.T) { SnapshotName: "fake-vs", SourceNamespace: "fake-ns", AccessMode: "fake-mode", + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, vscObj, }, + kubeClientObj: []runtime.Object{ + scObj, + }, err: "error to create backup pvc: unsupported access mode fake-mode", }, { @@ -342,6 +395,8 @@ func TestExpose(t *testing.T) { OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, AccessMode: AccessModeFileSystem, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, @@ -356,6 +411,9 @@ func TestExpose(t *testing.T) { }, }, }, + kubeClientObj: []runtime.Object{ + scObj, + }, err: "error to create backup pvc: error to create pvc: fake-create-error", }, { @@ -367,6 +425,8 @@ func TestExpose(t *testing.T) { AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, @@ -374,6 +434,7 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, }, kubeReactors: []reactor{ { @@ -395,6 +456,8 @@ func TestExpose(t *testing.T) { AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, @@ -402,6 +465,24 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, + }, + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, }, }, { @@ -413,6 +494,8 @@ func TestExpose(t *testing.T) { AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObject, @@ -420,6 +503,24 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, + }, + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, }, }, { @@ -432,6 +533,8 @@ func TestExpose(t *testing.T) { OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, VolumeSize: *resource.NewQuantity(567890, ""), + StorageClass: "fake-sc", + SourcePVName: "fake-pv", }, snapshotClientObj: []runtime.Object{ vsObjectWithoutRestoreSize, @@ -439,8 +542,26 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, }, expectedVolumeSize: resource.NewQuantity(567890, ""), + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, + }, }, { name: "backupPod mounts read only backupPVC", @@ -449,6 +570,7 @@ func TestExpose(t *testing.T) { SnapshotName: "fake-vs", SourceNamespace: "fake-ns", StorageClass: "fake-sc", + SourcePVName: "fake-pv", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, @@ -465,8 +587,26 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, }, expectedReadOnlyPVC: true, + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, + }, }, { name: "backupPod mounts read only backupPVC and storageClass specified in backupPVC config", @@ -475,6 +615,7 @@ func TestExpose(t *testing.T) { SnapshotName: "fake-vs", SourceNamespace: "fake-ns", StorageClass: "fake-sc", + SourcePVName: "fake-pv", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, @@ -491,9 +632,27 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, }, expectedReadOnlyPVC: true, expectedBackupPVCStorageClass: "fake-sc-read-only", + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, + }, }, { name: "backupPod mounts backupPVC with storageClass specified in backupPVC config", @@ -502,6 +661,7 @@ func TestExpose(t *testing.T) { SnapshotName: "fake-vs", SourceNamespace: "fake-ns", StorageClass: "fake-sc", + SourcePVName: "fake-pv", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, @@ -517,8 +677,26 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, }, expectedBackupPVCStorageClass: "fake-sc-read-only", + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, + }, }, { name: "Affinity per StorageClass", @@ -527,6 +705,7 @@ func TestExpose(t *testing.T) { SnapshotName: "fake-vs", SourceNamespace: "fake-ns", StorageClass: "fake-sc", + SourcePVName: "fake-pv", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, @@ -551,6 +730,7 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, }, expectedAffinity: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ @@ -563,6 +743,11 @@ func TestExpose(t *testing.T) { Operator: corev1api.NodeSelectorOpIn, Values: []string{"Linux"}, }, + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, }, }, }, @@ -577,6 +762,7 @@ func TestExpose(t *testing.T) { SnapshotName: "fake-vs", SourceNamespace: "fake-ns", StorageClass: "fake-sc", + SourcePVName: "fake-pv", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, @@ -606,6 +792,7 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, }, expectedBackupPVCStorageClass: "fake-sc-read-only", expectedAffinity: &corev1api.Affinity{ @@ -619,6 +806,11 @@ func TestExpose(t *testing.T) { Operator: corev1api.NodeSelectorOpIn, Values: []string{"amd64"}, }, + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, }, }, }, @@ -633,6 +825,7 @@ func TestExpose(t *testing.T) { SnapshotName: "fake-vs", SourceNamespace: "fake-ns", StorageClass: "fake-sc", + SourcePVName: "fake-pv", AccessMode: AccessModeFileSystem, OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, @@ -649,9 +842,26 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, }, expectedBackupPVCStorageClass: "fake-sc-read-only", - expectedAffinity: nil, + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, + }, }, { name: "IntolerateSourceNode, get source node fail", @@ -677,6 +887,7 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, }, kubeReactors: []reactor{ { @@ -687,7 +898,23 @@ func TestExpose(t *testing.T) { }, }, }, - expectedAffinity: nil, + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, + }, expectedPVCAnnotation: nil, }, { @@ -714,8 +941,25 @@ func TestExpose(t *testing.T) { }, kubeClientObj: []runtime.Object{ daemonSet, + scObj, + }, + expectedAffinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, + }, + }, + }, + }, + }, }, - expectedAffinity: nil, expectedPVCAnnotation: map[string]string{util.VSphereCNSFastCloneAnno: "true"}, }, { @@ -744,6 +988,7 @@ func TestExpose(t *testing.T) { daemonSet, volumeAttachement1, volumeAttachement2, + scObj, }, expectedAffinity: &corev1api.Affinity{ NodeAffinity: &corev1api.NodeAffinity{ @@ -751,6 +996,11 @@ func TestExpose(t *testing.T) { NodeSelectorTerms: []corev1api.NodeSelectorTerm{ { MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Operator: corev1api.NodeSelectorOpNotIn, + Values: []string{"windows"}, + }, { Key: "kubernetes.io/hostname", Operator: corev1api.NodeSelectorOpNotIn, @@ -844,6 +1094,8 @@ func TestExpose(t *testing.T) { if test.expectedAffinity != nil { assert.Equal(t, test.expectedAffinity, backupPod.Spec.Affinity) + } else { + assert.Nil(t, backupPod.Spec.Affinity) } if test.expectedPVCAnnotation != nil { diff --git a/pkg/exposer/generic_restore.go b/pkg/exposer/generic_restore.go index c10370072..b711e7364 100644 --- a/pkg/exposer/generic_restore.go +++ b/pkg/exposer/generic_restore.go @@ -493,13 +493,15 @@ func (e *genericRestoreExposer) createRestorePod( containerName := string(ownerObject.UID) volumeName := string(ownerObject.UID) - var podAffinity *corev1api.Affinity - if selectedNode == "" { - e.log.Infof("No selected node for restore pod. Try to get affinity from the node-agent config.") + nodeSelector := map[string]string{} + if selectedNode != "" { + affinity = nil + nodeSelector["kubernetes.io/hostname"] = selectedNode + e.log.Infof("Selected node for restore pod. Ignore affinity from the node-agent config.") + } - if affinity != nil { - podAffinity = kube.ToSystemAffinity([]*kube.LoadAffinity{affinity}) - } + if affinity == nil { + affinity = &kube.LoadAffinity{} } podInfo, err := getInheritedPodInfo(ctx, e.kubeClient, ownerObject.Namespace, nodeOS) @@ -566,7 +568,6 @@ func (e *genericRestoreExposer) createRestorePod( args = append(args, podInfo.logLevelArgs...) var securityCtx *corev1api.PodSecurityContext - nodeSelector := map[string]string{} podOS := corev1api.PodOS{} if nodeOS == kube.NodeOSWindows { userID := "ContainerAdministrator" @@ -576,9 +577,14 @@ func (e *genericRestoreExposer) createRestorePod( }, } - nodeSelector[kube.NodeOSLabel] = kube.NodeOSWindows podOS.Name = kube.NodeOSWindows + affinity.NodeSelector.MatchExpressions = append(affinity.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: kube.NodeOSLabel, + Values: []string{kube.NodeOSWindows}, + Operator: metav1.LabelSelectorOpIn, + }) + toleration = append(toleration, []corev1api.Toleration{ { Key: "os", @@ -599,10 +605,17 @@ func (e *genericRestoreExposer) createRestorePod( RunAsUser: &userID, } - nodeSelector[kube.NodeOSLabel] = kube.NodeOSLinux podOS.Name = kube.NodeOSLinux + + affinity.NodeSelector.MatchExpressions = append(affinity.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: kube.NodeOSLabel, + Values: []string{kube.NodeOSWindows}, + Operator: metav1.LabelSelectorOpNotIn, + }) } + podAffinity := kube.ToSystemAffinity(affinity, nil) + pod := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: restorePodName, @@ -656,7 +669,6 @@ func (e *genericRestoreExposer) createRestorePod( ServiceAccountName: podInfo.serviceAccount, TerminationGracePeriodSeconds: &gracePeriod, Volumes: volumes, - NodeName: selectedNode, RestartPolicy: corev1api.RestartPolicyNever, SecurityContext: securityCtx, Tolerations: toleration, diff --git a/pkg/exposer/pod_volume.go b/pkg/exposer/pod_volume.go index 246d04e28..aeb6f1903 100644 --- a/pkg/exposer/pod_volume.go +++ b/pkg/exposer/pod_volume.go @@ -434,6 +434,8 @@ func (e *podVolumeExposer) createHostingPod( args = append(args, podInfo.logFormatArgs...) args = append(args, podInfo.logLevelArgs...) + affinity := &kube.LoadAffinity{} + var securityCtx *corev1api.PodSecurityContext var containerSecurityCtx *corev1api.SecurityContext nodeSelector := map[string]string{} @@ -446,9 +448,14 @@ func (e *podVolumeExposer) createHostingPod( }, } - nodeSelector[kube.NodeOSLabel] = kube.NodeOSWindows podOS.Name = kube.NodeOSWindows + affinity.NodeSelector.MatchExpressions = append(affinity.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: kube.NodeOSLabel, + Values: []string{kube.NodeOSWindows}, + Operator: metav1.LabelSelectorOpIn, + }) + toleration = append(toleration, []corev1api.Toleration{ { Key: "os", @@ -472,10 +479,17 @@ func (e *podVolumeExposer) createHostingPod( Privileged: &privileged, } - nodeSelector[kube.NodeOSLabel] = kube.NodeOSLinux podOS.Name = kube.NodeOSLinux + + affinity.NodeSelector.MatchExpressions = append(affinity.NodeSelector.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: kube.NodeOSLabel, + Values: []string{kube.NodeOSWindows}, + Operator: metav1.LabelSelectorOpNotIn, + }) } + podAffinity := kube.ToSystemAffinity(affinity, nil) + pod := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: hostingPodName, @@ -495,6 +509,7 @@ func (e *podVolumeExposer) createHostingPod( Spec: corev1api.PodSpec{ NodeSelector: nodeSelector, OS: &podOS, + Affinity: podAffinity, Containers: []corev1api.Container{ { Name: containerName, diff --git a/pkg/install/daemonset.go b/pkg/install/daemonset.go index ee63f3736..771114e82 100644 --- a/pkg/install/daemonset.go +++ b/pkg/install/daemonset.go @@ -235,12 +235,28 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1api.DaemonSet if c.forWindows { daemonSet.Spec.Template.Spec.SecurityContext = nil daemonSet.Spec.Template.Spec.Containers[0].SecurityContext = nil - daemonSet.Spec.Template.Spec.NodeSelector = map[string]string{ - "kubernetes.io/os": "windows", - } daemonSet.Spec.Template.Spec.OS = &corev1api.PodOS{ Name: "windows", } + + daemonSet.Spec.Template.Spec.Affinity = &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Values: []string{"windows"}, + Operator: corev1api.NodeSelectorOpIn, + }, + }, + }, + }, + }, + }, + } + daemonSet.Spec.Template.Spec.Tolerations = []corev1api.Toleration{ { Key: "os", @@ -256,11 +272,22 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1api.DaemonSet }, } } else { - daemonSet.Spec.Template.Spec.NodeSelector = map[string]string{ - "kubernetes.io/os": "linux", - } - daemonSet.Spec.Template.Spec.OS = &corev1api.PodOS{ - Name: "linux", + daemonSet.Spec.Template.Spec.Affinity = &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Values: []string{"windows"}, + Operator: corev1api.NodeSelectorOpNotIn, + }, + }, + }, + }, + }, + }, } } diff --git a/pkg/install/daemonset_test.go b/pkg/install/daemonset_test.go index 139d3dcd0..0f4de11bd 100644 --- a/pkg/install/daemonset_test.go +++ b/pkg/install/daemonset_test.go @@ -34,8 +34,23 @@ func TestDaemonSet(t *testing.T) { assert.Equal(t, "velero", ds.ObjectMeta.Namespace) assert.Equal(t, "node-agent", ds.Spec.Template.ObjectMeta.Labels["name"]) assert.Equal(t, "node-agent", ds.Spec.Template.ObjectMeta.Labels["role"]) - assert.Equal(t, "linux", ds.Spec.Template.Spec.NodeSelector["kubernetes.io/os"]) - assert.Equal(t, "linux", string(ds.Spec.Template.Spec.OS.Name)) + assert.Equal(t, &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Values: []string{"windows"}, + Operator: corev1api.NodeSelectorOpNotIn, + }, + }, + }, + }, + }, + }, + }, ds.Spec.Template.Spec.Affinity) assert.Equal(t, corev1api.PodSecurityContext{RunAsUser: &userID}, *ds.Spec.Template.Spec.SecurityContext) assert.Equal(t, corev1api.SecurityContext{Privileged: &boolFalse}, *ds.Spec.Template.Spec.Containers[0].SecurityContext) assert.Len(t, ds.Spec.Template.Spec.Volumes, 3) @@ -80,8 +95,24 @@ func TestDaemonSet(t *testing.T) { assert.Equal(t, "velero", ds.ObjectMeta.Namespace) assert.Equal(t, "node-agent-windows", ds.Spec.Template.ObjectMeta.Labels["name"]) assert.Equal(t, "node-agent", ds.Spec.Template.ObjectMeta.Labels["role"]) - assert.Equal(t, "windows", ds.Spec.Template.Spec.NodeSelector["kubernetes.io/os"]) assert.Equal(t, "windows", string(ds.Spec.Template.Spec.OS.Name)) + assert.Equal(t, &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Values: []string{"windows"}, + Operator: corev1api.NodeSelectorOpIn, + }, + }, + }, + }, + }, + }, + }, ds.Spec.Template.Spec.Affinity) assert.Equal(t, (*corev1api.PodSecurityContext)(nil), ds.Spec.Template.Spec.SecurityContext) assert.Equal(t, (*corev1api.SecurityContext)(nil), ds.Spec.Template.Spec.Containers[0].SecurityContext) } diff --git a/pkg/install/deployment.go b/pkg/install/deployment.go index d1010d294..04ea40e04 100644 --- a/pkg/install/deployment.go +++ b/pkg/install/deployment.go @@ -364,12 +364,26 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1api.Deployme Spec: corev1api.PodSpec{ RestartPolicy: corev1api.RestartPolicyAlways, ServiceAccountName: c.serviceAccountName, - NodeSelector: map[string]string{ - "kubernetes.io/os": "linux", - }, OS: &corev1api.PodOS{ Name: "linux", }, + Affinity: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Values: []string{"windows"}, + Operator: corev1api.NodeSelectorOpNotIn, + }, + }, + }, + }, + }, + }, + }, Containers: []corev1api.Container{ { Name: "velero", diff --git a/pkg/install/deployment_test.go b/pkg/install/deployment_test.go index b8aeaa9dd..53b696f72 100644 --- a/pkg/install/deployment_test.go +++ b/pkg/install/deployment_test.go @@ -100,8 +100,23 @@ func TestDeployment(t *testing.T) { assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2) assert.Equal(t, "--repo-maintenance-job-configmap=test-repo-maintenance-config", deploy.Spec.Template.Spec.Containers[0].Args[1]) - assert.Equal(t, "linux", deploy.Spec.Template.Spec.NodeSelector["kubernetes.io/os"]) - assert.Equal(t, "linux", string(deploy.Spec.Template.Spec.OS.Name)) + assert.Equal(t, &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "kubernetes.io/os", + Values: []string{"windows"}, + Operator: corev1api.NodeSelectorOpNotIn, + }, + }, + }, + }, + }, + }, + }, deploy.Spec.Template.Spec.Affinity) } func TestDeploymentWithPriorityClassName(t *testing.T) { diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 30e67a7b6..86d78028c 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -80,6 +80,9 @@ const ( DataDownloadFailureTotal = "data_download_failure_total" DataDownloadCancelTotal = "data_download_cancel_total" + // schedule metrics + scheduleExpectedIntervalSeconds = "schedule_expected_interval_seconds" + // repo maintenance metrics repoMaintenanceSuccessTotal = "repo_maintenance_success_total" repoMaintenanceFailureTotal = "repo_maintenance_failure_total" @@ -347,6 +350,14 @@ func NewServerMetrics() *ServerMetrics { }, []string{scheduleLabel, backupNameLabel}, ), + scheduleExpectedIntervalSeconds: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricNamespace, + Name: scheduleExpectedIntervalSeconds, + Help: "Expected interval between consecutive scheduled backups, in seconds", + }, + []string{scheduleLabel}, + ), repoMaintenanceSuccessTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, @@ -644,6 +655,9 @@ func (m *ServerMetrics) RemoveSchedule(scheduleName string) { if c, ok := m.metrics[csiSnapshotFailureTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName, "") } + if g, ok := m.metrics[scheduleExpectedIntervalSeconds].(*prometheus.GaugeVec); ok { + g.DeleteLabelValues(scheduleName) + } } // InitMetricsForNode initializes counter metrics for a node. @@ -758,6 +772,14 @@ func (m *ServerMetrics) SetBackupLastSuccessfulTimestamp(backupSchedule string, } } +// SetScheduleExpectedIntervalSeconds records the expected interval in seconds, +// between consecutive backups for a schedule. +func (m *ServerMetrics) SetScheduleExpectedIntervalSeconds(scheduleName string, seconds float64) { + if g, ok := m.metrics[scheduleExpectedIntervalSeconds].(*prometheus.GaugeVec); ok { + g.WithLabelValues(scheduleName).Set(seconds) + } +} + // SetBackupTotal records the current number of existent backups. func (m *ServerMetrics) SetBackupTotal(numberOfBackups int64) { if g, ok := m.metrics[backupTotal].(prometheus.Gauge); ok { diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index 184e496ab..a24f2bf33 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -259,6 +259,90 @@ func TestMultipleAdhocBackupsShareMetrics(t *testing.T) { assert.Equal(t, float64(1), validationFailureMetric, "All adhoc validation failures should be counted together") } +// TestSetScheduleExpectedIntervalSeconds verifies that the expected interval metric +// is properly recorded for schedules. +func TestSetScheduleExpectedIntervalSeconds(t *testing.T) { + tests := []struct { + name string + scheduleName string + intervalSeconds float64 + description string + }{ + { + name: "every 5 minutes schedule", + scheduleName: "frequent-backup", + intervalSeconds: 300, + description: "Expected interval should be 5m in seconds", + }, + { + name: "daily schedule", + scheduleName: "daily-backup", + intervalSeconds: 86400, + description: "Expected interval should be 24h in seconds", + }, + { + name: "monthly schedule", + scheduleName: "monthly-backup", + intervalSeconds: 2678400, // 31 days in seconds + description: "Expected interval should be 31 days in seconds", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + m := NewServerMetrics() + m.SetScheduleExpectedIntervalSeconds(tc.scheduleName, tc.intervalSeconds) + + metric := getMetricValue(t, m.metrics[scheduleExpectedIntervalSeconds].(*prometheus.GaugeVec), tc.scheduleName) + assert.Equal(t, tc.intervalSeconds, metric, tc.description) + }) + } +} + +// TestScheduleExpectedIntervalNotInitializedByDefault verifies that the expected +// interval metric is not initialized by InitSchedule, so it only appears for +// schedules with a valid cron expression. +func TestScheduleExpectedIntervalNotInitializedByDefault(t *testing.T) { + m := NewServerMetrics() + m.InitSchedule("test-schedule") + + // The metric should not have any values after InitSchedule + ch := make(chan prometheus.Metric, 1) + m.metrics[scheduleExpectedIntervalSeconds].(*prometheus.GaugeVec).Collect(ch) + close(ch) + + count := 0 + for range ch { + count++ + } + assert.Equal(t, 0, count, "scheduleExpectedIntervalSeconds should not be initialized by InitSchedule") +} + +// TestRemoveScheduleCleansUpExpectedInterval verifies that RemoveSchedule +// cleans up the expected interval metric. +func TestRemoveScheduleCleansUpExpectedInterval(t *testing.T) { + m := NewServerMetrics() + m.InitSchedule("test-schedule") + m.SetScheduleExpectedIntervalSeconds("test-schedule", 3600) + + // Verify metric exists + metric := getMetricValue(t, m.metrics[scheduleExpectedIntervalSeconds].(*prometheus.GaugeVec), "test-schedule") + assert.Equal(t, float64(3600), metric) + + // Remove schedule and verify metric is cleaned up + m.RemoveSchedule("test-schedule") + + ch := make(chan prometheus.Metric, 1) + m.metrics[scheduleExpectedIntervalSeconds].(*prometheus.GaugeVec).Collect(ch) + close(ch) + + count := 0 + for range ch { + count++ + } + assert.Equal(t, 0, count, "scheduleExpectedIntervalSeconds should be removed after RemoveSchedule") +} + // TestInitScheduleWithEmptyName verifies that InitSchedule works correctly // with an empty schedule name (for adhoc backups). func TestInitScheduleWithEmptyName(t *testing.T) { diff --git a/pkg/nodeagent/node_agent.go b/pkg/nodeagent/node_agent.go index a5de2465c..87efb896a 100644 --- a/pkg/nodeagent/node_agent.go +++ b/pkg/nodeagent/node_agent.go @@ -97,7 +97,10 @@ func isRunningInNode(ctx context.Context, namespace string, nodeName string, crC } if crClient != nil { - err = crClient.List(ctx, pods, &ctrlclient.ListOptions{LabelSelector: parsedSelector}) + err = crClient.List(ctx, pods, &ctrlclient.ListOptions{ + LabelSelector: parsedSelector, + Namespace: namespace, + }) } else { pods, err = kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: parsedSelector.String()}) } diff --git a/pkg/nodeagent/node_agent_test.go b/pkg/nodeagent/node_agent_test.go index cb46ee569..168e91de1 100644 --- a/pkg/nodeagent/node_agent_test.go +++ b/pkg/nodeagent/node_agent_test.go @@ -118,28 +118,37 @@ func TestIsRunningInNode(t *testing.T) { Phase(corev1api.PodRunning). NodeName("fake-node"). Result() + nodeAgentPodOtherNs := builder.ForPod("other-ns", "fake-pod-other"). + Labels(map[string]string{"role": "node-agent"}). + Phase(corev1api.PodRunning). + NodeName("fake-node"). + Result() tests := []struct { name string kubeClientObj []runtime.Object + namespace string nodeName string expectErr string }{ { name: "node name is empty", + namespace: "fake-ns", expectErr: "node name is empty", }, { - name: "ds pod not found", - nodeName: "fake-node", + name: "ds pod not found", + namespace: "fake-ns", + nodeName: "fake-node", kubeClientObj: []runtime.Object{ nonNodeAgentPod, }, expectErr: "daemonset pod not found in running state in node fake-node", }, { - name: "ds po are not all running", - nodeName: "fake-node", + name: "ds po are not all running", + namespace: "fake-ns", + nodeName: "fake-node", kubeClientObj: []runtime.Object{ nodeAgentPodNotRunning, nodeAgentPodRunning1, @@ -147,8 +156,9 @@ func TestIsRunningInNode(t *testing.T) { expectErr: "daemonset pod not found in running state in node fake-node", }, { - name: "ds pods wrong node name", - nodeName: "fake-node", + name: "ds pods wrong node name", + namespace: "fake-ns", + nodeName: "fake-node", kubeClientObj: []runtime.Object{ nodeAgentPodNotRunning, nodeAgentPodRunning1, @@ -157,8 +167,9 @@ func TestIsRunningInNode(t *testing.T) { expectErr: "daemonset pod not found in running state in node fake-node", }, { - name: "succeed", - nodeName: "fake-node", + name: "succeed", + namespace: "fake-ns", + nodeName: "fake-node", kubeClientObj: []runtime.Object{ nodeAgentPodNotRunning, nodeAgentPodRunning1, @@ -166,6 +177,23 @@ func TestIsRunningInNode(t *testing.T) { nodeAgentPodRunning3, }, }, + { + name: "cross-namespace isolation - pod in wrong namespace on same node", + namespace: "fake-ns", + nodeName: "fake-node", + kubeClientObj: []runtime.Object{ + nodeAgentPodOtherNs, + }, + expectErr: "daemonset pod not found in running state in node fake-node", + }, + { + name: "cross-namespace isolation - pod in correct namespace on same node", + namespace: "other-ns", + nodeName: "fake-node", + kubeClientObj: []runtime.Object{ + nodeAgentPodOtherNs, + }, + }, } for _, test := range tests { @@ -175,7 +203,7 @@ func TestIsRunningInNode(t *testing.T) { fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build() - err := IsRunningInNode(t.Context(), "", test.nodeName, fakeClient) + err := IsRunningInNode(t.Context(), test.namespace, test.nodeName, fakeClient) if test.expectErr == "" { assert.NoError(t, err) } else { diff --git a/pkg/persistence/object_store.go b/pkg/persistence/object_store.go index eaf983819..fd441e943 100644 --- a/pkg/persistence/object_store.go +++ b/pkg/persistence/object_store.go @@ -149,7 +149,8 @@ func (b *objectBackupStoreGetter) Get(location *velerov1api.BackupStorageLocatio // if there are any slashes in the middle of 'bucket', the user // probably put / in the bucket field, which we // don't support. - if strings.Contains(bucket, "/") { + // Exception: MRAP ARNs (arn:aws:s3::...) legitimately contain slashes. + if strings.Contains(bucket, "/") && !strings.HasPrefix(bucket, "arn:aws:s3:") { return nil, errors.Errorf("backup storage location's bucket name %q must not contain a '/' (if using a prefix, put it in the 'Prefix' field instead)", location.Spec.ObjectStorage.Bucket) } diff --git a/pkg/persistence/object_store_test.go b/pkg/persistence/object_store_test.go index fac2f8d97..e9a3bde36 100644 --- a/pkg/persistence/object_store_test.go +++ b/pkg/persistence/object_store_test.go @@ -943,6 +943,24 @@ func TestNewObjectBackupStoreGetter(t *testing.T) { wantBucket: "bucket", wantPrefix: "prefix/", }, + { + name: "when the Bucket field is an MRAP ARN, it should be valid", + location: builder.ForBackupStorageLocation("", "").Provider("provider-1").Bucket("arn:aws:s3::123456789012:accesspoint/abcdef0123456.mrap").Result(), + objectStoreGetter: objectStoreGetter{ + "provider-1": newInMemoryObjectStore("arn:aws:s3::123456789012:accesspoint/abcdef0123456.mrap"), + }, + credFileStore: velerotest.NewFakeCredentialsFileStore("", nil), + wantBucket: "arn:aws:s3::123456789012:accesspoint/abcdef0123456.mrap", + }, + { + name: "when the Bucket field is an MRAP ARN with trailing slash, it should be valid and trimmed", + location: builder.ForBackupStorageLocation("", "").Provider("provider-1").Bucket("arn:aws:s3::123456789012:accesspoint/abcdef0123456.mrap/").Result(), + objectStoreGetter: objectStoreGetter{ + "provider-1": newInMemoryObjectStore("arn:aws:s3::123456789012:accesspoint/abcdef0123456.mrap"), + }, + credFileStore: velerotest.NewFakeCredentialsFileStore("", nil), + wantBucket: "arn:aws:s3::123456789012:accesspoint/abcdef0123456.mrap", + }, } for _, tc := range tests { diff --git a/pkg/plugin/utils/volumehelper/volume_policy_helper.go b/pkg/plugin/utils/volumehelper/volume_policy_helper.go index a19f8d7c8..843c23b06 100644 --- a/pkg/plugin/utils/volumehelper/volume_policy_helper.go +++ b/pkg/plugin/utils/volumehelper/volume_policy_helper.go @@ -26,6 +26,8 @@ import ( "github.com/vmware-tanzu/velero/internal/volumehelper" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/util/boolptr" + podvolumeutil "github.com/vmware-tanzu/velero/pkg/util/podvolume" + vhutil "github.com/vmware-tanzu/velero/pkg/util/volumehelper" ) // ShouldPerformSnapshotWithBackup is used for third-party plugins. @@ -66,7 +68,7 @@ func ShouldPerformSnapshotWithVolumeHelper( backup velerov1api.Backup, crClient crclient.Client, logger logrus.FieldLogger, - vh volumehelper.VolumeHelper, + vh vhutil.VolumeHelper, ) (bool, error) { // If a VolumeHelper is provided, use it directly if vh != nil { @@ -95,3 +97,45 @@ func ShouldPerformSnapshotWithVolumeHelper( return volumeHelperImpl.ShouldPerformSnapshot(unstructured, groupResource) } + +// NewVolumeHelperWithNamespaces creates a VolumeHelper with a PVC-to-Pod cache for improved performance. +// The cache is built internally from the provided namespaces list. +// This avoids O(N*M) complexity when there are many PVCs and pods. +// See issue #9179 for details. +// Returns an error if cache building fails - callers should not proceed with backup in this case. +func NewVolumeHelperWithNamespaces( + volumePolicy *resourcepolicies.Policies, + snapshotVolumes *bool, + logger logrus.FieldLogger, + client crclient.Client, + defaultVolumesToFSBackup bool, + backupExcludePVC bool, + namespaces []string, +) (vhutil.VolumeHelper, error) { + return volumehelper.NewVolumeHelperImplWithNamespaces( + volumePolicy, + snapshotVolumes, + logger, + client, + defaultVolumesToFSBackup, + backupExcludePVC, + namespaces, + ) +} + +// NewVolumeHelperWithCache creates a VolumeHelper using an externally managed PVC-to-Pod cache. +// This is used by plugins that build the cache lazily per-namespace (following the pattern from PR #9226). +// The cache can be nil, in which case PVC-to-Pod lookups will fall back to direct API calls. +func NewVolumeHelperWithCache( + backup velerov1api.Backup, + client crclient.Client, + logger logrus.FieldLogger, + pvcPodCache *podvolumeutil.PVCPodCache, +) (vhutil.VolumeHelper, error) { + return volumehelper.NewVolumeHelperImplWithCache( + backup, + client, + logger, + pvcPodCache, + ) +} diff --git a/pkg/podexec/pod_command_executor.go b/pkg/podexec/pod_command_executor.go index b1d7fbf59..0dbc28e95 100644 --- a/pkg/podexec/pod_command_executor.go +++ b/pkg/podexec/pod_command_executor.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "net/url" + "slices" "time" "github.com/pkg/errors" @@ -184,10 +185,22 @@ func (e *defaultPodCommandExecutor) ExecutePodCommand(log logrus.FieldLogger, it } func ensureContainerExists(pod *corev1api.Pod, container string) error { - for _, c := range pod.Spec.Containers { - if c.Name == container { - return nil - } + existsAsMainContainer := slices.ContainsFunc(pod.Spec.Containers, func(c corev1api.Container) bool { + return c.Name == container + }) + + if existsAsMainContainer { + return nil + } + + existsAsSidecar := slices.ContainsFunc(pod.Spec.InitContainers, func(c corev1api.Container) bool { + return c.RestartPolicy != nil && + *c.RestartPolicy == corev1api.ContainerRestartPolicyAlways && + c.Name == container + }) + + if existsAsSidecar { + return nil } return errors.Errorf("no such container: %q", container) diff --git a/pkg/podexec/pod_command_executor_test.go b/pkg/podexec/pod_command_executor_test.go index e28eb0458..3286ba42d 100644 --- a/pkg/podexec/pod_command_executor_test.go +++ b/pkg/podexec/pod_command_executor_test.go @@ -35,6 +35,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" "k8s.io/client-go/tools/remotecommand" + "k8s.io/utils/ptr" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerotest "github.com/vmware-tanzu/velero/pkg/test" @@ -253,6 +254,15 @@ func TestEnsureContainerExists(t *testing.T) { Name: "foo", }, }, + InitContainers: []corev1api.Container{ + { + Name: "baz", + }, + { + Name: "qux", + RestartPolicy: ptr.To(corev1api.ContainerRestartPolicyAlways), + }, + }, }, } @@ -260,7 +270,13 @@ func TestEnsureContainerExists(t *testing.T) { require.EqualError(t, err, `no such container: "bar"`) err = ensureContainerExists(pod, "foo") - assert.NoError(t, err) + require.NoError(t, err) + + err = ensureContainerExists(pod, "baz") + require.EqualError(t, err, `no such container: "baz"`) + + err = ensureContainerExists(pod, "qux") + require.NoError(t, err) } func TestPodCompeted(t *testing.T) { diff --git a/pkg/podvolume/backup_micro_service.go b/pkg/podvolume/backup_micro_service.go index 3c9cbab61..11ca66676 100644 --- a/pkg/podvolume/backup_micro_service.go +++ b/pkg/podvolume/backup_micro_service.go @@ -307,6 +307,7 @@ func (r *BackupMicroService) cancelPodVolumeBackup(pvb *velerov1api.PodVolumeBac fsBackup := r.dataPathMgr.GetAsyncBR(pvb.Name) if fsBackup == nil { r.OnDataPathCancelled(r.ctx, pvb.GetNamespace(), pvb.GetName()) + r.eventRecorder.EndingEvent(pvb, false, datapath.EventReasonStopped, "Data path for %s exited without start", pvb.Name) } else { fsBackup.Cancel() } diff --git a/pkg/podvolume/backup_micro_service_test.go b/pkg/podvolume/backup_micro_service_test.go index eb10b622a..a5b6cd1f4 100644 --- a/pkg/podvolume/backup_micro_service_test.go +++ b/pkg/podvolume/backup_micro_service_test.go @@ -258,8 +258,8 @@ func TestCancelPodVolumeBackup(t *testing.T) { }{ { name: "no fs backup", - expectedEventReason: datapath.EventReasonCancelled, - expectedEventMsg: "Data path for PVB fake-pvb canceled", + expectedEventReason: datapath.EventReasonStopped, + expectedEventMsg: "Data path for fake-pvb exited without start", expectedErr: datapath.ErrCancelled, }, } diff --git a/pkg/podvolume/backupper.go b/pkg/podvolume/backupper.go index fba3cb19a..6b534d5ed 100644 --- a/pkg/podvolume/backupper.go +++ b/pkg/podvolume/backupper.go @@ -210,11 +210,9 @@ func resultsKey(ns, name string) string { func (b *backupper) getMatchAction(resPolicies *resourcepolicies.Policies, pvc *corev1api.PersistentVolumeClaim, volume *corev1api.Volume) (*resourcepolicies.Action, error) { if pvc != nil { - pv := new(corev1api.PersistentVolume) - err := b.crClient.Get(context.TODO(), ctrlclient.ObjectKey{Name: pvc.Spec.VolumeName}, pv) - if err != nil { - return nil, errors.Wrapf(err, "error getting pv for pvc %s", pvc.Spec.VolumeName) - } + // Ignore err, if the PV is not available (Pending/Lost PVC or PV fetch failed) - try matching with PVC only + // GetPVForPVC returns nil for all error cases + pv, _ := kube.GetPVForPVC(pvc, b.crClient) vfd := resourcepolicies.NewVolumeFilterData(pv, nil, pvc) return resPolicies.GetMatchAction(vfd) } @@ -274,7 +272,7 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api. return nil, pvcSummary, []error{err} } - repositoryType := funcGetRepositoryType(b.uploaderType) + repositoryType := funcGetRepositoryType() if repositoryType == "" { err := errors.Errorf("empty repository type, uploader %s", b.uploaderType) skipAllPodVolumes(pod, volumesToBackup, err, pvcSummary, log) @@ -307,11 +305,6 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api. } } - repoIdentifier := "" - if repositoryType == velerov1api.BackupRepositoryTypeRestic { - repoIdentifier = repo.Spec.ResticIdentifier - } - for _, volumeName := range volumesToBackup { volume, ok := podVolumes[volumeName] if !ok { @@ -368,7 +361,7 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api. continue } - volumeBackup := newPodVolumeBackup(backup, pod, volume, repoIdentifier, b.uploaderType, pvc) + volumeBackup := newPodVolumeBackup(backup, pod, volume, "", b.uploaderType, pvc) // the PVB must be added into the indexer before creating it in API server otherwise unexpected behavior may happen: // the PVB may be handled very quickly by the controller and the informer handler will insert the PVB before "b.pvbIndexer.Add(volumeBackup)" runs, // this causes the PVB inserted by "b.pvbIndexer.Add(volumeBackup)" overrides the PVB in the indexer while the PVB inserted by "b.pvbIndexer.Add(volumeBackup)" diff --git a/pkg/podvolume/backupper_test.go b/pkg/podvolume/backupper_test.go index 6359df696..f7686978a 100644 --- a/pkg/podvolume/backupper_test.go +++ b/pkg/podvolume/backupper_test.go @@ -309,8 +309,8 @@ func createNodeObj() *corev1api.Node { func TestBackupPodVolumes(t *testing.T) { scheme := runtime.NewScheme() - velerov1api.AddToScheme(scheme) - corev1api.AddToScheme(scheme) + require.NoError(t, velerov1api.AddToScheme(scheme)) + require.NoError(t, corev1api.AddToScheme(scheme)) log := logrus.New() tests := []struct { @@ -580,7 +580,7 @@ func TestBackupPodVolumes(t *testing.T) { require.NoError(t, err) if test.mockGetRepositoryType { - funcGetRepositoryType = func(string) string { return "" } + funcGetRepositoryType = func() string { return "" } } else { funcGetRepositoryType = getRepositoryType } @@ -778,7 +778,7 @@ func TestWaitAllPodVolumesProcessed(t *testing.T) { backuper := newBackupper(c.ctx, log, nil, nil, informer, nil, "", &velerov1api.Backup{}) if c.pvb != nil { - backuper.pvbIndexer.Add(c.pvb) + require.NoError(t, backuper.pvbIndexer.Add(c.pvb)) backuper.wg.Add(1) } @@ -833,3 +833,185 @@ func TestPVCBackupSummary(t *testing.T) { assert.Empty(t, pbs.Skipped) assert.Len(t, pbs.Backedup, 2) } + +func TestGetMatchAction_PendingPVC(t *testing.T) { + // Create resource policies that skip Pending/Lost PVCs + resPolicies := &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "pvcPhase": []string{"Pending", "Lost"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + } + policies := &resourcepolicies.Policies{} + err := policies.BuildPolicy(resPolicies) + require.NoError(t, err) + + testCases := []struct { + name string + pvc *corev1api.PersistentVolumeClaim + volume *corev1api.Volume + pv *corev1api.PersistentVolume + expectedAction *resourcepolicies.Action + expectError bool + }{ + { + name: "Pending PVC with pvcPhase skip policy should return skip action", + pvc: builder.ForPersistentVolumeClaim("ns", "pending-pvc"). + StorageClass("test-sc"). + Phase(corev1api.ClaimPending). + Result(), + volume: &corev1api.Volume{ + Name: "test-volume", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "pending-pvc", + }, + }, + }, + pv: nil, + expectedAction: &resourcepolicies.Action{Type: resourcepolicies.Skip}, + expectError: false, + }, + { + name: "Lost PVC with pvcPhase skip policy should return skip action", + pvc: builder.ForPersistentVolumeClaim("ns", "lost-pvc"). + StorageClass("test-sc"). + Phase(corev1api.ClaimLost). + Result(), + volume: &corev1api.Volume{ + Name: "test-volume", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "lost-pvc", + }, + }, + }, + pv: nil, + expectedAction: &resourcepolicies.Action{Type: resourcepolicies.Skip}, + expectError: false, + }, + { + name: "Bound PVC with matching PV should not match pvcPhase policy", + pvc: builder.ForPersistentVolumeClaim("ns", "bound-pvc"). + StorageClass("test-sc"). + VolumeName("test-pv"). + Phase(corev1api.ClaimBound). + Result(), + volume: &corev1api.Volume{ + Name: "test-volume", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "bound-pvc", + }, + }, + }, + pv: builder.ForPersistentVolume("test-pv").StorageClass("test-sc").Result(), + expectedAction: nil, + expectError: false, + }, + { + name: "Pending PVC with no matching policy should return nil action", + pvc: builder.ForPersistentVolumeClaim("ns", "pending-pvc-no-match"). + StorageClass("test-sc"). + Phase(corev1api.ClaimPending). + Result(), + volume: &corev1api.Volume{ + Name: "test-volume", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "pending-pvc-no-match", + }, + }, + }, + pv: nil, + expectedAction: &resourcepolicies.Action{Type: resourcepolicies.Skip}, // Will match the pvcPhase policy + expectError: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Build fake client with PV if present + var objs []runtime.Object + if tc.pv != nil { + objs = append(objs, tc.pv) + } + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) + + b := &backupper{ + crClient: fakeClient, + } + + action, err := b.getMatchAction(policies, tc.pvc, tc.volume) + if tc.expectError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + if tc.expectedAction == nil { + assert.Nil(t, action) + } else { + require.NotNil(t, action) + assert.Equal(t, tc.expectedAction.Type, action.Type) + } + }) + } +} + +func TestGetMatchAction_PVCWithoutPVLookupError(t *testing.T) { + // Test that when a PVC has a VolumeName but the PV doesn't exist, + // the function ignores the error and tries to match with PVC only + resPolicies := &resourcepolicies.ResourcePolicies{ + Version: "v1", + VolumePolicies: []resourcepolicies.VolumePolicy{ + { + Conditions: map[string]any{ + "pvcPhase": []string{"Pending"}, + }, + Action: resourcepolicies.Action{ + Type: resourcepolicies.Skip, + }, + }, + }, + } + policies := &resourcepolicies.Policies{} + err := policies.BuildPolicy(resPolicies) + require.NoError(t, err) + + // Pending PVC without a matching PV in the cluster + pvc := builder.ForPersistentVolumeClaim("ns", "pending-pvc"). + StorageClass("test-sc"). + Phase(corev1api.ClaimPending). + Result() + + volume := &corev1api.Volume{ + Name: "test-volume", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "pending-pvc", + }, + }, + } + + // Empty client - no PV exists + fakeClient := velerotest.NewFakeControllerRuntimeClient(t) + + b := &backupper{ + crClient: fakeClient, + } + + // Should succeed even though PV lookup would fail + // because the function ignores PV lookup errors and uses PVC-only matching + action, err := b.getMatchAction(policies, pvc, volume) + require.NoError(t, err) + require.NotNil(t, action) + assert.Equal(t, resourcepolicies.Skip, action.Type) +} diff --git a/pkg/podvolume/restore_micro_service.go b/pkg/podvolume/restore_micro_service.go index 40d6aa1fb..decf94b1b 100644 --- a/pkg/podvolume/restore_micro_service.go +++ b/pkg/podvolume/restore_micro_service.go @@ -302,6 +302,7 @@ func (r *RestoreMicroService) cancelPodVolumeRestore(pvr *velerov1api.PodVolumeR fsBackup := r.dataPathMgr.GetAsyncBR(pvr.Name) if fsBackup == nil { r.OnPvrCancelled(r.ctx, pvr.GetNamespace(), pvr.GetName()) + r.eventRecorder.EndingEvent(pvr, false, datapath.EventReasonStopped, "Data path for %s exited without start", pvr.Name) } else { fsBackup.Cancel() } diff --git a/pkg/podvolume/restore_micro_service_test.go b/pkg/podvolume/restore_micro_service_test.go index 07e2056b0..2c94ddf46 100644 --- a/pkg/podvolume/restore_micro_service_test.go +++ b/pkg/podvolume/restore_micro_service_test.go @@ -284,8 +284,8 @@ func TestCancelPodVolumeRestore(t *testing.T) { }{ { name: "no fs restore", - expectedEventReason: datapath.EventReasonCancelled, - expectedEventMsg: "Data path for PVR fake-pvr canceled", + expectedEventReason: datapath.EventReasonStopped, + expectedEventMsg: "Data path for fake-pvr exited without start", expectedErr: datapath.ErrCancelled, }, } diff --git a/pkg/podvolume/restorer.go b/pkg/podvolume/restorer.go index 47219ae99..ce662d5de 100644 --- a/pkg/podvolume/restorer.go +++ b/pkg/podvolume/restorer.go @@ -166,11 +166,6 @@ func (r *restorer) RestorePodVolumes(data RestoreData, tracker *volume.RestoreVo podVolumes[podVolume.Name] = podVolume } - repoIdentifier := "" - if repositoryType == velerov1api.BackupRepositoryTypeRestic { - repoIdentifier = repo.Spec.ResticIdentifier - } - for volume, backupInfo := range volumesToRestore { volumeObj, ok := podVolumes[volume] var pvc *corev1api.PersistentVolumeClaim @@ -185,7 +180,7 @@ func (r *restorer) RestorePodVolumes(data RestoreData, tracker *volume.RestoreVo } } - volumeRestore := newPodVolumeRestore(data.Restore, data.Pod, data.BackupLocation, volume, backupInfo.snapshotID, backupInfo.snapshotSize, repoIdentifier, backupInfo.uploaderType, data.SourceNamespace, pvc) + volumeRestore := newPodVolumeRestore(data.Restore, data.Pod, data.BackupLocation, volume, backupInfo.snapshotID, backupInfo.snapshotSize, "", backupInfo.uploaderType, data.SourceNamespace, pvc) if err := veleroclient.CreateRetryGenerateName(r.crClient, r.ctx, volumeRestore); err != nil { errs = append(errs, errors.WithStack(err)) continue diff --git a/pkg/podvolume/restorer_test.go b/pkg/podvolume/restorer_test.go index 36a1fc034..e10146578 100644 --- a/pkg/podvolume/restorer_test.go +++ b/pkg/podvolume/restorer_test.go @@ -204,24 +204,6 @@ func TestRestorePodVolumes(t *testing.T) { }, }, }, - { - name: "get repository type fail", - pvbs: []*velerov1api.PodVolumeBackup{ - createPVBObj(true, true, 1, "restic"), - createPVBObj(true, true, 2, "kopia"), - }, - kubeClientObj: []runtime.Object{ - createNodeAgentDaemonset(), - }, - restoredPod: createPodObj(false, false, false, 2), - sourceNamespace: "fake-ns", - errs: []expectError{ - { - err: "multiple repository type in one backup", - prefixOnly: true, - }, - }, - }, { name: "ensure repo fail", pvbs: []*velerov1api.PodVolumeBackup{ diff --git a/pkg/podvolume/util.go b/pkg/podvolume/util.go index 1864e9615..bb827c732 100644 --- a/pkg/podvolume/util.go +++ b/pkg/podvolume/util.go @@ -62,12 +62,12 @@ func GetVolumeBackupsForPod(podVolumeBackups []*velerov1api.PodVolumeBackup, pod // GetPvbRepositoryType returns the repositoryType according to the PVB information func GetPvbRepositoryType(pvb *velerov1api.PodVolumeBackup) string { - return getRepositoryType(pvb.Spec.UploaderType) + return getRepositoryType() } // GetPvrRepositoryType returns the repositoryType according to the PVR information func GetPvrRepositoryType(pvr *velerov1api.PodVolumeRestore) string { - return getRepositoryType(pvr.Spec.UploaderType) + return getRepositoryType() } // getVolumeBackupInfoForPod returns a map, of volume name -> VolumeBackupInfo, @@ -97,7 +97,7 @@ func getVolumeBackupInfoForPod(podVolumeBackups []*velerov1api.PodVolumeBackup, snapshotID: pvb.Status.SnapshotID, snapshotSize: pvb.Status.Progress.TotalBytes, uploaderType: getUploaderTypeOrDefault(pvb.Spec.UploaderType), - repositoryType: getRepositoryType(pvb.Spec.UploaderType), + repositoryType: getRepositoryType(), } } @@ -111,7 +111,7 @@ func getVolumeBackupInfoForPod(podVolumeBackups []*velerov1api.PodVolumeBackup, } for k, v := range fromAnnntation { - volumes[k] = volumeBackupInfo{v, 0, uploader.ResticType, velerov1api.BackupRepositoryTypeRestic} + volumes[k] = volumeBackupInfo{v, 0, uploader.KopiaType, velerov1api.BackupRepositoryTypeKopia} } return volumes @@ -135,7 +135,7 @@ func GetSnapshotIdentifier(podVolumeBackups *velerov1api.PodVolumeBackupList) ma VolumeNamespace: item.Spec.Pod.Namespace, BackupStorageLocation: item.Spec.BackupStorageLocation, SnapshotID: item.Status.SnapshotID, - RepositoryType: getRepositoryType(item.Spec.UploaderType), + RepositoryType: getRepositoryType(), UploaderType: item.Spec.UploaderType, Source: item.Status.Path, RepoIdentifier: item.Spec.RepoIdentifier, @@ -164,27 +164,14 @@ func getUploaderTypeOrDefault(uploaderType string) string { if uploaderType != "" { return uploaderType } - return uploader.ResticType + return uploader.KopiaType } -// getRepositoryType returns the hardcode repositoryType for different backup methods - Restic or Kopia,uploaderType -// indicates the method. -// For Restic backup method, it is always hardcode to BackupRepositoryTypeRestic, never changed. -// For Kopia backup method, this means we hardcode repositoryType as BackupRepositoryTypeKopia for Unified Repo, -// at present (Kopia backup method is using Unified Repo). However, it doesn't mean we could deduce repositoryType -// from uploaderType for Unified Repo. -// TODO: post v1.10, refactor this function for Kopia backup method. In future, when we have multiple implementations of -// Unified Repo (besides Kopia), we will add the repositoryType to BSL, because by then, we are not able to hardcode -// the repositoryType to BackupRepositoryTypeKopia for Unified Repo. -func getRepositoryType(uploaderType string) string { - switch uploaderType { - case "", uploader.ResticType: - return velerov1api.BackupRepositoryTypeRestic - case uploader.KopiaType: - return velerov1api.BackupRepositoryTypeKopia - default: - return "" - } +// getRepositoryType returns the hardcode repositoryType. +// TODO: In future, when we have multiple implementations of Unified Repo (besides Kopia), we will add the repositoryType to BSL, +// because by then, we are not able to hardcode the repositoryType to BackupRepositoryTypeKopia for Unified Repo. +func getRepositoryType() string { + return velerov1api.BackupRepositoryTypeKopia } func isPVBMatchPod(pvb *velerov1api.PodVolumeBackup, podName string, namespace string) bool { diff --git a/pkg/repository/config/config.go b/pkg/repository/config/config.go index 46a5478e6..04761e95b 100644 --- a/pkg/repository/config/config.go +++ b/pkg/repository/config/config.go @@ -17,14 +17,7 @@ limitations under the License. package config import ( - "fmt" - "path" "strings" - - "github.com/pkg/errors" - - velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - "github.com/vmware-tanzu/velero/pkg/persistence" ) type BackendType string @@ -40,56 +33,6 @@ const ( CredentialsFileKey = "credentialsFile" ) -// this func is assigned to a package-level variable so it can be -// replaced when unit-testing -var getAWSBucketRegion = GetAWSBucketRegion - -// getRepoPrefix returns the prefix of the value of the --repo flag for -// restic commands, i.e. everything except the "/". -func getRepoPrefix(location *velerov1api.BackupStorageLocation) (string, error) { - var bucket, prefix string - - if location.Spec.ObjectStorage != nil { - layout := persistence.NewObjectStoreLayout(location.Spec.ObjectStorage.Prefix) - - bucket = location.Spec.ObjectStorage.Bucket - prefix = layout.GetResticDir() - } - - backendType := GetBackendType(location.Spec.Provider, location.Spec.Config) - - if repoPrefix := location.Spec.Config["resticRepoPrefix"]; repoPrefix != "" { - return repoPrefix, nil - } - - switch backendType { - case AWSBackend: - var url string - // non-AWS, S3-compatible object store - if s3Url := location.Spec.Config["s3Url"]; s3Url != "" { - url = strings.TrimSuffix(s3Url, "/") - } else { - var err error - region := location.Spec.Config["region"] - if region == "" { - region, err = getAWSBucketRegion(bucket, location.Spec.Config) - } - if err != nil { - return "", errors.Wrapf(err, "failed to detect the region via bucket: %s", bucket) - } - url = fmt.Sprintf("s3-%s.amazonaws.com", region) - } - - return fmt.Sprintf("s3:%s/%s", url, path.Join(bucket, prefix)), nil - case AzureBackend: - return fmt.Sprintf("azure:%s:/%s", bucket, prefix), nil - case GCPBackend: - return fmt.Sprintf("gs:%s:/%s", bucket, prefix), nil - } - - return "", errors.Errorf("invalid backend type %s, provider %s", backendType, location.Spec.Provider) -} - // GetBackendType returns a backend type that is known by Velero. // If the provider doesn't indicate a known backend type, but the endpoint is // specified, Velero regards it as a S3 compatible object store and return AWSBackend as the type. @@ -111,14 +54,3 @@ func GetBackendType(provider string, config map[string]string) BackendType { func IsBackendTypeValid(backendType BackendType) bool { return (backendType == AWSBackend || backendType == AzureBackend || backendType == GCPBackend || backendType == FSBackend) } - -// GetRepoIdentifier returns the string to be used as the value of the --repo flag in -// restic commands for the given repository. -func GetRepoIdentifier(location *velerov1api.BackupStorageLocation, name string) (string, error) { - prefix, err := getRepoPrefix(location) - if err != nil { - return "", err - } - - return fmt.Sprintf("%s/%s", strings.TrimSuffix(prefix, "/"), name), nil -} diff --git a/pkg/repository/config/config_test.go b/pkg/repository/config/config_test.go deleted file mode 100644 index aac5fc9bc..000000000 --- a/pkg/repository/config/config_test.go +++ /dev/null @@ -1,261 +0,0 @@ -/* -Copyright 2018, 2019 the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" -) - -func TestGetRepoIdentifier(t *testing.T) { - testCases := []struct { - name string - bsl *velerov1api.BackupStorageLocation - repoName string - getAWSBucketRegion func(s string, config map[string]string) (string, error) - expected string - expectedErr string - }{ - { - name: "error is returned if BSL uses unsupported provider and resticRepoPrefix is not set", - bsl: &velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "unsupported-provider", - StorageType: velerov1api.StorageType{ - ObjectStorage: &velerov1api.ObjectStorageLocation{ - Bucket: "bucket-2", - Prefix: "prefix-2", - }, - }, - }, - }, - repoName: "repo-1", - expectedErr: "invalid backend type velero.io/unsupported-provider, provider unsupported-provider", - }, - { - name: "resticRepoPrefix in BSL config is used if set", - bsl: &velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "custom-repo-identifier", - Config: map[string]string{ - "resticRepoPrefix": "custom:prefix:/restic", - }, - StorageType: velerov1api.StorageType{ - ObjectStorage: &velerov1api.ObjectStorageLocation{ - Bucket: "bucket", - Prefix: "prefix", - }, - }, - }, - }, - repoName: "repo-1", - expected: "custom:prefix:/restic/repo-1", - }, - { - name: "s3Url in BSL config is used", - bsl: &velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "custom-repo-identifier", - Config: map[string]string{ - "s3Url": "s3Url", - }, - StorageType: velerov1api.StorageType{ - ObjectStorage: &velerov1api.ObjectStorageLocation{ - Bucket: "bucket", - Prefix: "prefix", - }, - }, - }, - }, - repoName: "repo-1", - expected: "s3:s3Url/bucket/prefix/restic/repo-1", - }, - { - name: "s3.amazonaws.com URL format is used if region cannot be determined for AWS BSL", - bsl: &velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "aws", - StorageType: velerov1api.StorageType{ - ObjectStorage: &velerov1api.ObjectStorageLocation{ - Bucket: "bucket", - }, - }, - }, - }, - repoName: "repo-1", - getAWSBucketRegion: func(s string, config map[string]string) (string, error) { - return "", errors.New("no region found") - }, - expected: "", - expectedErr: "failed to detect the region via bucket: bucket: no region found", - }, - { - name: "s3.s3-.amazonaws.com URL format is used if region can be determined for AWS BSL", - bsl: &velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "aws", - StorageType: velerov1api.StorageType{ - ObjectStorage: &velerov1api.ObjectStorageLocation{ - Bucket: "bucket", - }, - }, - }, - }, - repoName: "repo-1", - getAWSBucketRegion: func(string, map[string]string) (string, error) { - return "eu-west-1", nil - }, - expected: "s3:s3-eu-west-1.amazonaws.com/bucket/restic/repo-1", - }, - { - name: "prefix is included in repo identifier if set for AWS BSL", - bsl: &velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "aws", - StorageType: velerov1api.StorageType{ - ObjectStorage: &velerov1api.ObjectStorageLocation{ - Bucket: "bucket", - Prefix: "prefix", - }, - }, - }, - }, - repoName: "repo-1", - getAWSBucketRegion: func(s string, config map[string]string) (string, error) { - return "eu-west-1", nil - }, - expected: "s3:s3-eu-west-1.amazonaws.com/bucket/prefix/restic/repo-1", - }, - { - name: "s3Url is used in repo identifier if set for AWS BSL", - bsl: &velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "aws", - Config: map[string]string{ - "s3Url": "alternate-url", - }, - StorageType: velerov1api.StorageType{ - ObjectStorage: &velerov1api.ObjectStorageLocation{ - Bucket: "bucket", - Prefix: "prefix", - }, - }, - }, - }, - repoName: "repo-1", - getAWSBucketRegion: func(s string, config map[string]string) (string, error) { - return "eu-west-1", nil - }, - expected: "s3:alternate-url/bucket/prefix/restic/repo-1", - }, - { - name: "region is used in repo identifier if set for AWS BSL", - bsl: &velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "aws", - Config: map[string]string{ - "region": "us-west-1", - }, - StorageType: velerov1api.StorageType{ - ObjectStorage: &velerov1api.ObjectStorageLocation{ - Bucket: "bucket", - Prefix: "prefix", - }, - }, - }, - }, - repoName: "aws-repo", - getAWSBucketRegion: func(s string, config map[string]string) (string, error) { - return "eu-west-1", nil - }, - expected: "s3:s3-us-west-1.amazonaws.com/bucket/prefix/restic/aws-repo", - }, - { - name: "trailing slash in s3Url is not included in repo identifier for AWS BSL", - bsl: &velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "aws", - Config: map[string]string{ - "s3Url": "alternate-url-with-trailing-slash/", - }, - StorageType: velerov1api.StorageType{ - ObjectStorage: &velerov1api.ObjectStorageLocation{ - Bucket: "bucket", - Prefix: "prefix", - }, - }, - }, - }, - repoName: "aws-repo", - getAWSBucketRegion: func(s string, config map[string]string) (string, error) { - return "eu-west-1", nil - }, - expected: "s3:alternate-url-with-trailing-slash/bucket/prefix/restic/aws-repo", - }, - { - name: "repo identifier includes bucket and prefix for Azure BSL", - bsl: &velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "azure", - StorageType: velerov1api.StorageType{ - ObjectStorage: &velerov1api.ObjectStorageLocation{ - Bucket: "azure-bucket", - Prefix: "azure-prefix", - }, - }, - }, - }, - repoName: "azure-repo", - expected: "azure:azure-bucket:/azure-prefix/restic/azure-repo", - }, - { - name: "repo identifier includes bucket and prefix for GCP BSL", - bsl: &velerov1api.BackupStorageLocation{ - Spec: velerov1api.BackupStorageLocationSpec{ - Provider: "gcp", - StorageType: velerov1api.StorageType{ - ObjectStorage: &velerov1api.ObjectStorageLocation{ - Bucket: "gcp-bucket", - Prefix: "gcp-prefix", - }, - }, - }, - }, - repoName: "gcp-repo", - expected: "gs:gcp-bucket:/gcp-prefix/restic/gcp-repo", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - getAWSBucketRegion = tc.getAWSBucketRegion - id, err := GetRepoIdentifier(tc.bsl, tc.repoName) - assert.Equal(t, tc.expected, id) - if tc.expectedErr == "" { - assert.NoError(t, err) - } else { - require.EqualError(t, err, tc.expectedErr) - assert.Empty(t, id) - } - }) - } -} diff --git a/pkg/repository/maintenance/maintenance.go b/pkg/repository/maintenance/maintenance.go index 496d07703..747d89f52 100644 --- a/pkg/repository/maintenance/maintenance.go +++ b/pkg/repository/maintenance/maintenance.go @@ -38,6 +38,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/constant" velerolabel "github.com/vmware-tanzu/velero/pkg/label" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util" @@ -574,15 +575,32 @@ func buildJob( // Set resource limits and requests cpuRequest := DefaultMaintenanceJobCPURequest memRequest := DefaultMaintenanceJobMemRequest + ephemeralStorageRequest := constant.DefaultEphemeralStorageRequest cpuLimit := DefaultMaintenanceJobCPULimit memLimit := DefaultMaintenanceJobMemLimit + ephemeralStorageLimit := constant.DefaultEphemeralStorageLimit if config != nil && config.PodResources != nil { cpuRequest = config.PodResources.CPURequest memRequest = config.PodResources.MemoryRequest cpuLimit = config.PodResources.CPULimit memLimit = config.PodResources.MemoryLimit + // To make the PodResources ConfigMap without ephemeral storage request/limit backward compatible, + // need to avoid set value as empty, because empty string will cause parsing error. + if config.PodResources.EphemeralStorageRequest != "" { + ephemeralStorageRequest = config.PodResources.EphemeralStorageRequest + } + if config.PodResources.EphemeralStorageLimit != "" { + ephemeralStorageLimit = config.PodResources.EphemeralStorageLimit + } } - resources, err := kube.ParseResourceRequirements(cpuRequest, memRequest, cpuLimit, memLimit) + resources, err := kube.ParseResourceRequirements( + cpuRequest, + memRequest, + ephemeralStorageRequest, + cpuLimit, + memLimit, + ephemeralStorageLimit, + ) if err != nil { return nil, errors.Wrap(err, "failed to parse resource requirements for maintenance job") } @@ -671,8 +689,7 @@ func buildJob( } if config != nil && len(config.LoadAffinities) > 0 { - // Maintenance job only takes the first loadAffinity. - affinity := kube.ToSystemAffinity([]*kube.LoadAffinity{config.LoadAffinities[0]}) + affinity := kube.ToSystemAffinity(config.LoadAffinities[0], nil) job.Spec.Template.Spec.Affinity = affinity } diff --git a/pkg/repository/manager/manager.go b/pkg/repository/manager/manager.go index abe76299f..4d03931c0 100644 --- a/pkg/repository/manager/manager.go +++ b/pkg/repository/manager/manager.go @@ -109,10 +109,6 @@ func NewManager( log: log, } - mgr.providers[velerov1api.BackupRepositoryTypeRestic] = provider.NewResticRepositoryProvider(credentials.CredentialGetter{ - FromFile: credentialFileStore, - FromSecret: credentialSecretStore, - }, mgr.fileSystem, mgr.log) mgr.providers[velerov1api.BackupRepositoryTypeKopia] = provider.NewUnifiedRepoProvider(credentials.CredentialGetter{ FromFile: credentialFileStore, FromSecret: credentialSecretStore, @@ -275,8 +271,6 @@ func (m *manager) ClientSideCacheLimit(repo *velerov1api.BackupRepository) (int6 func (m *manager) getRepositoryProvider(repo *velerov1api.BackupRepository) (provider.Provider, error) { switch repo.Spec.RepositoryType { - case "", velerov1api.BackupRepositoryTypeRestic: - return m.providers[velerov1api.BackupRepositoryTypeRestic], nil case velerov1api.BackupRepositoryTypeKopia: return m.providers[velerov1api.BackupRepositoryTypeKopia], nil default: diff --git a/pkg/repository/manager/manager_test.go b/pkg/repository/manager/manager_test.go index ea38c7448..88d8db481 100644 --- a/pkg/repository/manager/manager_test.go +++ b/pkg/repository/manager/manager_test.go @@ -32,15 +32,13 @@ func TestGetRepositoryProvider(t *testing.T) { repo := &velerov1.BackupRepository{} // empty repository type - provider, err := mgr.getRepositoryProvider(repo) - require.NoError(t, err) - assert.NotNil(t, provider) + _, err := mgr.getRepositoryProvider(repo) + require.Error(t, err) - // valid repository type - repo.Spec.RepositoryType = velerov1.BackupRepositoryTypeRestic - provider, err = mgr.getRepositoryProvider(repo) - require.NoError(t, err) - assert.NotNil(t, provider) + // invalid repository type + repo.Spec.RepositoryType = "restic" + _, err = mgr.getRepositoryProvider(repo) + require.Error(t, err) // invalid repository type repo.Spec.RepositoryType = "unknown" @@ -61,6 +59,6 @@ func TestGetRepositoryConfigProvider(t *testing.T) { assert.NotNil(t, provider) // invalid repository type - _, err = mgr.getRepositoryProvider(velerov1.BackupRepositoryTypeRestic) + _, err = mgr.getRepositoryProvider("restic") require.Error(t, err) } diff --git a/pkg/repository/provider/restic.go b/pkg/repository/provider/restic.go deleted file mode 100644 index 51a46bbf1..000000000 --- a/pkg/repository/provider/restic.go +++ /dev/null @@ -1,99 +0,0 @@ -/* -Copyright the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package provider - -import ( - "context" - "strings" - "time" - - "github.com/sirupsen/logrus" - - "github.com/vmware-tanzu/velero/internal/credentials" - "github.com/vmware-tanzu/velero/pkg/repository/restic" - "github.com/vmware-tanzu/velero/pkg/util/filesystem" -) - -func NewResticRepositoryProvider(credGetter credentials.CredentialGetter, fs filesystem.Interface, log logrus.FieldLogger) Provider { - return &resticRepositoryProvider{ - svc: restic.NewRepositoryService(credGetter, fs, log), - } -} - -type resticRepositoryProvider struct { - svc *restic.RepositoryService -} - -func (r *resticRepositoryProvider) InitRepo(ctx context.Context, param RepoParam) error { - return r.svc.InitRepo(param.BackupLocation, param.BackupRepo) -} - -func (r *resticRepositoryProvider) ConnectToRepo(ctx context.Context, param RepoParam) error { - return r.svc.ConnectToRepo(param.BackupLocation, param.BackupRepo) -} - -func (r *resticRepositoryProvider) PrepareRepo(ctx context.Context, param RepoParam) error { - if err := r.ConnectToRepo(ctx, param); err != nil { - // If the repository has not yet been initialized, the error message will always include - // the following string. This is the only scenario where we should try to initialize it. - // Other errors (e.g. "already locked") should be returned as-is since the repository - // does already exist, but it can't be connected to. - if strings.Contains(err.Error(), "Is there a repository at the following location?") { - return r.InitRepo(ctx, param) - } - - return err - } - - return nil -} - -func (r *resticRepositoryProvider) BoostRepoConnect(ctx context.Context, param RepoParam) error { - return nil -} - -func (r *resticRepositoryProvider) PruneRepo(ctx context.Context, param RepoParam) error { - return r.svc.PruneRepo(param.BackupLocation, param.BackupRepo) -} - -func (r *resticRepositoryProvider) EnsureUnlockRepo(ctx context.Context, param RepoParam) error { - return r.svc.UnlockRepo(param.BackupLocation, param.BackupRepo) -} - -func (r *resticRepositoryProvider) Forget(ctx context.Context, snapshotID string, param RepoParam) error { - return r.svc.Forget(param.BackupLocation, param.BackupRepo, snapshotID) -} - -func (r *resticRepositoryProvider) BatchForget(ctx context.Context, snapshotIDs []string, param RepoParam) []error { - errs := []error{} - for _, snapshot := range snapshotIDs { - err := r.Forget(ctx, snapshot, param) - if err != nil { - errs = append(errs, err) - } - } - - return errs -} - -func (r *resticRepositoryProvider) DefaultMaintenanceFrequency() time.Duration { - return r.svc.DefaultMaintenanceFrequency() -} - -func (r *resticRepositoryProvider) ClientSideCacheLimit(repoOption map[string]string) int64 { - return 0 -} diff --git a/pkg/repository/provider/unified_repo.go b/pkg/repository/provider/unified_repo.go index 8d764f0e9..2af59f190 100644 --- a/pkg/repository/provider/unified_repo.go +++ b/pkg/repository/provider/unified_repo.go @@ -208,14 +208,16 @@ func (urp *unifiedRepoProvider) PrepareRepo(ctx context.Context, param RepoParam return errors.Wrap(err, "error to get repo options") } - if created, err := urp.repoService.IsCreated(ctx, *repoOption); err != nil { + readOnly := (param.BackupLocation.Spec.AccessMode == velerov1api.BackupStorageLocationAccessModeReadOnly) + + if ready, err := urp.repoService.IsReady(ctx, *repoOption, readOnly); err != nil { return errors.Wrap(err, "error to check backup repo") - } else if created { + } else if ready { log.Info("Repo has already been initialized") return nil } - if param.BackupLocation.Spec.AccessMode == velerov1api.BackupStorageLocationAccessModeReadOnly { + if readOnly { return errors.Errorf("cannot create new backup repo for read-only backup storage location %s/%s", param.BackupLocation.Namespace, param.BackupLocation.Name) } diff --git a/pkg/repository/provider/unified_repo_test.go b/pkg/repository/provider/unified_repo_test.go index 80ed55eab..e0e0a8b8f 100644 --- a/pkg/repository/provider/unified_repo_test.go +++ b/pkg/repository/provider/unified_repo_test.go @@ -613,7 +613,7 @@ func TestPrepareRepo(t *testing.T) { getter *credmock.SecretStore repoService *reposervicenmocks.BackupRepoService retFuncCreate func(context.Context, udmrepo.RepoOptions) error - retFuncCheck func(context.Context, udmrepo.RepoOptions) (bool, error) + retFuncCheck func(context.Context, udmrepo.RepoOptions, bool) (bool, error) credStoreReturn string credStoreError error readOnlyBSL bool @@ -656,7 +656,7 @@ func TestPrepareRepo(t *testing.T) { }, }, repoService: new(reposervicenmocks.BackupRepoService), - retFuncCheck: func(ctx context.Context, repoOption udmrepo.RepoOptions) (bool, error) { + retFuncCheck: func(ctx context.Context, repoOption udmrepo.RepoOptions, readOnly bool) (bool, error) { return false, errors.New("fake-error") }, expectedErr: "error to check backup repo: fake-error", @@ -674,7 +674,7 @@ func TestPrepareRepo(t *testing.T) { }, }, repoService: new(reposervicenmocks.BackupRepoService), - retFuncCheck: func(ctx context.Context, repoOption udmrepo.RepoOptions) (bool, error) { + retFuncCheck: func(ctx context.Context, repoOption udmrepo.RepoOptions, readOnly bool) (bool, error) { return true, nil }, retFuncCreate: func(ctx context.Context, repoOption udmrepo.RepoOptions) error { @@ -695,7 +695,7 @@ func TestPrepareRepo(t *testing.T) { }, }, repoService: new(reposervicenmocks.BackupRepoService), - retFuncCheck: func(ctx context.Context, repoOption udmrepo.RepoOptions) (bool, error) { + retFuncCheck: func(ctx context.Context, repoOption udmrepo.RepoOptions, readOnly bool) (bool, error) { return false, nil }, retFuncCreate: func(ctx context.Context, repoOption udmrepo.RepoOptions) error { @@ -716,7 +716,7 @@ func TestPrepareRepo(t *testing.T) { }, }, repoService: new(reposervicenmocks.BackupRepoService), - retFuncCheck: func(ctx context.Context, repoOption udmrepo.RepoOptions) (bool, error) { + retFuncCheck: func(ctx context.Context, repoOption udmrepo.RepoOptions, readOnly bool) (bool, error) { return false, nil }, retFuncCreate: func(ctx context.Context, repoOption udmrepo.RepoOptions) error { @@ -737,7 +737,7 @@ func TestPrepareRepo(t *testing.T) { }, }, repoService: new(reposervicenmocks.BackupRepoService), - retFuncCheck: func(ctx context.Context, repoOption udmrepo.RepoOptions) (bool, error) { + retFuncCheck: func(ctx context.Context, repoOption udmrepo.RepoOptions, readOnly bool) (bool, error) { return false, nil }, retFuncCreate: func(ctx context.Context, repoOption udmrepo.RepoOptions) error { @@ -758,7 +758,7 @@ func TestPrepareRepo(t *testing.T) { }, }, repoService: new(reposervicenmocks.BackupRepoService), - retFuncCheck: func(ctx context.Context, repoOption udmrepo.RepoOptions) (bool, error) { + retFuncCheck: func(ctx context.Context, repoOption udmrepo.RepoOptions, readOnly bool) (bool, error) { return false, nil }, retFuncCreate: func(ctx context.Context, repoOption udmrepo.RepoOptions) error { @@ -785,7 +785,7 @@ func TestPrepareRepo(t *testing.T) { log: velerotest.NewLogger(), } - tc.repoService.On("IsCreated", mock.Anything, mock.Anything).Return(tc.retFuncCheck) + tc.repoService.On("IsReady", mock.Anything, mock.Anything, mock.Anything).Return(tc.retFuncCheck) tc.repoService.On("Create", mock.Anything, mock.Anything, mock.Anything).Return(tc.retFuncCreate) if tc.readOnlyBSL { diff --git a/pkg/repository/restic/repository.go b/pkg/repository/restic/repository.go deleted file mode 100644 index 7260542e1..000000000 --- a/pkg/repository/restic/repository.go +++ /dev/null @@ -1,147 +0,0 @@ -/* -Copyright the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package restic - -import ( - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - - "github.com/vmware-tanzu/velero/internal/credentials" - velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - repokey "github.com/vmware-tanzu/velero/pkg/repository/keys" - "github.com/vmware-tanzu/velero/pkg/restic" - veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" - "github.com/vmware-tanzu/velero/pkg/util/filesystem" -) - -func NewRepositoryService(credGetter credentials.CredentialGetter, fs filesystem.Interface, log logrus.FieldLogger) *RepositoryService { - return &RepositoryService{ - credGetter: credGetter, - fileSystem: fs, - log: log, - } -} - -type RepositoryService struct { - credGetter credentials.CredentialGetter - fileSystem filesystem.Interface - log logrus.FieldLogger -} - -func (r *RepositoryService) InitRepo(bsl *velerov1api.BackupStorageLocation, repo *velerov1api.BackupRepository) error { - return r.exec(restic.InitCommand(repo.Spec.ResticIdentifier), bsl) -} - -func (r *RepositoryService) ConnectToRepo(bsl *velerov1api.BackupStorageLocation, repo *velerov1api.BackupRepository) error { - snapshotsCmd := restic.SnapshotsCommand(repo.Spec.ResticIdentifier) - // use the '--latest=1' flag to minimize the amount of data fetched since - // we're just validating that the repo exists and can be authenticated - // to. - // "--last" is replaced by "--latest=1" in restic v0.12.1 - snapshotsCmd.ExtraFlags = append(snapshotsCmd.ExtraFlags, "--latest=1") - - return r.exec(snapshotsCmd, bsl) -} - -func (r *RepositoryService) PruneRepo(bsl *velerov1api.BackupStorageLocation, repo *velerov1api.BackupRepository) error { - return r.exec(restic.PruneCommand(repo.Spec.ResticIdentifier), bsl) -} - -func (r *RepositoryService) UnlockRepo(bsl *velerov1api.BackupStorageLocation, repo *velerov1api.BackupRepository) error { - return r.exec(restic.UnlockCommand(repo.Spec.ResticIdentifier), bsl) -} - -func (r *RepositoryService) Forget(bsl *velerov1api.BackupStorageLocation, repo *velerov1api.BackupRepository, snapshotID string) error { - return r.exec(restic.ForgetCommand(repo.Spec.ResticIdentifier, snapshotID), bsl) -} - -func (r *RepositoryService) DefaultMaintenanceFrequency() time.Duration { - return restic.DefaultMaintenanceFrequency -} - -func (r *RepositoryService) exec(cmd *restic.Command, bsl *velerov1api.BackupStorageLocation) error { - file, err := r.credGetter.FromFile.Path(repokey.RepoKeySelector()) - if err != nil { - return err - } - // ignore error since there's nothing we can do and it's a temp file. - defer os.Remove(file) - - cmd.PasswordFile = file - - // if there's a caCert on the ObjectStorage, write it to disk so that it can be passed to restic - var caCertFile string - if bsl.Spec.ObjectStorage != nil { - var caCertData []byte - - // Try CACertRef first (new method), then fall back to CACert (deprecated) - if bsl.Spec.ObjectStorage.CACertRef != nil { - caCertString, err := r.credGetter.FromSecret.Get(bsl.Spec.ObjectStorage.CACertRef) - if err != nil { - return errors.Wrap(err, "error getting CA certificate from secret") - } - caCertData = []byte(caCertString) - } else if bsl.Spec.ObjectStorage.CACert != nil { - caCertData = bsl.Spec.ObjectStorage.CACert - } - - if caCertData != nil { - caCertFile, err = restic.TempCACertFile(caCertData, bsl.Name, r.fileSystem) - if err != nil { - return errors.Wrap(err, "error creating temp cacert file") - } - // ignore error since there's nothing we can do and it's a temp file. - defer os.Remove(caCertFile) - } - } - cmd.CACertFile = caCertFile - - // CmdEnv uses credGetter.FromFile (not FromSecret) to get cloud provider credentials. - // FromFile materializes the BSL's Credential secret to a file path that cloud SDKs - // can read (e.g., AWS_SHARED_CREDENTIALS_FILE). This is different from caCertRef above, - // which uses FromSecret to read the CA certificate data directly into memory, then - // writes it to a temp file because restic CLI only accepts file paths (--cacert flag). - env, err := restic.CmdEnv(bsl, r.credGetter.FromFile) - if err != nil { - return err - } - cmd.Env = env - - // #4820: restrieve insecureSkipTLSVerify from BSL configuration for - // AWS plugin. If nothing is return, that means insecureSkipTLSVerify - // is not enable for Restic command. - skipTLSRet := restic.GetInsecureSkipTLSVerifyFromBSL(bsl, r.log) - if len(skipTLSRet) > 0 { - cmd.ExtraFlags = append(cmd.ExtraFlags, skipTLSRet) - } - - stdout, stderr, err := veleroexec.RunCommandWithLog(cmd.Cmd(), r.log) - r.log.WithFields(logrus.Fields{ - "repository": cmd.RepoName(), - "command": cmd.String(), - "stdout": stdout, - "stderr": stderr, - }).Debugf("Ran restic command") - if err != nil { - return errors.Wrapf(err, "error running command=%s, stdout=%s, stderr=%s", cmd.String(), stdout, stderr) - } - - return nil -} diff --git a/pkg/repository/udmrepo/kopialib/lib_repo.go b/pkg/repository/udmrepo/kopialib/lib_repo.go index 18d2cea50..e6c46ae66 100644 --- a/pkg/repository/udmrepo/kopialib/lib_repo.go +++ b/pkg/repository/udmrepo/kopialib/lib_repo.go @@ -129,20 +129,28 @@ func (ks *kopiaRepoService) Connect(ctx context.Context, repoOption udmrepo.Repo return ConnectBackupRepo(repoCtx, repoOption, ks.logger) } -func (ks *kopiaRepoService) IsCreated(ctx context.Context, repoOption udmrepo.RepoOptions) (bool, error) { +var funcGetRepositoryStatus = GetRepositoryStatus + +func (ks *kopiaRepoService) IsReady(ctx context.Context, repoOption udmrepo.RepoOptions, readOnly bool) (bool, error) { repoCtx := kopia.SetupKopiaLog(ctx, ks.logger) - status, err := GetRepositoryStatus(repoCtx, repoOption, ks.logger) + status, err := funcGetRepositoryStatus(repoCtx, repoOption, ks.logger) if err != nil { return false, err } - if status != RepoStatusCreated { - ks.logger.Infof("Repo is not fully created, status %v", status) - return false, nil + if status == RepoStatusCreated { + return true, nil } - return true, nil + if status == RepoStatusNotInitialized && readOnly { + ks.logger.Warnf("Repo is not initialized, could be for read") + return true, nil + } + + ks.logger.Infof("Repo is not fully created, status %v", status) + + return false, nil } func (ks *kopiaRepoService) Open(ctx context.Context, repoOption udmrepo.RepoOptions) (udmrepo.BackupRepo, error) { diff --git a/pkg/repository/udmrepo/kopialib/lib_repo_test.go b/pkg/repository/udmrepo/kopialib/lib_repo_test.go index b80243423..2feabaeca 100644 --- a/pkg/repository/udmrepo/kopialib/lib_repo_test.go +++ b/pkg/repository/udmrepo/kopialib/lib_repo_test.go @@ -27,6 +27,7 @@ import ( "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/repo/object" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -1197,3 +1198,79 @@ func TestClientSideCacheLimit(t *testing.T) { }) } } + +func TestIsReady(t *testing.T) { + testCases := []struct { + name string + funGetStatus func(context.Context, udmrepo.RepoOptions, logrus.FieldLogger) (RepoStatus, error) + readOnly bool + expected bool + expectedErr string + }{ + { + name: "get status error", + funGetStatus: func(context.Context, udmrepo.RepoOptions, logrus.FieldLogger) (RepoStatus, error) { + return RepoStatusUnknown, errors.New("fake-get-error") + }, + expectedErr: "fake-get-error", + }, + { + name: "success", + funGetStatus: func(context.Context, udmrepo.RepoOptions, logrus.FieldLogger) (RepoStatus, error) { + return RepoStatusCreated, nil + }, + expected: true, + }, + { + name: "not initialized, not readonly", + funGetStatus: func(context.Context, udmrepo.RepoOptions, logrus.FieldLogger) (RepoStatus, error) { + return RepoStatusNotInitialized, nil + }, + }, + { + name: "not initialized, readonly", + funGetStatus: func(context.Context, udmrepo.RepoOptions, logrus.FieldLogger) (RepoStatus, error) { + return RepoStatusNotInitialized, nil + }, + readOnly: true, + expected: true, + }, + { + name: "other status 1", + funGetStatus: func(context.Context, udmrepo.RepoOptions, logrus.FieldLogger) (RepoStatus, error) { + return RepoStatusUnknown, nil + }, + }, + { + name: "other status 2", + funGetStatus: func(context.Context, udmrepo.RepoOptions, logrus.FieldLogger) (RepoStatus, error) { + return RepoStatusCorrupted, nil + }, + }, + { + name: "other status 3", + funGetStatus: func(context.Context, udmrepo.RepoOptions, logrus.FieldLogger) (RepoStatus, error) { + return RepoStatusSystemNotCreated, nil + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ks := &kopiaRepoService{ + logger: velerotest.NewLogger(), + } + + funcGetRepositoryStatus = tc.funGetStatus + ready, err := ks.IsReady(t.Context(), udmrepo.RepoOptions{}, tc.readOnly) + + if tc.expectedErr != "" { + require.EqualError(t, err, tc.expectedErr) + } else { + require.NoError(t, err) + } + + require.Equal(t, tc.expected, ready) + }) + } +} diff --git a/pkg/repository/udmrepo/mocks/BackupRepoService.go b/pkg/repository/udmrepo/mocks/BackupRepoService.go index c9f46e32f..34a24ef97 100644 --- a/pkg/repository/udmrepo/mocks/BackupRepoService.go +++ b/pkg/repository/udmrepo/mocks/BackupRepoService.go @@ -1,169 +1,17 @@ -// Code generated by mockery v2.53.2. DO NOT EDIT. +// Code generated by mockery; DO NOT EDIT. +// github.com/vektra/mockery +// template: testify package mocks import ( - context "context" - time "time" + "context" + "time" mock "github.com/stretchr/testify/mock" - - udmrepo "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" + "github.com/vmware-tanzu/velero/pkg/repository/udmrepo" ) -// BackupRepoService is an autogenerated mock type for the BackupRepoService type -type BackupRepoService struct { - mock.Mock -} - -// ClientSideCacheLimit provides a mock function with given fields: repoOption -func (_m *BackupRepoService) ClientSideCacheLimit(repoOption map[string]string) int64 { - ret := _m.Called(repoOption) - - if len(ret) == 0 { - panic("no return value specified for ClientSideCacheLimit") - } - - var r0 int64 - if rf, ok := ret.Get(0).(func(map[string]string) int64); ok { - r0 = rf(repoOption) - } else { - r0 = ret.Get(0).(int64) - } - - return r0 -} - -// Connect provides a mock function with given fields: ctx, repoOption -func (_m *BackupRepoService) Connect(ctx context.Context, repoOption udmrepo.RepoOptions) error { - ret := _m.Called(ctx, repoOption) - - if len(ret) == 0 { - panic("no return value specified for Connect") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, udmrepo.RepoOptions) error); ok { - r0 = rf(ctx, repoOption) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Create provides a mock function with given fields: ctx, repoOption -func (_m *BackupRepoService) Create(ctx context.Context, repoOption udmrepo.RepoOptions) error { - ret := _m.Called(ctx, repoOption) - - if len(ret) == 0 { - panic("no return value specified for Create") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, udmrepo.RepoOptions) error); ok { - r0 = rf(ctx, repoOption) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// DefaultMaintenanceFrequency provides a mock function with no fields -func (_m *BackupRepoService) DefaultMaintenanceFrequency() time.Duration { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for DefaultMaintenanceFrequency") - } - - var r0 time.Duration - if rf, ok := ret.Get(0).(func() time.Duration); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(time.Duration) - } - - return r0 -} - -// IsCreated provides a mock function with given fields: ctx, repoOption -func (_m *BackupRepoService) IsCreated(ctx context.Context, repoOption udmrepo.RepoOptions) (bool, error) { - ret := _m.Called(ctx, repoOption) - - if len(ret) == 0 { - panic("no return value specified for IsCreated") - } - - var r0 bool - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, udmrepo.RepoOptions) (bool, error)); ok { - return rf(ctx, repoOption) - } - if rf, ok := ret.Get(0).(func(context.Context, udmrepo.RepoOptions) bool); ok { - r0 = rf(ctx, repoOption) - } else { - r0 = ret.Get(0).(bool) - } - - if rf, ok := ret.Get(1).(func(context.Context, udmrepo.RepoOptions) error); ok { - r1 = rf(ctx, repoOption) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Maintain provides a mock function with given fields: ctx, repoOption -func (_m *BackupRepoService) Maintain(ctx context.Context, repoOption udmrepo.RepoOptions) error { - ret := _m.Called(ctx, repoOption) - - if len(ret) == 0 { - panic("no return value specified for Maintain") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, udmrepo.RepoOptions) error); ok { - r0 = rf(ctx, repoOption) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Open provides a mock function with given fields: ctx, repoOption -func (_m *BackupRepoService) Open(ctx context.Context, repoOption udmrepo.RepoOptions) (udmrepo.BackupRepo, error) { - ret := _m.Called(ctx, repoOption) - - if len(ret) == 0 { - panic("no return value specified for Open") - } - - var r0 udmrepo.BackupRepo - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, udmrepo.RepoOptions) (udmrepo.BackupRepo, error)); ok { - return rf(ctx, repoOption) - } - if rf, ok := ret.Get(0).(func(context.Context, udmrepo.RepoOptions) udmrepo.BackupRepo); ok { - r0 = rf(ctx, repoOption) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(udmrepo.BackupRepo) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, udmrepo.RepoOptions) error); ok { - r1 = rf(ctx, repoOption) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // NewBackupRepoService creates a new instance of BackupRepoService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewBackupRepoService(t interface { @@ -177,3 +25,422 @@ func NewBackupRepoService(t interface { return mock } + +// BackupRepoService is an autogenerated mock type for the BackupRepoService type +type BackupRepoService struct { + mock.Mock +} + +type BackupRepoService_Expecter struct { + mock *mock.Mock +} + +func (_m *BackupRepoService) EXPECT() *BackupRepoService_Expecter { + return &BackupRepoService_Expecter{mock: &_m.Mock} +} + +// ClientSideCacheLimit provides a mock function for the type BackupRepoService +func (_mock *BackupRepoService) ClientSideCacheLimit(repoOption map[string]string) int64 { + ret := _mock.Called(repoOption) + + if len(ret) == 0 { + panic("no return value specified for ClientSideCacheLimit") + } + + var r0 int64 + if returnFunc, ok := ret.Get(0).(func(map[string]string) int64); ok { + r0 = returnFunc(repoOption) + } else { + r0 = ret.Get(0).(int64) + } + return r0 +} + +// BackupRepoService_ClientSideCacheLimit_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ClientSideCacheLimit' +type BackupRepoService_ClientSideCacheLimit_Call struct { + *mock.Call +} + +// ClientSideCacheLimit is a helper method to define mock.On call +// - repoOption map[string]string +func (_e *BackupRepoService_Expecter) ClientSideCacheLimit(repoOption interface{}) *BackupRepoService_ClientSideCacheLimit_Call { + return &BackupRepoService_ClientSideCacheLimit_Call{Call: _e.mock.On("ClientSideCacheLimit", repoOption)} +} + +func (_c *BackupRepoService_ClientSideCacheLimit_Call) Run(run func(repoOption map[string]string)) *BackupRepoService_ClientSideCacheLimit_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 map[string]string + if args[0] != nil { + arg0 = args[0].(map[string]string) + } + run( + arg0, + ) + }) + return _c +} + +func (_c *BackupRepoService_ClientSideCacheLimit_Call) Return(n int64) *BackupRepoService_ClientSideCacheLimit_Call { + _c.Call.Return(n) + return _c +} + +func (_c *BackupRepoService_ClientSideCacheLimit_Call) RunAndReturn(run func(repoOption map[string]string) int64) *BackupRepoService_ClientSideCacheLimit_Call { + _c.Call.Return(run) + return _c +} + +// Connect provides a mock function for the type BackupRepoService +func (_mock *BackupRepoService) Connect(ctx context.Context, repoOption udmrepo.RepoOptions) error { + ret := _mock.Called(ctx, repoOption) + + if len(ret) == 0 { + panic("no return value specified for Connect") + } + + var r0 error + if returnFunc, ok := ret.Get(0).(func(context.Context, udmrepo.RepoOptions) error); ok { + r0 = returnFunc(ctx, repoOption) + } else { + r0 = ret.Error(0) + } + return r0 +} + +// BackupRepoService_Connect_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Connect' +type BackupRepoService_Connect_Call struct { + *mock.Call +} + +// Connect is a helper method to define mock.On call +// - ctx context.Context +// - repoOption udmrepo.RepoOptions +func (_e *BackupRepoService_Expecter) Connect(ctx interface{}, repoOption interface{}) *BackupRepoService_Connect_Call { + return &BackupRepoService_Connect_Call{Call: _e.mock.On("Connect", ctx, repoOption)} +} + +func (_c *BackupRepoService_Connect_Call) Run(run func(ctx context.Context, repoOption udmrepo.RepoOptions)) *BackupRepoService_Connect_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 udmrepo.RepoOptions + if args[1] != nil { + arg1 = args[1].(udmrepo.RepoOptions) + } + run( + arg0, + arg1, + ) + }) + return _c +} + +func (_c *BackupRepoService_Connect_Call) Return(err error) *BackupRepoService_Connect_Call { + _c.Call.Return(err) + return _c +} + +func (_c *BackupRepoService_Connect_Call) RunAndReturn(run func(ctx context.Context, repoOption udmrepo.RepoOptions) error) *BackupRepoService_Connect_Call { + _c.Call.Return(run) + return _c +} + +// Create provides a mock function for the type BackupRepoService +func (_mock *BackupRepoService) Create(ctx context.Context, repoOption udmrepo.RepoOptions) error { + ret := _mock.Called(ctx, repoOption) + + if len(ret) == 0 { + panic("no return value specified for Create") + } + + var r0 error + if returnFunc, ok := ret.Get(0).(func(context.Context, udmrepo.RepoOptions) error); ok { + r0 = returnFunc(ctx, repoOption) + } else { + r0 = ret.Error(0) + } + return r0 +} + +// BackupRepoService_Create_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Create' +type BackupRepoService_Create_Call struct { + *mock.Call +} + +// Create is a helper method to define mock.On call +// - ctx context.Context +// - repoOption udmrepo.RepoOptions +func (_e *BackupRepoService_Expecter) Create(ctx interface{}, repoOption interface{}) *BackupRepoService_Create_Call { + return &BackupRepoService_Create_Call{Call: _e.mock.On("Create", ctx, repoOption)} +} + +func (_c *BackupRepoService_Create_Call) Run(run func(ctx context.Context, repoOption udmrepo.RepoOptions)) *BackupRepoService_Create_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 udmrepo.RepoOptions + if args[1] != nil { + arg1 = args[1].(udmrepo.RepoOptions) + } + run( + arg0, + arg1, + ) + }) + return _c +} + +func (_c *BackupRepoService_Create_Call) Return(err error) *BackupRepoService_Create_Call { + _c.Call.Return(err) + return _c +} + +func (_c *BackupRepoService_Create_Call) RunAndReturn(run func(ctx context.Context, repoOption udmrepo.RepoOptions) error) *BackupRepoService_Create_Call { + _c.Call.Return(run) + return _c +} + +// DefaultMaintenanceFrequency provides a mock function for the type BackupRepoService +func (_mock *BackupRepoService) DefaultMaintenanceFrequency() time.Duration { + ret := _mock.Called() + + if len(ret) == 0 { + panic("no return value specified for DefaultMaintenanceFrequency") + } + + var r0 time.Duration + if returnFunc, ok := ret.Get(0).(func() time.Duration); ok { + r0 = returnFunc() + } else { + r0 = ret.Get(0).(time.Duration) + } + return r0 +} + +// BackupRepoService_DefaultMaintenanceFrequency_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DefaultMaintenanceFrequency' +type BackupRepoService_DefaultMaintenanceFrequency_Call struct { + *mock.Call +} + +// DefaultMaintenanceFrequency is a helper method to define mock.On call +func (_e *BackupRepoService_Expecter) DefaultMaintenanceFrequency() *BackupRepoService_DefaultMaintenanceFrequency_Call { + return &BackupRepoService_DefaultMaintenanceFrequency_Call{Call: _e.mock.On("DefaultMaintenanceFrequency")} +} + +func (_c *BackupRepoService_DefaultMaintenanceFrequency_Call) Run(run func()) *BackupRepoService_DefaultMaintenanceFrequency_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *BackupRepoService_DefaultMaintenanceFrequency_Call) Return(duration time.Duration) *BackupRepoService_DefaultMaintenanceFrequency_Call { + _c.Call.Return(duration) + return _c +} + +func (_c *BackupRepoService_DefaultMaintenanceFrequency_Call) RunAndReturn(run func() time.Duration) *BackupRepoService_DefaultMaintenanceFrequency_Call { + _c.Call.Return(run) + return _c +} + +// IsReady provides a mock function for the type BackupRepoService +func (_mock *BackupRepoService) IsReady(ctx context.Context, repoOption udmrepo.RepoOptions, readOnly bool) (bool, error) { + ret := _mock.Called(ctx, repoOption, readOnly) + + if len(ret) == 0 { + panic("no return value specified for IsReady") + } + + var r0 bool + var r1 error + if returnFunc, ok := ret.Get(0).(func(context.Context, udmrepo.RepoOptions, bool) (bool, error)); ok { + return returnFunc(ctx, repoOption, readOnly) + } + if returnFunc, ok := ret.Get(0).(func(context.Context, udmrepo.RepoOptions, bool) bool); ok { + r0 = returnFunc(ctx, repoOption, readOnly) + } else { + r0 = ret.Get(0).(bool) + } + if returnFunc, ok := ret.Get(1).(func(context.Context, udmrepo.RepoOptions, bool) error); ok { + r1 = returnFunc(ctx, repoOption, readOnly) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// BackupRepoService_IsReady_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IsReady' +type BackupRepoService_IsReady_Call struct { + *mock.Call +} + +// IsReady is a helper method to define mock.On call +// - ctx context.Context +// - repoOption udmrepo.RepoOptions +// - readOnly bool +func (_e *BackupRepoService_Expecter) IsReady(ctx interface{}, repoOption interface{}, readOnly interface{}) *BackupRepoService_IsReady_Call { + return &BackupRepoService_IsReady_Call{Call: _e.mock.On("IsReady", ctx, repoOption, readOnly)} +} + +func (_c *BackupRepoService_IsReady_Call) Run(run func(ctx context.Context, repoOption udmrepo.RepoOptions, readOnly bool)) *BackupRepoService_IsReady_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 udmrepo.RepoOptions + if args[1] != nil { + arg1 = args[1].(udmrepo.RepoOptions) + } + var arg2 bool + if args[2] != nil { + arg2 = args[2].(bool) + } + run( + arg0, + arg1, + arg2, + ) + }) + return _c +} + +func (_c *BackupRepoService_IsReady_Call) Return(b bool, err error) *BackupRepoService_IsReady_Call { + _c.Call.Return(b, err) + return _c +} + +func (_c *BackupRepoService_IsReady_Call) RunAndReturn(run func(ctx context.Context, repoOption udmrepo.RepoOptions, readOnly bool) (bool, error)) *BackupRepoService_IsReady_Call { + _c.Call.Return(run) + return _c +} + +// Maintain provides a mock function for the type BackupRepoService +func (_mock *BackupRepoService) Maintain(ctx context.Context, repoOption udmrepo.RepoOptions) error { + ret := _mock.Called(ctx, repoOption) + + if len(ret) == 0 { + panic("no return value specified for Maintain") + } + + var r0 error + if returnFunc, ok := ret.Get(0).(func(context.Context, udmrepo.RepoOptions) error); ok { + r0 = returnFunc(ctx, repoOption) + } else { + r0 = ret.Error(0) + } + return r0 +} + +// BackupRepoService_Maintain_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Maintain' +type BackupRepoService_Maintain_Call struct { + *mock.Call +} + +// Maintain is a helper method to define mock.On call +// - ctx context.Context +// - repoOption udmrepo.RepoOptions +func (_e *BackupRepoService_Expecter) Maintain(ctx interface{}, repoOption interface{}) *BackupRepoService_Maintain_Call { + return &BackupRepoService_Maintain_Call{Call: _e.mock.On("Maintain", ctx, repoOption)} +} + +func (_c *BackupRepoService_Maintain_Call) Run(run func(ctx context.Context, repoOption udmrepo.RepoOptions)) *BackupRepoService_Maintain_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 udmrepo.RepoOptions + if args[1] != nil { + arg1 = args[1].(udmrepo.RepoOptions) + } + run( + arg0, + arg1, + ) + }) + return _c +} + +func (_c *BackupRepoService_Maintain_Call) Return(err error) *BackupRepoService_Maintain_Call { + _c.Call.Return(err) + return _c +} + +func (_c *BackupRepoService_Maintain_Call) RunAndReturn(run func(ctx context.Context, repoOption udmrepo.RepoOptions) error) *BackupRepoService_Maintain_Call { + _c.Call.Return(run) + return _c +} + +// Open provides a mock function for the type BackupRepoService +func (_mock *BackupRepoService) Open(ctx context.Context, repoOption udmrepo.RepoOptions) (udmrepo.BackupRepo, error) { + ret := _mock.Called(ctx, repoOption) + + if len(ret) == 0 { + panic("no return value specified for Open") + } + + var r0 udmrepo.BackupRepo + var r1 error + if returnFunc, ok := ret.Get(0).(func(context.Context, udmrepo.RepoOptions) (udmrepo.BackupRepo, error)); ok { + return returnFunc(ctx, repoOption) + } + if returnFunc, ok := ret.Get(0).(func(context.Context, udmrepo.RepoOptions) udmrepo.BackupRepo); ok { + r0 = returnFunc(ctx, repoOption) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(udmrepo.BackupRepo) + } + } + if returnFunc, ok := ret.Get(1).(func(context.Context, udmrepo.RepoOptions) error); ok { + r1 = returnFunc(ctx, repoOption) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// BackupRepoService_Open_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Open' +type BackupRepoService_Open_Call struct { + *mock.Call +} + +// Open is a helper method to define mock.On call +// - ctx context.Context +// - repoOption udmrepo.RepoOptions +func (_e *BackupRepoService_Expecter) Open(ctx interface{}, repoOption interface{}) *BackupRepoService_Open_Call { + return &BackupRepoService_Open_Call{Call: _e.mock.On("Open", ctx, repoOption)} +} + +func (_c *BackupRepoService_Open_Call) Run(run func(ctx context.Context, repoOption udmrepo.RepoOptions)) *BackupRepoService_Open_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 udmrepo.RepoOptions + if args[1] != nil { + arg1 = args[1].(udmrepo.RepoOptions) + } + run( + arg0, + arg1, + ) + }) + return _c +} + +func (_c *BackupRepoService_Open_Call) Return(backupRepo udmrepo.BackupRepo, err error) *BackupRepoService_Open_Call { + _c.Call.Return(backupRepo, err) + return _c +} + +func (_c *BackupRepoService_Open_Call) RunAndReturn(run func(ctx context.Context, repoOption udmrepo.RepoOptions) (udmrepo.BackupRepo, error)) *BackupRepoService_Open_Call { + _c.Call.Return(run) + return _c +} diff --git a/pkg/repository/udmrepo/repo.go b/pkg/repository/udmrepo/repo.go index 5bf8c1573..1e7e4d011 100644 --- a/pkg/repository/udmrepo/repo.go +++ b/pkg/repository/udmrepo/repo.go @@ -85,9 +85,9 @@ type BackupRepoService interface { // repoOption: option to the backup repository and the underlying backup storage. Connect(ctx context.Context, repoOption RepoOptions) error - // IsCreated checks if the backup repository has been created in the underlying backup storage. + // IsReady checks if the backup repository has been ready in the underlying backup storage. // repoOption: option to the underlying backup storage - IsCreated(ctx context.Context, repoOption RepoOptions) (bool, error) + IsReady(ctx context.Context, repoOption RepoOptions, readOnly bool) (bool, error) // Open opens an backup repository that has been created/connected. // repoOption: options to open the backup repository and the underlying storage. diff --git a/pkg/restore/actions/csi/volumesnapshot_action.go b/pkg/restore/actions/csi/volumesnapshot_action.go index 6d4bc1eda..dec33d4ef 100644 --- a/pkg/restore/actions/csi/volumesnapshot_action.go +++ b/pkg/restore/actions/csi/volumesnapshot_action.go @@ -17,11 +17,16 @@ limitations under the License. package csi import ( + "context" "fmt" + volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" + corev1api "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" crclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -65,6 +70,165 @@ func resetVolumeSnapshotAnnotation(vs *snapshotv1api.VolumeSnapshot) { string(snapshotv1api.VolumeSnapshotContentRetain) } +// ensureStubVGSCExists creates a stub VolumeGroupSnapshotContent if the snapshot +// was created as part of a VolumeGroupSnapshot. This is needed for CSI drivers +// like Ceph RBD that populate volumeGroupSnapshotHandle on pre-provisioned snapshots. +// The CSI snapshot controller requires a VGSC with matching handle to exist. +func (p *volumeSnapshotRestoreItemAction) ensureStubVGSCExists( + ctx context.Context, + vs *snapshotv1api.VolumeSnapshot, + restore *velerov1api.Restore, +) error { + vgsh, ok := vs.Annotations[velerov1api.VolumeGroupSnapshotHandleAnnotation] + if !ok || vgsh == "" { + // No VolumeGroupSnapshotHandle, nothing to do + return nil + } + + snapshotHandle, ok := vs.Annotations[velerov1api.VolumeSnapshotHandleAnnotation] + if !ok || snapshotHandle == "" { + p.log.Warnf("VS %s/%s has VolumeGroupSnapshotHandle but no SnapshotHandle annotation", + vs.Namespace, vs.Name) + return nil + } + + driver, ok := vs.Annotations[velerov1api.DriverNameAnnotation] + if !ok || driver == "" { + p.log.Warnf("VS %s/%s has VolumeGroupSnapshotHandle but no Driver annotation", + vs.Namespace, vs.Name) + return nil + } + + // Generate a deterministic name for the stub VGSC based on the group handle + vgscName := util.GenerateSha256FromRestoreUIDAndVsName(string(restore.UID), vgsh) + + // Check if VGSC already exists + existingVGSC := &volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent{} + err := p.crClient.Get(ctx, crclient.ObjectKey{Name: vgscName}, existingVGSC) + if err == nil { + // VGSC already exists, add this snapshot handle if not already present + p.log.Infof("Stub VGSC %s already exists for VolumeGroupSnapshotHandle %s", vgscName, vgsh) + return p.addSnapshotHandleToVGSC(ctx, existingVGSC, snapshotHandle) + } + if !apierrors.IsNotFound(err) { + return errors.Wrapf(err, "failed to check for existing VGSC %s", vgscName) + } + + // Create stub VGSC + p.log.Infof("Creating stub VGSC %s for VolumeGroupSnapshotHandle %s", vgscName, vgsh) + + // Look up VolumeGroupSnapshotClass to get secret annotations + vgscAnnotations := map[string]string{} + vgscList := &volumegroupsnapshotv1beta2.VolumeGroupSnapshotClassList{} + if err := p.crClient.List(ctx, vgscList); err == nil { + for _, vgsClass := range vgscList.Items { + if vgsClass.Driver == driver { + // Found matching class, extract secret parameters + if secretName, ok := vgsClass.Parameters["csi.storage.k8s.io/group-snapshotter-secret-name"]; ok { + vgscAnnotations["groupsnapshot.storage.kubernetes.io/deletion-secret-name"] = secretName + } + if secretNS, ok := vgsClass.Parameters["csi.storage.k8s.io/group-snapshotter-secret-namespace"]; ok { + vgscAnnotations["groupsnapshot.storage.kubernetes.io/deletion-secret-namespace"] = secretNS + } + break + } + } + } + + vgsc := &volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: vgscName, + Labels: map[string]string{ + velerov1api.RestoreNameLabel: restore.Name, + }, + Annotations: vgscAnnotations, + }, + Spec: volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentSpec{ + DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, + Driver: driver, + Source: volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentSource{ + GroupSnapshotHandles: &volumegroupsnapshotv1beta2.GroupSnapshotHandles{ + VolumeGroupSnapshotHandle: vgsh, + VolumeSnapshotHandles: []string{snapshotHandle}, + }, + }, + VolumeGroupSnapshotRef: corev1api.ObjectReference{ + Name: "stub-vgs-" + vgscName[:8], + Namespace: vs.Namespace, + }, + }, + } + + if err := p.crClient.Create(ctx, vgsc); err != nil { + if apierrors.IsAlreadyExists(err) { + // Another VS restore created the VGSC between our Get and Create. + // Re-fetch and add our snapshot handle. + p.log.Infof("Stub VGSC %s was created by another VS restore, adding our handle", vgscName) + raceVGSC := &volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent{} + if getErr := p.crClient.Get(ctx, crclient.ObjectKey{Name: vgscName}, raceVGSC); getErr != nil { + return errors.Wrapf(getErr, "failed to get VGSC %s after race", vgscName) + } + return p.addSnapshotHandleToVGSC(ctx, raceVGSC, snapshotHandle) + } + return errors.Wrapf(err, "failed to create stub VGSC %s", vgscName) + } + + // Re-fetch to get server-assigned metadata (resourceVersion) needed for patching + createdVGSC := &volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent{} + if err := p.crClient.Get(ctx, crclient.ObjectKey{Name: vgscName}, createdVGSC); err != nil { + p.log.Warnf("Failed to fetch stub VGSC %s for status patch: %v", vgscName, err) + return nil + } + + // Set volumeGroupSnapshotHandle in status using Patch to avoid conflicts with the CSI controller. + patchBase := createdVGSC.DeepCopy() + if createdVGSC.Status == nil { + createdVGSC.Status = &volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentStatus{} + } + createdVGSC.Status.VolumeGroupSnapshotHandle = &vgsh + if err := p.crClient.Status().Patch(ctx, createdVGSC, crclient.MergeFrom(patchBase)); err != nil { + p.log.Warnf("Failed to patch stub VGSC %s status: %v", vgscName, err) + } + + p.log.Infof("Successfully created stub VGSC %s", vgscName) + return nil +} + +// addSnapshotHandleToVGSC adds a snapshot handle to an existing VGSC if not already present. +// This is needed when multiple VolumeSnapshots from the same VolumeGroupSnapshot are restored. +func (p *volumeSnapshotRestoreItemAction) addSnapshotHandleToVGSC( + ctx context.Context, + vgsc *volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent, + snapshotHandle string, +) error { + // Check if handle is already in the list + if vgsc.Spec.Source.GroupSnapshotHandles != nil { + for _, handle := range vgsc.Spec.Source.GroupSnapshotHandles.VolumeSnapshotHandles { + if handle == snapshotHandle { + p.log.Infof("Snapshot handle %s already present in VGSC %s", snapshotHandle, vgsc.Name) + return nil + } + } + } + + // Add the snapshot handle to the list + patchBase := vgsc.DeepCopy() + if vgsc.Spec.Source.GroupSnapshotHandles == nil { + vgsc.Spec.Source.GroupSnapshotHandles = &volumegroupsnapshotv1beta2.GroupSnapshotHandles{} + } + vgsc.Spec.Source.GroupSnapshotHandles.VolumeSnapshotHandles = append( + vgsc.Spec.Source.GroupSnapshotHandles.VolumeSnapshotHandles, + snapshotHandle, + ) + + if err := p.crClient.Patch(ctx, vgsc, crclient.MergeFrom(patchBase)); err != nil { + return errors.Wrapf(err, "failed to add snapshot handle to VGSC %s", vgsc.Name) + } + + p.log.Infof("Added snapshot handle %s to existing VGSC %s", snapshotHandle, vgsc.Name) + return nil +} + func (p *volumeSnapshotRestoreItemAction) Execute( input *velero.RestoreItemActionExecuteInput, ) (*velero.RestoreItemActionExecuteOutput, error) { @@ -90,6 +254,13 @@ func (p *volumeSnapshotRestoreItemAction) Execute( errors.Wrapf(err, "failed to convert input.Item from unstructured") } + // Create stub VGSC if this snapshot was created via VolumeGroupSnapshot + // This must happen before VSC is created, as the CSI controller requires VGSC to exist + if err := p.ensureStubVGSCExists(context.Background(), &vsFromBackup, input.Restore); err != nil { + p.log.Warnf("Failed to create stub VGSC for VS %s/%s: %v", vsFromBackup.Namespace, vsFromBackup.Name, err) + // Continue with restore, VGSC creation failure should not block restore + } + generatedName := util.GenerateSha256FromRestoreUIDAndVsName(string(input.Restore.UID), vsFromBackup.Name) // Reset Spec to convert the VolumeSnapshot from using diff --git a/pkg/restore/actions/csi/volumesnapshot_action_test.go b/pkg/restore/actions/csi/volumesnapshot_action_test.go index 4fb37e301..de3e592c0 100644 --- a/pkg/restore/actions/csi/volumesnapshot_action_test.go +++ b/pkg/restore/actions/csi/volumesnapshot_action_test.go @@ -17,9 +17,11 @@ limitations under the License. package csi import ( + "context" "fmt" "testing" + volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -27,6 +29,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + crclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" @@ -219,3 +222,244 @@ func TestNewVolumeSnapshotRestoreItemAction(t *testing.T) { _, err1 := plugin1(logger) require.NoError(t, err1) } + +func TestEnsureStubVGSCExists(t *testing.T) { + testDriver := "rbd.csi.ceph.com" + testVGSHandle := "vgs-handle-123" + testSnapshotHandle := "snap-handle-456" + + tests := []struct { + name string + vs *snapshotv1api.VolumeSnapshot + restore *velerov1api.Restore + existingVGSC *volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent + expectVGSC bool + expectErr bool + expectedHandle string + }{ + { + name: "VS without VolumeGroupSnapshotHandle annotation - no VGSC created", + vs: &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vs", + Namespace: "test-ns", + Annotations: map[string]string{ + velerov1api.VolumeSnapshotHandleAnnotation: testSnapshotHandle, + velerov1api.DriverNameAnnotation: testDriver, + }, + }, + }, + restore: builder.ForRestore("velero", "restore").ObjectMeta(builder.WithUID("restore-uid")).Result(), + expectVGSC: false, + expectErr: false, + }, + { + name: "VS with VolumeGroupSnapshotHandle but no SnapshotHandle - no VGSC created", + vs: &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vs", + Namespace: "test-ns", + Annotations: map[string]string{ + velerov1api.VolumeGroupSnapshotHandleAnnotation: testVGSHandle, + velerov1api.DriverNameAnnotation: testDriver, + }, + }, + }, + restore: builder.ForRestore("velero", "restore").ObjectMeta(builder.WithUID("restore-uid")).Result(), + expectVGSC: false, + expectErr: false, + }, + { + name: "VS with VolumeGroupSnapshotHandle but no Driver annotation - no VGSC created", + vs: &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vs", + Namespace: "test-ns", + Annotations: map[string]string{ + velerov1api.VolumeGroupSnapshotHandleAnnotation: testVGSHandle, + velerov1api.VolumeSnapshotHandleAnnotation: testSnapshotHandle, + }, + }, + }, + restore: builder.ForRestore("velero", "restore").ObjectMeta(builder.WithUID("restore-uid")).Result(), + expectVGSC: false, + expectErr: false, + }, + { + name: "VS with all required annotations - VGSC should be created", + vs: &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vs", + Namespace: "test-ns", + Annotations: map[string]string{ + velerov1api.VolumeGroupSnapshotHandleAnnotation: testVGSHandle, + velerov1api.VolumeSnapshotHandleAnnotation: testSnapshotHandle, + velerov1api.DriverNameAnnotation: testDriver, + }, + }, + }, + restore: builder.ForRestore("velero", "restore").ObjectMeta(builder.WithUID("restore-uid")).Result(), + expectVGSC: true, + expectErr: false, + expectedHandle: testSnapshotHandle, + }, + { + name: "VGSC already exists - should add snapshot handle", + vs: &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vs-2", + Namespace: "test-ns", + Annotations: map[string]string{ + velerov1api.VolumeGroupSnapshotHandleAnnotation: testVGSHandle, + velerov1api.VolumeSnapshotHandleAnnotation: "snap-handle-789", + velerov1api.DriverNameAnnotation: testDriver, + }, + }, + }, + restore: builder.ForRestore("velero", "restore").ObjectMeta(builder.WithUID("restore-uid")).Result(), + existingVGSC: &volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: util.GenerateSha256FromRestoreUIDAndVsName("restore-uid", testVGSHandle), + }, + Spec: volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentSpec{ + Driver: testDriver, + DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, + Source: volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentSource{ + GroupSnapshotHandles: &volumegroupsnapshotv1beta2.GroupSnapshotHandles{ + VolumeGroupSnapshotHandle: testVGSHandle, + VolumeSnapshotHandles: []string{testSnapshotHandle}, + }, + }, + }, + }, + expectVGSC: true, + expectErr: false, + expectedHandle: "snap-handle-789", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + crClient := velerotest.NewFakeControllerRuntimeClient(t) + + // Create existing VGSC if provided + if tc.existingVGSC != nil { + require.NoError(t, crClient.Create(context.Background(), tc.existingVGSC)) + } + + p := &volumeSnapshotRestoreItemAction{ + log: logrus.StandardLogger(), + crClient: crClient, + } + + err := p.ensureStubVGSCExists(context.Background(), tc.vs, tc.restore) + + if tc.expectErr { + require.Error(t, err) + return + } + require.NoError(t, err) + + // Check if VGSC was created/updated + vgscName := util.GenerateSha256FromRestoreUIDAndVsName(string(tc.restore.UID), tc.vs.Annotations[velerov1api.VolumeGroupSnapshotHandleAnnotation]) + vgsc := &volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent{} + getErr := crClient.Get(context.Background(), crclient.ObjectKey{Name: vgscName}, vgsc) + + if tc.expectVGSC { + require.NoError(t, getErr) + require.NotNil(t, vgsc.Spec.Source.GroupSnapshotHandles) + require.Contains(t, vgsc.Spec.Source.GroupSnapshotHandles.VolumeSnapshotHandles, tc.expectedHandle) + } else { + // If no VGSC expected, it's okay if Get returns not found or if vgscName is empty + if tc.vs.Annotations[velerov1api.VolumeGroupSnapshotHandleAnnotation] != "" { + require.Error(t, getErr) + } + } + }) + } +} + +func TestAddSnapshotHandleToVGSC(t *testing.T) { + testDriver := "rbd.csi.ceph.com" + testVGSHandle := "vgs-handle-123" + + tests := []struct { + name string + existingHandles []string + nilGroupSnapshotHandles bool + newHandle string + expectedHandles []string + }{ + { + name: "Add new handle to empty list", + existingHandles: []string{}, + newHandle: "snap-1", + expectedHandles: []string{"snap-1"}, + }, + { + name: "Add new handle to existing list", + existingHandles: []string{"snap-1"}, + newHandle: "snap-2", + expectedHandles: []string{"snap-1", "snap-2"}, + }, + { + name: "Handle already exists - no change", + existingHandles: []string{"snap-1", "snap-2"}, + newHandle: "snap-1", + expectedHandles: []string{"snap-1", "snap-2"}, + }, + { + name: "Nil GroupSnapshotHandles - should initialize and add", + nilGroupSnapshotHandles: true, + newHandle: "snap-1", + expectedHandles: []string{"snap-1"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + crClient := velerotest.NewFakeControllerRuntimeClient(t) + + var source volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentSource + if tc.nilGroupSnapshotHandles { + source = volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentSource{} + } else { + source = volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentSource{ + GroupSnapshotHandles: &volumegroupsnapshotv1beta2.GroupSnapshotHandles{ + VolumeGroupSnapshotHandle: testVGSHandle, + VolumeSnapshotHandles: tc.existingHandles, + }, + } + } + + existingVGSC := &volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vgsc", + }, + Spec: volumegroupsnapshotv1beta2.VolumeGroupSnapshotContentSpec{ + Driver: testDriver, + DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, + Source: source, + }, + } + require.NoError(t, crClient.Create(context.Background(), existingVGSC)) + + // Re-fetch to get the created object with proper metadata + fetchedVGSC := &volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent{} + require.NoError(t, crClient.Get(context.Background(), crclient.ObjectKey{Name: "test-vgsc"}, fetchedVGSC)) + + p := &volumeSnapshotRestoreItemAction{ + log: logrus.StandardLogger(), + crClient: crClient, + } + + err := p.addSnapshotHandleToVGSC(context.Background(), fetchedVGSC, tc.newHandle) + require.NoError(t, err) + + // Verify the VGSC has expected handles + updatedVGSC := &volumegroupsnapshotv1beta2.VolumeGroupSnapshotContent{} + require.NoError(t, crClient.Get(context.Background(), crclient.ObjectKey{Name: "test-vgsc"}, updatedVGSC)) + require.ElementsMatch(t, tc.expectedHandles, updatedVGSC.Spec.Source.GroupSnapshotHandles.VolumeSnapshotHandles) + }) + } +} diff --git a/pkg/restore/actions/csi/volumesnapshotcontent_action.go b/pkg/restore/actions/csi/volumesnapshotcontent_action.go index 0a268eab2..00a25c86f 100644 --- a/pkg/restore/actions/csi/volumesnapshotcontent_action.go +++ b/pkg/restore/actions/csi/volumesnapshotcontent_action.go @@ -17,6 +17,8 @@ limitations under the License. package csi import ( + "context" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -108,12 +110,23 @@ func (p *volumeSnapshotContentRestoreItemAction) Execute( return nil, errors.Errorf("fail to get snapshot handle from VSC %s status", vsc.Name) } - if vsc.Spec.VolumeSnapshotClassName != nil { - // Delete VolumeSnapshotClass from the VolumeSnapshotContent. - // This is necessary to make the restore independent of the VolumeSnapshotClass. - vsc.Spec.VolumeSnapshotClassName = nil - p.log.Debugf("Deleted VolumeSnapshotClassName from VolumeSnapshotContent %s to make restore independent of VolumeSnapshotClass", - vsc.Name) + // Look up a VolumeSnapshotClass matching the driver for credential lookup. + // Some CSI drivers (e.g., Ceph RBD) need credentials for snapshot verification. + // Instead of keeping the original class name (which may not exist on target cluster), + // we find a matching class by driver to make restore portable. + vsc.Spec.VolumeSnapshotClassName = nil + vscList := &snapshotv1api.VolumeSnapshotClassList{} + if err := p.client.List(context.Background(), vscList); err == nil { + for i := range vscList.Items { + if vscList.Items[i].Driver == vsc.Spec.Driver { + vsc.Spec.VolumeSnapshotClassName = &vscList.Items[i].Name + p.log.Infof("Set VolumeSnapshotClassName to %s for VSC %s based on driver match", + vscList.Items[i].Name, vsc.Name) + break + } + } + } else { + p.log.Warnf("Failed to list VolumeSnapshotClasses: %v", err) } additionalItems := []velero.ResourceIdentifier{} diff --git a/pkg/restore/actions/pod_volume_restore_action.go b/pkg/restore/actions/pod_volume_restore_action.go index eb7cb8f6d..e26a53034 100644 --- a/pkg/restore/actions/pod_volume_restore_action.go +++ b/pkg/restore/actions/pod_volume_restore_action.go @@ -101,6 +101,7 @@ func (a *PodVolumeRestoreAction) Execute(input *velero.RestoreItemActionExecuteI opts := &ctrlclient.ListOptions{ LabelSelector: label.NewSelectorForBackup(input.Restore.Spec.BackupName), + Namespace: input.Restore.Namespace, } podVolumeBackupList := new(velerov1api.PodVolumeBackupList) if err := a.crClient.List(context.TODO(), podVolumeBackupList, opts); err != nil { @@ -163,12 +164,19 @@ func (a *PodVolumeRestoreAction) Execute(input *velero.RestoreItemActionExecuteI memLimit = defaultMemRequestLimit } - resourceReqs, err := kube.ParseResourceRequirements(cpuRequest, memRequest, cpuLimit, memLimit) + resourceReqs, err := kube.ParseCPUAndMemoryResources( + cpuRequest, + memRequest, + cpuLimit, + memLimit, + ) if err != nil { log.Errorf("couldn't parse resource requirements: %s.", err) - resourceReqs, _ = kube.ParseResourceRequirements( - defaultCPURequestLimit, defaultMemRequestLimit, // requests - defaultCPURequestLimit, defaultMemRequestLimit, // limits + resourceReqs, _ = kube.ParseCPUAndMemoryResources( + defaultCPURequestLimit, + defaultMemRequestLimit, + defaultCPURequestLimit, + defaultMemRequestLimit, ) } diff --git a/pkg/restore/actions/pod_volume_restore_action_test.go b/pkg/restore/actions/pod_volume_restore_action_test.go index 43df2e35a..bc9662ab7 100644 --- a/pkg/restore/actions/pod_volume_restore_action_test.go +++ b/pkg/restore/actions/pod_volume_restore_action_test.go @@ -117,9 +117,11 @@ func TestGetImage(t *testing.T) { // TestPodVolumeRestoreActionExecute tests the pod volume restore item action plugin's Execute method. func TestPodVolumeRestoreActionExecute(t *testing.T) { - resourceReqs, _ := kube.ParseResourceRequirements( - defaultCPURequestLimit, defaultMemRequestLimit, // requests - defaultCPURequestLimit, defaultMemRequestLimit, // limits + resourceReqs, _ := kube.ParseCPUAndMemoryResources( + defaultCPURequestLimit, + defaultMemRequestLimit, + defaultCPURequestLimit, + defaultMemRequestLimit, ) id := int64(1000) securityContext := corev1api.SecurityContext{ @@ -348,6 +350,28 @@ func TestPodVolumeRestoreActionExecute(t *testing.T) { VolumeMounts(builder.ForVolumeMount("myvol", "/restores/myvol").Result()). Command([]string{"/velero-restore-helper"}).Result()).Result(), }, + { + name: "pod volume backups in a different namespace are ignored when looking for matches due to namespace scoping", + pod: builder.ForPod("ns-1", "my-pod"). + Volumes( + builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), + ). + Result(), + podVolumeBackups: []runtime.Object{ + builder.ForPodVolumeBackup("other-ns", "pvb-1"). + PodName("my-pod"). + PodNamespace("ns-1"). + Volume("myvol"). + ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, backupName)). + SnapshotID("foo"). + Result(), + }, + want: builder.ForPod("ns-1", "my-pod"). + Volumes( + builder.ForVolume("myvol").PersistentVolumeClaimSource("pvc-1").Result(), + ). + Result(), + }, } veleroDeployment := &appsv1api.Deployment{ diff --git a/pkg/restore/actions/service_action.go b/pkg/restore/actions/service_action.go index 9de75228e..34ba85e93 100644 --- a/pkg/restore/actions/service_action.go +++ b/pkg/restore/actions/service_action.go @@ -94,21 +94,18 @@ func deleteHealthCheckNodePort(service *corev1api.Service) error { // Search HealthCheckNodePort from server's last-applied-configuration // annotation(HealthCheckNodePort is specified by `kubectl apply` command) - lastAppliedConfig, ok := service.Annotations[annotationLastAppliedConfig] - if ok { - appliedServiceUnstructured := new(map[string]any) - if err := json.Unmarshal([]byte(lastAppliedConfig), appliedServiceUnstructured); err != nil { + if lastAppliedConfig, ok := service.Annotations[annotationLastAppliedConfig]; ok { + var appliedConfig struct { + Spec struct { + HealthCheckNodePort *int32 `json:"healthCheckNodePort"` + } `json:"spec"` + } + + if err := json.Unmarshal([]byte(lastAppliedConfig), &appliedConfig); err != nil { return errors.WithStack(err) } - healthCheckNodePort, exist, err := unstructured.NestedFloat64(*appliedServiceUnstructured, "spec", "healthCheckNodePort") - if err != nil { - return errors.WithStack(err) - } - - // Found healthCheckNodePort in lastAppliedConfig annotation, - // and the value is not 0. No need to delete, return. - if exist && healthCheckNodePort != 0 { + if appliedConfig.Spec.HealthCheckNodePort != nil && *appliedConfig.Spec.HealthCheckNodePort != 0 { return nil } } diff --git a/pkg/restore/actions/service_action_test.go b/pkg/restore/actions/service_action_test.go index 42c0d290d..f9a01d5d4 100644 --- a/pkg/restore/actions/service_action_test.go +++ b/pkg/restore/actions/service_action_test.go @@ -644,6 +644,36 @@ func TestServiceActionExecute(t *testing.T) { }, }, }, + { + name: "If PreserveNodePorts is false and HealthCheckNodePort is null in last-applied-configuration, it should not crash and the port should be cleared.", + obj: corev1api.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-1", + Annotations: map[string]string{ + "kubectl.kubernetes.io/last-applied-configuration": `{"spec":{"healthCheckNodePort":null}}`, + }, + }, + Spec: corev1api.ServiceSpec{ + HealthCheckNodePort: 8080, + ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeLocal, + Type: corev1api.ServiceTypeLoadBalancer, + }, + }, + restore: builder.ForRestore(api.DefaultNamespace, "").PreserveNodePorts(false).Result(), + expectedRes: corev1api.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-1", + Annotations: map[string]string{ + "kubectl.kubernetes.io/last-applied-configuration": `{"spec":{"healthCheckNodePort":null}}`, + }, + }, + Spec: corev1api.ServiceSpec{ + HealthCheckNodePort: 0, + ExternalTrafficPolicy: corev1api.ServiceExternalTrafficPolicyTypeLocal, + Type: corev1api.ServiceTypeLoadBalancer, + }, + }, + }, } for _, test := range tests { diff --git a/pkg/test/fake_controller_runtime_client.go b/pkg/test/fake_controller_runtime_client.go index e22220404..90ee95d11 100644 --- a/pkg/test/fake_controller_runtime_client.go +++ b/pkg/test/fake_controller_runtime_client.go @@ -19,7 +19,7 @@ package test import ( "testing" - volumegroupsnapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta1" + volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/stretchr/testify/require" @@ -45,6 +45,7 @@ func NewFakeControllerRuntimeClientBuilder(t *testing.T) *k8sfake.ClientBuilder require.NoError(t, appsv1api.AddToScheme(scheme)) require.NoError(t, snapshotv1api.AddToScheme(scheme)) require.NoError(t, storagev1api.AddToScheme(scheme)) + require.NoError(t, volumegroupsnapshotv1beta2.AddToScheme(scheme)) return k8sfake.NewClientBuilder().WithScheme(scheme) } @@ -60,7 +61,7 @@ func NewFakeControllerRuntimeClient(t *testing.T, initObjs ...runtime.Object) cl require.NoError(t, snapshotv1api.AddToScheme(scheme)) require.NoError(t, storagev1api.AddToScheme(scheme)) require.NoError(t, batchv1api.AddToScheme(scheme)) - require.NoError(t, volumegroupsnapshotv1beta1.AddToScheme(scheme)) + require.NoError(t, volumegroupsnapshotv1beta2.AddToScheme(scheme)) return k8sfake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(initObjs...).Build() } diff --git a/pkg/uploader/kopia/block_restore.go b/pkg/uploader/kopia/block_restore.go index b9fba99fc..4f28a59de 100644 --- a/pkg/uploader/kopia/block_restore.go +++ b/pkg/uploader/kopia/block_restore.go @@ -35,6 +35,7 @@ type BlockOutput struct { *restore.FilesystemOutput targetFileName string + targetFile *os.File } var _ restore.Output = &BlockOutput{} @@ -52,7 +53,7 @@ func (o *BlockOutput) WriteFile(ctx context.Context, relativePath string, remote if err != nil { return errors.Wrapf(err, "failed to open file %s", o.targetFileName) } - defer targetFile.Close() + o.targetFile = targetFile buffer := make([]byte, bufferSize) @@ -101,3 +102,23 @@ func (o *BlockOutput) BeginDirectory(ctx context.Context, relativePath string, e return nil } + +func (o *BlockOutput) Flush() error { + if o.targetFile != nil { + if err := o.targetFile.Sync(); err != nil { + return errors.Wrapf(err, "error syncing block dev %v", o.targetFileName) + } + } + + return nil +} + +func (o *BlockOutput) Terminate() error { + if o.targetFile != nil { + if err := o.targetFile.Close(); err != nil { + return errors.Wrapf(err, "error closing block dev %v", o.targetFileName) + } + } + + return nil +} diff --git a/pkg/uploader/kopia/block_restore_windows.go b/pkg/uploader/kopia/block_restore_windows.go index 702e0a5e2..c747c3a91 100644 --- a/pkg/uploader/kopia/block_restore_windows.go +++ b/pkg/uploader/kopia/block_restore_windows.go @@ -40,3 +40,11 @@ func (o *BlockOutput) WriteFile(ctx context.Context, relativePath string, remote func (o *BlockOutput) BeginDirectory(ctx context.Context, relativePath string, e fs.Directory) error { return fmt.Errorf("block mode is not supported for Windows") } + +func (o *BlockOutput) Flush() error { + return flushVolume(o.targetFileName) +} + +func (o *BlockOutput) Terminate() error { + return nil +} diff --git a/pkg/uploader/kopia/flush_volume_linux.go b/pkg/uploader/kopia/flush_volume_linux.go new file mode 100644 index 000000000..98234e1b9 --- /dev/null +++ b/pkg/uploader/kopia/flush_volume_linux.go @@ -0,0 +1,50 @@ +//go:build linux +// +build linux + +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kopia + +import ( + "os" + + "github.com/pkg/errors" + "golang.org/x/sys/unix" +) + +func flushVolume(dirPath string) error { + dir, err := os.Open(dirPath) + if err != nil { + return errors.Wrapf(err, "error opening dir %v", dirPath) + } + + raw, err := dir.SyscallConn() + if err != nil { + return errors.Wrapf(err, "error getting handle of dir %v", dirPath) + } + + var syncErr error + if err := raw.Control(func(fd uintptr) { + if e := unix.Syncfs(int(fd)); e != nil { + syncErr = e + } + }); err != nil { + return errors.Wrapf(err, "error calling fs sync from %v", dirPath) + } + + return errors.Wrapf(syncErr, "error syncing fs from %v", dirPath) +} diff --git a/pkg/uploader/kopia/flush_volume_other.go b/pkg/uploader/kopia/flush_volume_other.go new file mode 100644 index 000000000..0c0959bda --- /dev/null +++ b/pkg/uploader/kopia/flush_volume_other.go @@ -0,0 +1,24 @@ +//go:build !linux +// +build !linux + +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kopia + +func flushVolume(_ string) error { + return errFlushUnsupported +} diff --git a/pkg/uploader/kopia/restore_output.go b/pkg/uploader/kopia/restore_output.go new file mode 100644 index 000000000..74311d38a --- /dev/null +++ b/pkg/uploader/kopia/restore_output.go @@ -0,0 +1,30 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kopia + +import ( + "github.com/kopia/kopia/snapshot/restore" + "github.com/pkg/errors" +) + +var errFlushUnsupported = errors.New("flush is not supported") + +type RestoreOutput interface { + restore.Output + Flush() error + Terminate() error +} diff --git a/pkg/uploader/kopia/snapshot.go b/pkg/uploader/kopia/snapshot.go index fce620eb7..1924ed35b 100644 --- a/pkg/uploader/kopia/snapshot.go +++ b/pkg/uploader/kopia/snapshot.go @@ -53,6 +53,7 @@ var loadSnapshotFunc = snapshot.LoadSnapshot var listSnapshotsFunc = snapshot.ListSnapshots var filesystemEntryFunc = snapshotfs.FilesystemEntryFromIDWithPath var restoreEntryFunc = restore.Entry +var flushVolumeFunc = flushVolume const UploaderConfigMultipartKey = "uploader-multipart" const MaxErrorReported = 10 @@ -375,6 +376,18 @@ func findPreviousSnapshotManifest(ctx context.Context, rep repo.Repository, sour return result, nil } +type fileSystemRestoreOutput struct { + *restore.FilesystemOutput +} + +func (o *fileSystemRestoreOutput) Flush() error { + return flushVolumeFunc(o.TargetPath) +} + +func (o *fileSystemRestoreOutput) Terminate() error { + return nil +} + // Restore restore specific sourcePath with given snapshotID and update progress func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, uploaderCfg map[string]string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { @@ -434,13 +447,23 @@ func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, return 0, 0, errors.Wrap(err, "error to init output") } - var output restore.Output = fsOutput + var output RestoreOutput if volMode == uploader.PersistentVolumeBlock { output = &BlockOutput{ FilesystemOutput: fsOutput, } + } else { + output = &fileSystemRestoreOutput{ + FilesystemOutput: fsOutput, + } } + defer func() { + if err := output.Terminate(); err != nil { + log.Warnf("error terminating restore output for %v", path) + } + }() + stat, err := restoreEntryFunc(kopiaCtx, rep, output, rootEntry, restore.Options{ Parallel: restoreConcurrency, RestoreDirEntryAtDepth: math.MaxInt32, @@ -453,5 +476,16 @@ func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, if err != nil { return 0, 0, errors.Wrapf(err, "Failed to copy snapshot data to the target") } + + if err := output.Flush(); err != nil { + if err == errFlushUnsupported { + log.Warnf("Skip flushing data for %v under the current OS %v", path, runtime.GOOS) + } else { + return 0, 0, errors.Wrapf(err, "Failed to flush data to target") + } + } else { + log.Infof("Flush done for volume dir %v", path) + } + return stat.RestoredTotalFileSize, stat.RestoredFileCount, nil } diff --git a/pkg/uploader/kopia/snapshot_test.go b/pkg/uploader/kopia/snapshot_test.go index 2423ba590..984b92af5 100644 --- a/pkg/uploader/kopia/snapshot_test.go +++ b/pkg/uploader/kopia/snapshot_test.go @@ -675,6 +675,7 @@ func TestRestore(t *testing.T) { invalidManifestType bool filesystemEntryFunc func(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Entry, error) restoreEntryFunc func(ctx context.Context, rep repo.Repository, output restore.Output, rootEntry fs.Entry, options restore.Options) (restore.Stats, error) + flushVolumeFunc func(string) error dest string expectedBytes int64 expectedCount int32 @@ -757,6 +758,30 @@ func TestRestore(t *testing.T) { volMode: uploader.PersistentVolumeBlock, dest: "/tmp", }, + { + name: "Flush is not supported", + filesystemEntryFunc: func(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Entry, error) { + return snapshotfs.EntryFromDirEntry(rep, &snapshot.DirEntry{Type: snapshot.EntryTypeFile}), nil + }, + restoreEntryFunc: func(ctx context.Context, rep repo.Repository, output restore.Output, rootEntry fs.Entry, options restore.Options) (restore.Stats, error) { + return restore.Stats{}, nil + }, + flushVolumeFunc: func(string) error { return errFlushUnsupported }, + snapshotID: "snapshot-123", + expectedError: nil, + }, + { + name: "Flush fails", + filesystemEntryFunc: func(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Entry, error) { + return snapshotfs.EntryFromDirEntry(rep, &snapshot.DirEntry{Type: snapshot.EntryTypeFile}), nil + }, + restoreEntryFunc: func(ctx context.Context, rep repo.Repository, output restore.Output, rootEntry fs.Entry, options restore.Options) (restore.Stats, error) { + return restore.Stats{}, nil + }, + flushVolumeFunc: func(string) error { return errors.New("fake-flush-error") }, + snapshotID: "snapshot-123", + expectedError: errors.New("fake-flush-error"), + }, } em := &manifest.EntryMetadata{ @@ -784,6 +809,10 @@ func TestRestore(t *testing.T) { restoreEntryFunc = tc.restoreEntryFunc } + if tc.flushVolumeFunc != nil { + flushVolumeFunc = tc.flushVolumeFunc + } + repoWriterMock := &repomocks.RepositoryWriter{} repoWriterMock.On("GetManifest", mock.Anything, mock.Anything, mock.Anything).Return(em, nil) repoWriterMock.On("OpenObject", mock.Anything, mock.Anything).Return(em, nil) diff --git a/pkg/uploader/provider/kopia_test.go b/pkg/uploader/provider/kopia_test.go index 74eaa67f7..734bdb176 100644 --- a/pkg/uploader/provider/kopia_test.go +++ b/pkg/uploader/provider/kopia_test.go @@ -294,6 +294,10 @@ func TestGetPassword(t *testing.T) { } } +type MockCredentialGetter struct { + mock.Mock +} + func (m *MockCredentialGetter) GetCredentials() (string, error) { args := m.Called() return args.String(0), args.Error(1) diff --git a/pkg/uploader/provider/provider.go b/pkg/uploader/provider/provider.go index fe1dd3091..95a34b1a0 100644 --- a/pkg/uploader/provider/provider.go +++ b/pkg/uploader/provider/provider.go @@ -87,6 +87,6 @@ func NewUploaderProvider( if uploaderType == uploader.KopiaType { return NewKopiaUploaderProvider(requesterType, ctx, credGetter, backupRepo, log) } else { - return NewResticUploaderProvider(repoIdentifier, bsl, credGetter, repoKeySelector, log) + return nil, errors.Errorf("unsupported uploader type %v", uploaderType) } } diff --git a/pkg/uploader/provider/provider_test.go b/pkg/uploader/provider/provider_test.go index 199091e32..8f447725b 100644 --- a/pkg/uploader/provider/provider_test.go +++ b/pkg/uploader/provider/provider_test.go @@ -75,7 +75,7 @@ func TestNewUploaderProvider(t *testing.T) { UploaderType: "restic", RequestorType: "requester", needFromFile: true, - ExpectedError: "", + ExpectedError: "unsupported uploader type restic", }, } diff --git a/pkg/uploader/provider/restic.go b/pkg/uploader/provider/restic.go deleted file mode 100644 index 93b907be9..000000000 --- a/pkg/uploader/provider/restic.go +++ /dev/null @@ -1,269 +0,0 @@ -/* -Copyright The Velero Contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package provider - -import ( - "context" - "fmt" - "os" - "strings" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - corev1api "k8s.io/api/core/v1" - - "github.com/vmware-tanzu/velero/internal/credentials" - velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - "github.com/vmware-tanzu/velero/pkg/restic" - "github.com/vmware-tanzu/velero/pkg/uploader" - uploaderutil "github.com/vmware-tanzu/velero/pkg/uploader/util" - "github.com/vmware-tanzu/velero/pkg/util/filesystem" -) - -// resticBackupCMDFunc and resticRestoreCMDFunc are mainly used to make testing more convenient -var resticBackupCMDFunc = restic.BackupCommand -var resticBackupFunc = restic.RunBackup -var resticGetSnapshotFunc = restic.GetSnapshotCommand -var resticGetSnapshotIDFunc = restic.GetSnapshotID -var resticRestoreCMDFunc = restic.RestoreCommand -var resticTempCACertFileFunc = restic.TempCACertFile -var resticCmdEnvFunc = restic.CmdEnv - -type resticProvider struct { - repoIdentifier string - credentialsFile string - caCertFile string - cmdEnv []string - extraFlags []string - bsl *velerov1api.BackupStorageLocation - log logrus.FieldLogger -} - -func NewResticUploaderProvider( - repoIdentifier string, - bsl *velerov1api.BackupStorageLocation, - credGetter *credentials.CredentialGetter, - repoKeySelector *corev1api.SecretKeySelector, - log logrus.FieldLogger, -) (Provider, error) { - provider := resticProvider{ - repoIdentifier: repoIdentifier, - bsl: bsl, - log: log, - } - - var err error - provider.credentialsFile, err = credGetter.FromFile.Path(repoKeySelector) - if err != nil { - return nil, errors.Wrap(err, "error creating temp restic credentials file") - } - - // if there's a caCert on the ObjectStorage, write it to disk so that it can be passed to restic - if bsl.Spec.ObjectStorage != nil { - var caCertData []byte - - // Try CACertRef first (new method), then fall back to CACert (deprecated) - if bsl.Spec.ObjectStorage.CACertRef != nil { - caCertString, err := credGetter.FromSecret.Get(bsl.Spec.ObjectStorage.CACertRef) - if err != nil { - return nil, errors.Wrap(err, "error getting CA certificate from secret") - } - caCertData = []byte(caCertString) - } else if bsl.Spec.ObjectStorage.CACert != nil { - caCertData = bsl.Spec.ObjectStorage.CACert - } - - if caCertData != nil { - provider.caCertFile, err = resticTempCACertFileFunc(caCertData, bsl.Name, filesystem.NewFileSystem()) - if err != nil { - return nil, errors.Wrap(err, "error create temp cert file") - } - } - } - - provider.cmdEnv, err = resticCmdEnvFunc(bsl, credGetter.FromFile) - if err != nil { - return nil, errors.Wrap(err, "error generating repository cmnd env") - } - - // #4820: restrieve insecureSkipTLSVerify from BSL configuration for - // AWS plugin. If nothing is return, that means insecureSkipTLSVerify - // is not enable for Restic command. - skipTLSRet := restic.GetInsecureSkipTLSVerifyFromBSL(bsl, log) - if len(skipTLSRet) > 0 { - provider.extraFlags = append(provider.extraFlags, skipTLSRet) - } - - return &provider, nil -} - -func (rp *resticProvider) Close(ctx context.Context) error { - _, err := os.Stat(rp.credentialsFile) - if err == nil { - return os.Remove(rp.credentialsFile) - } else if !os.IsNotExist(err) { - return errors.Errorf("failed to get file %s info with error %v", rp.credentialsFile, err) - } - - _, err = os.Stat(rp.caCertFile) - if err == nil { - return os.Remove(rp.caCertFile) - } else if !os.IsNotExist(err) { - return errors.Errorf("failed to get file %s info with error %v", rp.caCertFile, err) - } - return nil -} - -// RunBackup runs a `backup` command and watches the output to provide -// progress updates to the caller and return snapshotID, isEmptySnapshot, error -func (rp *resticProvider) RunBackup( - ctx context.Context, - path string, - realSource string, - tags map[string]string, - forceFull bool, - parentSnapshot string, - volMode uploader.PersistentVolumeMode, - uploaderCfg map[string]string, - updater uploader.ProgressUpdater) (string, bool, int64, int64, error) { - if updater == nil { - return "", false, 0, 0, errors.New("Need to initial backup progress updater first") - } - - if path == "" { - return "", false, 0, 0, errors.New("path is empty") - } - - if realSource != "" { - return "", false, 0, 0, errors.New("real source is not empty, this is not supported by restic uploader") - } - - if volMode == uploader.PersistentVolumeBlock { - return "", false, 0, 0, errors.New("unable to support block mode") - } - - log := rp.log.WithFields(logrus.Fields{ - "path": path, - "parentSnapshot": parentSnapshot, - }) - - if len(uploaderCfg) > 0 { - parallelFilesUpload, err := uploaderutil.GetParallelFilesUpload(uploaderCfg) - if err != nil { - return "", false, 0, 0, errors.Wrap(err, "failed to get uploader config") - } - if parallelFilesUpload > 0 { - log.Warnf("ParallelFilesUpload is set to %d, but restic does not support parallel file uploads. Ignoring.", parallelFilesUpload) - } - } - - backupCmd := resticBackupCMDFunc(rp.repoIdentifier, rp.credentialsFile, path, tags) - backupCmd.Env = rp.cmdEnv - backupCmd.CACertFile = rp.caCertFile - if len(rp.extraFlags) != 0 { - backupCmd.ExtraFlags = append(backupCmd.ExtraFlags, rp.extraFlags...) - } - - if parentSnapshot != "" { - backupCmd.ExtraFlags = append(backupCmd.ExtraFlags, fmt.Sprintf("--parent=%s", parentSnapshot)) - } - - summary, stderrBuf, err := resticBackupFunc(backupCmd, log, updater) - if err != nil { - if strings.Contains(stderrBuf, "snapshot is empty") { - log.Debugf("Restic backup got empty dir with %s path", path) - return "", true, 0, 0, nil - } - return "", false, 0, 0, errors.WithStack(fmt.Errorf("error running restic backup command %s with error: %v stderr: %v", backupCmd.String(), err, stderrBuf)) - } - // GetSnapshotID - snapshotIDCmd := resticGetSnapshotFunc(rp.repoIdentifier, rp.credentialsFile, tags) - snapshotIDCmd.Env = rp.cmdEnv - snapshotIDCmd.CACertFile = rp.caCertFile - if len(rp.extraFlags) != 0 { - snapshotIDCmd.ExtraFlags = append(snapshotIDCmd.ExtraFlags, rp.extraFlags...) - } - snapshotID, err := resticGetSnapshotIDFunc(snapshotIDCmd) - if err != nil { - return "", false, 0, 0, errors.WithStack(fmt.Errorf("error getting snapshot id with error: %v", err)) - } - log.Infof("Run command=%s, stdout=%s, stderr=%s", backupCmd.String(), summary, stderrBuf) - return snapshotID, false, 0, 0, nil -} - -// RunRestore runs a `restore` command and monitors the volume size to -// provide progress updates to the caller. -func (rp *resticProvider) RunRestore( - ctx context.Context, - snapshotID string, - volumePath string, - volMode uploader.PersistentVolumeMode, - uploaderCfg map[string]string, - updater uploader.ProgressUpdater) (int64, error) { - if updater == nil { - return 0, errors.New("Need to initial backup progress updater first") - } - log := rp.log.WithFields(logrus.Fields{ - "snapshotID": snapshotID, - "volumePath": volumePath, - }) - - if volMode == uploader.PersistentVolumeBlock { - return 0, errors.New("unable to support block mode") - } - - restoreCmd := resticRestoreCMDFunc(rp.repoIdentifier, rp.credentialsFile, snapshotID, volumePath) - restoreCmd.Env = rp.cmdEnv - restoreCmd.CACertFile = rp.caCertFile - if len(rp.extraFlags) != 0 { - restoreCmd.ExtraFlags = append(restoreCmd.ExtraFlags, rp.extraFlags...) - } - - extraFlags, err := rp.parseRestoreExtraFlags(uploaderCfg) - if err != nil { - return 0, errors.Wrap(err, "failed to parse uploader config") - } else if len(extraFlags) != 0 { - restoreCmd.ExtraFlags = append(restoreCmd.ExtraFlags, extraFlags...) - } - - stdout, stderr, err := restic.RunRestore(restoreCmd, log, updater) - - log.Infof("Run command=%v, stdout=%s, stderr=%s", restoreCmd, stdout, stderr) - return 0, err -} - -func (rp *resticProvider) parseRestoreExtraFlags(uploaderCfg map[string]string) ([]string, error) { - extraFlags := []string{} - if len(uploaderCfg) == 0 { - return extraFlags, nil - } - - writeSparseFiles, err := uploaderutil.GetWriteSparseFiles(uploaderCfg) - if err != nil { - return extraFlags, errors.Wrap(err, "failed to get uploader config") - } - - if writeSparseFiles { - extraFlags = append(extraFlags, "--sparse") - } - - if restoreConcurrency, err := uploaderutil.GetRestoreConcurrency(uploaderCfg); err == nil && restoreConcurrency > 0 { - return extraFlags, errors.New("restic does not support parallel restore") - } - - return extraFlags, nil -} diff --git a/pkg/uploader/provider/restic_test.go b/pkg/uploader/provider/restic_test.go deleted file mode 100644 index 24eb11e04..000000000 --- a/pkg/uploader/provider/restic_test.go +++ /dev/null @@ -1,464 +0,0 @@ -/* -Copyright The Velero Contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package provider - -import ( - "errors" - "os" - "reflect" - "strings" - "testing" - - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - corev1api "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - - "github.com/vmware-tanzu/velero/internal/credentials" - velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - "github.com/vmware-tanzu/velero/pkg/builder" - "github.com/vmware-tanzu/velero/pkg/restic" - "github.com/vmware-tanzu/velero/pkg/uploader" - "github.com/vmware-tanzu/velero/pkg/util" - "github.com/vmware-tanzu/velero/pkg/util/filesystem" -) - -func TestResticRunBackup(t *testing.T) { - testCases := []struct { - name string - nilUpdater bool - parentSnapshot string - rp *resticProvider - volMode uploader.PersistentVolumeMode - hookBackupFunc func(string, string, string, map[string]string) *restic.Command - hookResticBackupFunc func(*restic.Command, logrus.FieldLogger, uploader.ProgressUpdater) (string, string, error) - hookResticGetSnapshotFunc func(string, string, map[string]string) *restic.Command - hookResticGetSnapshotIDFunc func(*restic.Command) (string, error) - errorHandleFunc func(err error) bool - }{ - { - name: "nil uploader", - rp: &resticProvider{log: logrus.New()}, - nilUpdater: true, - hookBackupFunc: func(repoIdentifier string, passwordFile string, path string, tags map[string]string) *restic.Command { - return &restic.Command{Command: "date"} - }, - errorHandleFunc: func(err error) bool { - return strings.Contains(err.Error(), "Need to initial backup progress updater first") - }, - }, - { - name: "wrong restic execute command", - rp: &resticProvider{log: logrus.New()}, - hookBackupFunc: func(repoIdentifier string, passwordFile string, path string, tags map[string]string) *restic.Command { - return &restic.Command{Command: "date"} - }, - errorHandleFunc: func(err error) bool { - return strings.Contains(err.Error(), "error running") - }, - }, { - name: "has parent snapshot", - rp: &resticProvider{log: logrus.New()}, - parentSnapshot: "parentSnapshot", - hookBackupFunc: func(repoIdentifier string, passwordFile string, path string, tags map[string]string) *restic.Command { - return &restic.Command{Command: "date"} - }, - hookResticBackupFunc: func(*restic.Command, logrus.FieldLogger, uploader.ProgressUpdater) (string, string, error) { - return "", "", nil - }, - - hookResticGetSnapshotIDFunc: func(*restic.Command) (string, error) { return "test-snapshot-id", nil }, - errorHandleFunc: func(err error) bool { - return err == nil - }, - }, - { - name: "has extra flags", - rp: &resticProvider{log: logrus.New(), extraFlags: []string{"testFlags"}}, - hookBackupFunc: func(string, string, string, map[string]string) *restic.Command { - return &restic.Command{Command: "date"} - }, - hookResticBackupFunc: func(*restic.Command, logrus.FieldLogger, uploader.ProgressUpdater) (string, string, error) { - return "", "", nil - }, - hookResticGetSnapshotIDFunc: func(*restic.Command) (string, error) { return "test-snapshot-id", nil }, - errorHandleFunc: func(err error) bool { - return err == nil - }, - }, - { - name: "failed to get snapshot id", - rp: &resticProvider{log: logrus.New(), extraFlags: []string{"testFlags"}}, - hookBackupFunc: func(string, string, string, map[string]string) *restic.Command { - return &restic.Command{Command: "date"} - }, - hookResticBackupFunc: func(*restic.Command, logrus.FieldLogger, uploader.ProgressUpdater) (string, string, error) { - return "", "", nil - }, - hookResticGetSnapshotIDFunc: func(*restic.Command) (string, error) { - return "test-snapshot-id", errors.New("failed to get snapshot id") - }, - errorHandleFunc: func(err error) bool { - return strings.Contains(err.Error(), "failed to get snapshot id") - }, - }, - { - name: "failed to use block mode", - rp: &resticProvider{log: logrus.New(), extraFlags: []string{"testFlags"}}, - volMode: uploader.PersistentVolumeBlock, - errorHandleFunc: func(err error) bool { - return strings.Contains(err.Error(), "unable to support block mode") - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - var err error - parentSnapshot := tc.parentSnapshot - if tc.hookBackupFunc != nil { - resticBackupCMDFunc = tc.hookBackupFunc - } - if tc.hookResticBackupFunc != nil { - resticBackupFunc = tc.hookResticBackupFunc - } - if tc.hookResticGetSnapshotFunc != nil { - resticGetSnapshotFunc = tc.hookResticGetSnapshotFunc - } - if tc.hookResticGetSnapshotIDFunc != nil { - resticGetSnapshotIDFunc = tc.hookResticGetSnapshotIDFunc - } - if tc.volMode == "" { - tc.volMode = uploader.PersistentVolumeFilesystem - } - if !tc.nilUpdater { - updater := FakeBackupProgressUpdater{PodVolumeBackup: &velerov1api.PodVolumeBackup{}, Log: tc.rp.log, Ctx: t.Context(), Cli: fake.NewClientBuilder().WithScheme(util.VeleroScheme).Build()} - _, _, _, _, err = tc.rp.RunBackup(t.Context(), "var", "", map[string]string{}, false, parentSnapshot, tc.volMode, map[string]string{}, &updater) - } else { - _, _, _, _, err = tc.rp.RunBackup(t.Context(), "var", "", map[string]string{}, false, parentSnapshot, tc.volMode, map[string]string{}, nil) - } - - tc.rp.log.Infof("test name %v error %v", tc.name, err) - require.True(t, tc.errorHandleFunc(err)) - }) - } -} - -func TestResticRunRestore(t *testing.T) { - resticRestoreCMDFunc = func(repoIdentifier, passwordFile, snapshotID, target string) *restic.Command { - return &restic.Command{Args: []string{""}} - } - testCases := []struct { - name string - rp *resticProvider - nilUpdater bool - hookResticRestoreFunc func(repoIdentifier, passwordFile, snapshotID, target string) *restic.Command - errorHandleFunc func(err error) bool - volMode uploader.PersistentVolumeMode - }{ - { - name: "wrong restic execute command", - rp: &resticProvider{log: logrus.New()}, - nilUpdater: true, - errorHandleFunc: func(err error) bool { - return strings.Contains(err.Error(), "Need to initial backup progress updater first") - }, - }, - { - name: "has extral flags", - rp: &resticProvider{log: logrus.New(), extraFlags: []string{"test-extra-flags"}}, - hookResticRestoreFunc: func(repoIdentifier, passwordFile, snapshotID, target string) *restic.Command { - return &restic.Command{Args: []string{"date"}} - }, - errorHandleFunc: func(err error) bool { - return strings.Contains(err.Error(), "error running command") - }, - }, - { - name: "wrong restic execute command", - rp: &resticProvider{log: logrus.New()}, - hookResticRestoreFunc: func(repoIdentifier, passwordFile, snapshotID, target string) *restic.Command { - return &restic.Command{Args: []string{"date"}} - }, - errorHandleFunc: func(err error) bool { - return strings.Contains(err.Error(), "error running command") - }, - }, - { - name: "error block volume mode", - rp: &resticProvider{log: logrus.New()}, - errorHandleFunc: func(err error) bool { - return strings.Contains(err.Error(), "unable to support block mode") - }, - volMode: uploader.PersistentVolumeBlock, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - if tc.volMode == "" { - tc.volMode = uploader.PersistentVolumeFilesystem - } - resticRestoreCMDFunc = tc.hookResticRestoreFunc - if tc.volMode == "" { - tc.volMode = uploader.PersistentVolumeFilesystem - } - var err error - if !tc.nilUpdater { - updater := FakeBackupProgressUpdater{PodVolumeBackup: &velerov1api.PodVolumeBackup{}, Log: tc.rp.log, Ctx: t.Context(), Cli: fake.NewClientBuilder().WithScheme(util.VeleroScheme).Build()} - _, err = tc.rp.RunRestore(t.Context(), "", "var", tc.volMode, map[string]string{}, &updater) - } else { - _, err = tc.rp.RunRestore(t.Context(), "", "var", tc.volMode, map[string]string{}, nil) - } - - tc.rp.log.Infof("test name %v error %v", tc.name, err) - require.True(t, tc.errorHandleFunc(err)) - }) - } -} - -func TestClose(t *testing.T) { - t.Run("Delete existing credentials file", func(t *testing.T) { - // Create temporary files for the credentials and caCert - credentialsFile, err := os.CreateTemp(t.TempDir(), "credentialsFile") - if err != nil { - t.Fatalf("failed to create temp file: %v", err) - } - defer os.Remove(credentialsFile.Name()) - - caCertFile, err := os.CreateTemp(t.TempDir(), "caCertFile") - if err != nil { - t.Fatalf("failed to create temp file: %v", err) - } - defer os.Remove(caCertFile.Name()) - rp := &resticProvider{ - credentialsFile: credentialsFile.Name(), - caCertFile: caCertFile.Name(), - } - // Test deleting an existing credentials file - err = rp.Close(t.Context()) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - - _, err = os.Stat(rp.credentialsFile) - if !os.IsNotExist(err) { - t.Errorf("expected credentials file to be deleted, got error: %v", err) - } - }) - - t.Run("Delete existing caCert file", func(t *testing.T) { - // Create temporary files for the credentials and caCert - caCertFile, err := os.CreateTemp(t.TempDir(), "caCertFile") - if err != nil { - t.Fatalf("failed to create temp file: %v", err) - } - defer os.Remove(caCertFile.Name()) - rp := &resticProvider{ - credentialsFile: "", - caCertFile: "", - } - err = rp.Close(t.Context()) - // Test deleting an existing caCert file - if err != nil { - t.Errorf("unexpected error: %v", err) - } - - _, err = os.Stat(rp.caCertFile) - if !os.IsNotExist(err) { - t.Errorf("expected caCert file to be deleted, got error: %v", err) - } - }) -} - -type MockCredentialGetter struct { - mock.Mock -} - -func (m *MockCredentialGetter) Path(selector *corev1api.SecretKeySelector) (string, error) { - args := m.Called(selector) - return args.Get(0).(string), args.Error(1) -} - -func TestNewResticUploaderProvider(t *testing.T) { - testCases := []struct { - name string - emptyBSL bool - mockCredFunc func(*MockCredentialGetter, *corev1api.SecretKeySelector) - resticCmdEnvFunc func(backupLocation *velerov1api.BackupStorageLocation, credentialFileStore credentials.FileStore) ([]string, error) - resticTempCACertFileFunc func(caCert []byte, bsl string, fs filesystem.Interface) (string, error) - checkFunc func(t *testing.T, provider Provider, err error) - }{ - { - name: "No error in creating temp credentials file", - mockCredFunc: func(credGetter *MockCredentialGetter, repoKeySelector *corev1api.SecretKeySelector) { - credGetter.On("Path", repoKeySelector).Return("temp-credentials", nil) - }, - checkFunc: func(t *testing.T, provider Provider, err error) { - t.Helper() - require.NoError(t, err) - assert.NotNil(t, provider) - }, - }, { - name: "Error in creating temp credentials file", - mockCredFunc: func(credGetter *MockCredentialGetter, repoKeySelector *corev1api.SecretKeySelector) { - credGetter.On("Path", repoKeySelector).Return("", errors.New("error creating temp credentials file")) - }, - checkFunc: func(t *testing.T, provider Provider, err error) { - t.Helper() - require.Error(t, err) - assert.Nil(t, provider) - }, - }, { - name: "ObjectStorage with CACert present and creating CACert file failed", - mockCredFunc: func(credGetter *MockCredentialGetter, repoKeySelector *corev1api.SecretKeySelector) { - credGetter.On("Path", repoKeySelector).Return("temp-credentials", nil) - }, - resticTempCACertFileFunc: func(caCert []byte, bsl string, fs filesystem.Interface) (string, error) { - return "", errors.New("error writing CACert file") - }, - checkFunc: func(t *testing.T, provider Provider, err error) { - t.Helper() - require.Error(t, err) - assert.Nil(t, provider) - }, - }, { - name: "Generating repository cmd failed", - mockCredFunc: func(credGetter *MockCredentialGetter, repoKeySelector *corev1api.SecretKeySelector) { - credGetter.On("Path", repoKeySelector).Return("temp-credentials", nil) - }, - resticTempCACertFileFunc: func(caCert []byte, bsl string, fs filesystem.Interface) (string, error) { - return "test-ca", nil - }, - resticCmdEnvFunc: func(backupLocation *velerov1api.BackupStorageLocation, credentialFileStore credentials.FileStore) ([]string, error) { - return nil, errors.New("error generating repository cmnd env") - }, - checkFunc: func(t *testing.T, provider Provider, err error) { - t.Helper() - require.Error(t, err) - assert.Nil(t, provider) - }, - }, { - name: "New provider with not nil bsl", - mockCredFunc: func(credGetter *MockCredentialGetter, repoKeySelector *corev1api.SecretKeySelector) { - credGetter.On("Path", repoKeySelector).Return("temp-credentials", nil) - }, - resticTempCACertFileFunc: func(caCert []byte, bsl string, fs filesystem.Interface) (string, error) { - return "test-ca", nil - }, - resticCmdEnvFunc: func(backupLocation *velerov1api.BackupStorageLocation, credentialFileStore credentials.FileStore) ([]string, error) { - return nil, nil - }, - checkFunc: func(t *testing.T, provider Provider, err error) { - t.Helper() - require.NoError(t, err) - assert.NotNil(t, provider) - }, - }, - { - name: "New provider with nil bsl", - emptyBSL: true, - mockCredFunc: func(credGetter *MockCredentialGetter, repoKeySelector *corev1api.SecretKeySelector) { - credGetter.On("Path", repoKeySelector).Return("temp-credentials", nil) - }, - resticTempCACertFileFunc: func(caCert []byte, bsl string, fs filesystem.Interface) (string, error) { - return "test-ca", nil - }, - resticCmdEnvFunc: func(backupLocation *velerov1api.BackupStorageLocation, credentialFileStore credentials.FileStore) ([]string, error) { - return nil, nil - }, - checkFunc: func(t *testing.T, provider Provider, err error) { - t.Helper() - require.NoError(t, err) - assert.NotNil(t, provider) - }, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - repoIdentifier := "my-repo" - bsl := &velerov1api.BackupStorageLocation{} - if !tc.emptyBSL { - bsl = builder.ForBackupStorageLocation("test-ns", "test-name").CACert([]byte("my-cert")).Result() - } - credGetter := &credentials.CredentialGetter{} - repoKeySelector := &corev1api.SecretKeySelector{} - log := logrus.New() - - // Mock CredentialGetter - mockCredGetter := &MockCredentialGetter{} - credGetter.FromFile = mockCredGetter - tc.mockCredFunc(mockCredGetter, repoKeySelector) - if tc.resticCmdEnvFunc != nil { - resticCmdEnvFunc = tc.resticCmdEnvFunc - } - if tc.resticTempCACertFileFunc != nil { - resticTempCACertFileFunc = tc.resticTempCACertFileFunc - } - provider, err := NewResticUploaderProvider(repoIdentifier, bsl, credGetter, repoKeySelector, log) - tc.checkFunc(t, provider, err) - }) - } -} - -func TestParseUploaderConfig(t *testing.T) { - rp := &resticProvider{} - - testCases := []struct { - name string - uploaderConfig map[string]string - expectedFlags []string - }{ - { - name: "SparseFilesEnabled", - uploaderConfig: map[string]string{ - "WriteSparseFiles": "true", - }, - expectedFlags: []string{"--sparse"}, - }, - { - name: "SparseFilesDisabled", - uploaderConfig: map[string]string{ - "writeSparseFiles": "false", - }, - expectedFlags: []string{}, - }, - { - name: "RestoreConcorrency", - uploaderConfig: map[string]string{ - "Parallel": "5", - }, - expectedFlags: []string{}, - }, - } - - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - result, err := rp.parseRestoreExtraFlags(testCase.uploaderConfig) - if err != nil { - t.Errorf("Test case %s failed with error: %v", testCase.name, err) - return - } - - if !reflect.DeepEqual(result, testCase.expectedFlags) { - t.Errorf("Test case %s failed. Expected: %v, Got: %v", testCase.name, testCase.expectedFlags, result) - } - }) - } -} diff --git a/pkg/uploader/types.go b/pkg/uploader/types.go index f69cbf072..52f8ca5bf 100644 --- a/pkg/uploader/types.go +++ b/pkg/uploader/types.go @@ -22,7 +22,6 @@ import ( ) const ( - ResticType = "restic" KopiaType = "kopia" SnapshotRequesterTag = "snapshot-requester" SnapshotUploaderTag = "snapshot-uploader" diff --git a/pkg/util/collections/includes_excludes.go b/pkg/util/collections/includes_excludes.go index b3d18d068..ab63eaa72 100644 --- a/pkg/util/collections/includes_excludes.go +++ b/pkg/util/collections/includes_excludes.go @@ -666,10 +666,22 @@ func validateNamespaceName(ns string) []error { return nil } - // Kubernetes does not allow asterisks in namespaces but Velero uses them as - // wildcards. Replace asterisks with an arbitrary letter to pass Kubernetes - // validation. - tmpNamespace := strings.ReplaceAll(ns, "*", "x") + // Validate the namespace name to ensure it is a valid wildcard pattern + if err := wildcard.ValidateNamespaceName(ns); err != nil { + return []error{err} + } + + // Kubernetes does not allow wildcard characters in namespaces but Velero uses them + // for glob patterns. Replace wildcard characters with valid characters to pass + // Kubernetes validation. + tmpNamespace := ns + + // Replace glob wildcard characters with valid alphanumeric characters + // Note: Validation of wildcard patterns is handled by the wildcard package. + tmpNamespace = strings.ReplaceAll(tmpNamespace, "*", "x") // matches any sequence + tmpNamespace = strings.ReplaceAll(tmpNamespace, "?", "x") // matches single character + tmpNamespace = strings.ReplaceAll(tmpNamespace, "[", "x") // character class start + tmpNamespace = strings.ReplaceAll(tmpNamespace, "]", "x") // character class end if errMsgs := validation.ValidateNamespaceName(tmpNamespace, false); errMsgs != nil { for _, msg := range errMsgs { diff --git a/pkg/util/collections/includes_excludes_test.go b/pkg/util/collections/includes_excludes_test.go index b7fdcd3af..1d700a729 100644 --- a/pkg/util/collections/includes_excludes_test.go +++ b/pkg/util/collections/includes_excludes_test.go @@ -289,6 +289,54 @@ func TestValidateNamespaceIncludesExcludes(t *testing.T) { excludes: []string{"bar"}, wantErr: true, }, + { + name: "glob characters in includes should not error", + includes: []string{"kube-*", "test-?", "ns-[0-9]"}, + excludes: []string{}, + wantErr: false, + }, + { + name: "glob characters in excludes should not error", + includes: []string{"default"}, + excludes: []string{"test-*", "app-?", "ns-[1-5]"}, + wantErr: false, + }, + { + name: "character class in includes should not error", + includes: []string{"ns-[abc]", "test-[0-9]"}, + excludes: []string{}, + wantErr: false, + }, + { + name: "mixed glob patterns should not error", + includes: []string{"kube-*", "test-?"}, + excludes: []string{"*-test", "debug-[0-9]"}, + wantErr: false, + }, + { + name: "pipe character in includes should error", + includes: []string{"namespace|other"}, + excludes: []string{}, + wantErr: true, + }, + { + name: "parentheses in includes should error", + includes: []string{"namespace(prod)", "test-(dev)"}, + excludes: []string{}, + wantErr: true, + }, + { + name: "exclamation mark in includes should error", + includes: []string{"!namespace", "test!"}, + excludes: []string{}, + wantErr: true, + }, + { + name: "unsupported characters in excludes should error", + includes: []string{"default"}, + excludes: []string{"test|prod", "app(staging)"}, + wantErr: true, + }, } for _, tc := range tests { @@ -1082,16 +1130,6 @@ func TestExpandIncludesExcludes(t *testing.T) { expectedWildcardExpanded: true, expectError: false, }, - { - name: "brace wildcard pattern", - includes: []string{"app-{prod,dev}"}, - excludes: []string{}, - activeNamespaces: []string{"app-prod", "app-dev", "app-test", "default"}, - expectedIncludes: []string{"app-prod", "app-dev"}, - expectedExcludes: []string{}, - expectedWildcardExpanded: true, - expectError: false, - }, { name: "empty activeNamespaces with wildcards", includes: []string{"kube-*"}, @@ -1233,13 +1271,6 @@ func TestResolveNamespaceList(t *testing.T) { expectedNamespaces: []string{"kube-system", "kube-public"}, preExpandWildcards: true, }, - { - name: "complex wildcard pattern", - includes: []string{"app-{prod,dev}", "kube-*"}, - excludes: []string{"*-test"}, - activeNamespaces: []string{"app-prod", "app-dev", "app-test", "kube-system", "kube-test", "default"}, - expectedNamespaces: []string{"app-prod", "app-dev", "kube-system"}, - }, { name: "question mark wildcard pattern", includes: []string{"ns-?"}, diff --git a/pkg/util/kube/node.go b/pkg/util/kube/node.go index da68183a5..ba6853624 100644 --- a/pkg/util/kube/node.go +++ b/pkg/util/kube/node.go @@ -17,7 +17,6 @@ package kube import ( "context" - "fmt" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -34,6 +33,11 @@ const ( NodeOSLabel = "kubernetes.io/os" ) +var realNodeOSMap = map[string]string{ + "linux": NodeOSLinux, + "windows": NodeOSWindows, +} + func IsLinuxNode(ctx context.Context, nodeName string, client client.Client) error { node := &corev1api.Node{} if err := client.Get(ctx, types.NamespacedName{Name: nodeName}, node); err != nil { @@ -41,12 +45,11 @@ func IsLinuxNode(ctx context.Context, nodeName string, client client.Client) err } os, found := node.Labels[NodeOSLabel] - if !found { return errors.Errorf("no os type label for node %s", nodeName) } - if os != NodeOSLinux { + if getRealOS(os) != NodeOSLinux { return errors.Errorf("os type %s for node %s is not linux", os, nodeName) } @@ -72,7 +75,7 @@ func withOSNode(ctx context.Context, client client.Client, osType string, log lo for _, node := range nodeList.Items { os, found := node.Labels[NodeOSLabel] - if os == osType { + if getRealOS(os) == osType { return true } @@ -98,7 +101,7 @@ func GetNodeOS(ctx context.Context, nodeName string, nodeClient corev1client.Cor return "", nil } - return node.Labels[NodeOSLabel], nil + return getRealOS(node.Labels[NodeOSLabel]), nil } func HasNodeWithOS(ctx context.Context, os string, nodeClient corev1client.CoreV1Interface) error { @@ -106,14 +109,29 @@ func HasNodeWithOS(ctx context.Context, os string, nodeClient corev1client.CoreV return errors.New("invalid node OS") } - nodes, err := nodeClient.Nodes().List(ctx, metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", NodeOSLabel, os)}) + nodes, err := nodeClient.Nodes().List(ctx, metav1.ListOptions{}) if err != nil { return errors.Wrapf(err, "error listing nodes with OS %s", os) } - if len(nodes.Items) == 0 { - return errors.Errorf("node with OS %s doesn't exist", os) + for _, node := range nodes.Items { + osLabel, found := node.Labels[NodeOSLabel] + if !found { + continue + } + + if getRealOS(osLabel) == os { + return nil + } } - return nil + return errors.Errorf("node with OS %s doesn't exist", os) +} + +func getRealOS(osLabel string) string { + if os, found := realNodeOSMap[osLabel]; !found { + return NodeOSLinux + } else { + return os + } } diff --git a/pkg/util/kube/pod.go b/pkg/util/kube/pod.go index 2aeb45a1c..4ff05b43e 100644 --- a/pkg/util/kube/pod.go +++ b/pkg/util/kube/pod.go @@ -40,10 +40,12 @@ type LoadAffinity struct { } type PodResources struct { - CPURequest string `json:"cpuRequest,omitempty"` - MemoryRequest string `json:"memoryRequest,omitempty"` - CPULimit string `json:"cpuLimit,omitempty"` - MemoryLimit string `json:"memoryLimit,omitempty"` + CPURequest string `json:"cpuRequest,omitempty"` + CPULimit string `json:"cpuLimit,omitempty"` + MemoryRequest string `json:"memoryRequest,omitempty"` + MemoryLimit string `json:"memoryLimit,omitempty"` + EphemeralStorageRequest string `json:"ephemeralStorageRequest,omitempty"` + EphemeralStorageLimit string `json:"ephemeralStorageLimit,omitempty"` } // IsPodRunning does a well-rounded check to make sure the specified pod is running stably. @@ -230,14 +232,9 @@ func CollectPodLogs(ctx context.Context, podGetter corev1client.CoreV1Interface, return nil } -func ToSystemAffinity(loadAffinities []*LoadAffinity) *corev1api.Affinity { - if len(loadAffinities) == 0 { - return nil - } - nodeSelectorTermList := make([]corev1api.NodeSelectorTerm, 0) - - for _, loadAffinity := range loadAffinities { - requirements := []corev1api.NodeSelectorRequirement{} +func ToSystemAffinity(loadAffinity *LoadAffinity, volumeTopology *corev1api.NodeSelector) *corev1api.Affinity { + requirements := []corev1api.NodeSelectorRequirement{} + if loadAffinity != nil { for k, v := range loadAffinity.NodeSelector.MatchLabels { requirements = append(requirements, corev1api.NodeSelectorRequirement{ Key: k, @@ -253,25 +250,25 @@ func ToSystemAffinity(loadAffinities []*LoadAffinity) *corev1api.Affinity { Operator: corev1api.NodeSelectorOperator(exp.Operator), }) } - - nodeSelectorTermList = append( - nodeSelectorTermList, - corev1api.NodeSelectorTerm{ - MatchExpressions: requirements, - }, - ) } - if len(nodeSelectorTermList) > 0 { - result := new(corev1api.Affinity) - result.NodeAffinity = new(corev1api.NodeAffinity) - result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = new(corev1api.NodeSelector) - result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = nodeSelectorTermList + result := new(corev1api.Affinity) + result.NodeAffinity = new(corev1api.NodeAffinity) + result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = new(corev1api.NodeSelector) - return result + if volumeTopology != nil { + result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, volumeTopology.NodeSelectorTerms...) + } else if len(requirements) > 0 { + result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = make([]corev1api.NodeSelectorTerm, 1) + } else { + return nil } - return nil + for i := range result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { + result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[i].MatchExpressions = append(result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[i].MatchExpressions, requirements...) + } + + return result } func DiagnosePod(pod *corev1api.Pod, events *corev1api.EventList) string { diff --git a/pkg/util/kube/pod_test.go b/pkg/util/kube/pod_test.go index 6751e8b6e..aa8d4db99 100644 --- a/pkg/util/kube/pod_test.go +++ b/pkg/util/kube/pod_test.go @@ -747,24 +747,23 @@ func TestCollectPodLogs(t *testing.T) { func TestToSystemAffinity(t *testing.T) { tests := []struct { name string - loadAffinities []*LoadAffinity + loadAffinity *LoadAffinity + volumeTopology *corev1api.NodeSelector expected *corev1api.Affinity }{ { name: "loadAffinity is nil", }, { - name: "loadAffinity is empty", - loadAffinities: []*LoadAffinity{}, + name: "loadAffinity is empty", + loadAffinity: &LoadAffinity{}, }, { name: "with match label", - loadAffinities: []*LoadAffinity{ - { - NodeSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "key-1": "value-1", - }, + loadAffinity: &LoadAffinity{ + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "key-1": "value-1", }, }, }, @@ -788,23 +787,21 @@ func TestToSystemAffinity(t *testing.T) { }, { name: "with match expression", - loadAffinities: []*LoadAffinity{ - { - NodeSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "key-2": "value-2", + loadAffinity: &LoadAffinity{ + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "key-2": "value-2", + }, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "key-3", + Values: []string{"value-3-1", "value-3-2"}, + Operator: metav1.LabelSelectorOpNotIn, }, - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "key-3", - Values: []string{"value-3-1", "value-3-2"}, - Operator: metav1.LabelSelectorOpNotIn, - }, - { - Key: "key-4", - Values: []string{"value-4-1", "value-4-2", "value-4-3"}, - Operator: metav1.LabelSelectorOpDoesNotExist, - }, + { + Key: "key-4", + Values: []string{"value-4-1", "value-4-2", "value-4-3"}, + Operator: metav1.LabelSelectorOpDoesNotExist, }, }, }, @@ -838,19 +835,49 @@ func TestToSystemAffinity(t *testing.T) { }, }, { - name: "multiple load affinities", - loadAffinities: []*LoadAffinity{ - { - NodeSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "key-1": "value-1", + name: "with olume topology", + volumeTopology: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "key-5", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-6", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, }, }, - }, - { - NodeSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "key-2": "value-2", + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "key-7", + Values: []string{"value-7-1", "value-7-2", "value-7-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-8", + Values: []string{"value-8-1", "value-8-2", "value-8-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + }, + }, + { + MatchFields: []corev1api.NodeSelectorRequirement{ + { + Key: "key-9", + Values: []string{"value-9-1", "value-9-2", "value-9-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-a", + Values: []string{"value-a-1", "value-a-2", "value-a-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, }, }, }, @@ -862,10 +889,177 @@ func TestToSystemAffinity(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "key-1", - Values: []string{"value-1"}, + Key: "key-5", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-6", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + }, + }, + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "key-7", + Values: []string{"value-7-1", "value-7-2", "value-7-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-8", + Values: []string{"value-8-1", "value-8-2", "value-8-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + }, + }, + { + MatchFields: []corev1api.NodeSelectorRequirement{ + { + Key: "key-9", + Values: []string{"value-9-1", "value-9-2", "value-9-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-a", + Values: []string{"value-a-1", "value-a-2", "value-a-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "with match expression and volume topology", + loadAffinity: &LoadAffinity{ + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "key-2": "value-2", + }, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "key-3", + Values: []string{"value-3-1", "value-3-2"}, + Operator: metav1.LabelSelectorOpNotIn, + }, + { + Key: "key-4", + Values: []string{"value-4-1", "value-4-2", "value-4-3"}, + Operator: metav1.LabelSelectorOpDoesNotExist, + }, + }, + }, + }, + volumeTopology: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "key-5", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-6", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + }, + }, + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "key-7", + Values: []string{"value-7-1", "value-7-2", "value-7-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-8", + Values: []string{"value-8-1", "value-8-2", "value-8-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + }, + }, + { + MatchFields: []corev1api.NodeSelectorRequirement{ + { + Key: "key-9", + Values: []string{"value-9-1", "value-9-2", "value-9-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-a", + Values: []string{"value-a-1", "value-a-2", "value-a-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + }, + }, + }, + }, + expected: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "key-5", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-6", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-2", + Values: []string{"value-2"}, Operator: corev1api.NodeSelectorOpIn, }, + { + Key: "key-3", + Values: []string{"value-3-1", "value-3-2"}, + Operator: corev1api.NodeSelectorOpNotIn, + }, + { + Key: "key-4", + Values: []string{"value-4-1", "value-4-2", "value-4-3"}, + Operator: corev1api.NodeSelectorOpDoesNotExist, + }, + }, + }, + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "key-7", + Values: []string{"value-7-1", "value-7-2", "value-7-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-8", + Values: []string{"value-8-1", "value-8-2", "value-8-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-2", + Values: []string{"value-2"}, + Operator: corev1api.NodeSelectorOpIn, + }, + { + Key: "key-3", + Values: []string{"value-3-1", "value-3-2"}, + Operator: corev1api.NodeSelectorOpNotIn, + }, + { + Key: "key-4", + Values: []string{"value-4-1", "value-4-2", "value-4-3"}, + Operator: corev1api.NodeSelectorOpDoesNotExist, + }, }, }, { @@ -875,6 +1069,28 @@ func TestToSystemAffinity(t *testing.T) { Values: []string{"value-2"}, Operator: corev1api.NodeSelectorOpIn, }, + { + Key: "key-3", + Values: []string{"value-3-1", "value-3-2"}, + Operator: corev1api.NodeSelectorOpNotIn, + }, + { + Key: "key-4", + Values: []string{"value-4-1", "value-4-2", "value-4-3"}, + Operator: corev1api.NodeSelectorOpDoesNotExist, + }, + }, + MatchFields: []corev1api.NodeSelectorRequirement{ + { + Key: "key-9", + Values: []string{"value-9-1", "value-9-2", "value-9-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-a", + Values: []string{"value-a-1", "value-a-2", "value-a-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, }, }, }, @@ -886,7 +1102,7 @@ func TestToSystemAffinity(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - affinity := ToSystemAffinity(test.loadAffinities) + affinity := ToSystemAffinity(test.loadAffinity, test.volumeTopology) assert.True(t, reflect.DeepEqual(affinity, test.expected)) }) } diff --git a/pkg/util/kube/pvc_pv.go b/pkg/util/kube/pvc_pv.go index 786cef2a5..d5d2e2041 100644 --- a/pkg/util/kube/pvc_pv.go +++ b/pkg/util/kube/pvc_pv.go @@ -417,19 +417,19 @@ func MakePodPVCAttachment(volumeName string, volumeMode *corev1api.PersistentVol return volumeMounts, volumeDevices, volumePath } +// GetPVForPVC returns the PersistentVolume backing a PVC +// returns PV, error. +// PV will be nil on error func GetPVForPVC( pvc *corev1api.PersistentVolumeClaim, crClient crclient.Client, ) (*corev1api.PersistentVolume, error) { if pvc.Spec.VolumeName == "" { - return nil, errors.Errorf("PVC %s/%s has no volume backing this claim", - pvc.Namespace, pvc.Name) + return nil, errors.Errorf("PVC %s/%s has no volume backing this claim", pvc.Namespace, pvc.Name) } if pvc.Status.Phase != corev1api.ClaimBound { - // TODO: confirm if this PVC should be snapshotted if it has no PV bound - return nil, - errors.Errorf("PVC %s/%s is in phase %v and is not bound to a volume", - pvc.Namespace, pvc.Name, pvc.Status.Phase) + return nil, errors.Errorf("PVC %s/%s is in phase %v and is not bound to a volume", + pvc.Namespace, pvc.Name, pvc.Status.Phase) } pv := &corev1api.PersistentVolume{} @@ -580,3 +580,29 @@ func GetPVAttachedNodes(ctx context.Context, pv string, storageClient storagev1. return nodes, nil } + +func GetVolumeTopology(ctx context.Context, volumeClient corev1client.CoreV1Interface, storageClient storagev1.StorageV1Interface, pvName string, scName string) (*corev1api.NodeSelector, error) { + if pvName == "" || scName == "" { + return nil, errors.Errorf("invalid parameter, pv %s, sc %s", pvName, scName) + } + + sc, err := storageClient.StorageClasses().Get(ctx, scName, metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrapf(err, "error getting storage class %s", scName) + } + + if sc.VolumeBindingMode == nil || *sc.VolumeBindingMode != storagev1api.VolumeBindingWaitForFirstConsumer { + return nil, nil + } + + pv, err := volumeClient.PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrapf(err, "error getting PV %s", pvName) + } + + if pv.Spec.NodeAffinity == nil { + return nil, nil + } + + return pv.Spec.NodeAffinity.Required, nil +} diff --git a/pkg/util/kube/pvc_pv_test.go b/pkg/util/kube/pvc_pv_test.go index d94efa62e..63b8e1edd 100644 --- a/pkg/util/kube/pvc_pv_test.go +++ b/pkg/util/kube/pvc_pv_test.go @@ -1909,3 +1909,143 @@ func TestGetPVCAttachingNodeOS(t *testing.T) { }) } } + +func TestGetVolumeTopology(t *testing.T) { + pvWithoutNodeAffinity := &corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-pv", + }, + } + + pvWithNodeAffinity := &corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-pv", + }, + Spec: corev1api.PersistentVolumeSpec{ + NodeAffinity: &corev1api.VolumeNodeAffinity{ + Required: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "fake-key", + }, + }, + }, + }, + }, + }, + }, + } + + scObjWithoutVolumeBind := &storagev1api.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-storage-class", + }, + } + + volumeBindImmediate := storagev1api.VolumeBindingImmediate + scObjWithImeediateBind := &storagev1api.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-storage-class", + }, + VolumeBindingMode: &volumeBindImmediate, + } + + volumeBindWffc := storagev1api.VolumeBindingWaitForFirstConsumer + scObjWithWffcBind := &storagev1api.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-storage-class", + }, + VolumeBindingMode: &volumeBindWffc, + } + + tests := []struct { + name string + pvName string + scName string + kubeClientObj []runtime.Object + expectedErr string + expected *corev1api.NodeSelector + }{ + { + name: "invalid pvName", + scName: "fake-storage-class", + expectedErr: "invalid parameter, pv , sc fake-storage-class", + }, + { + name: "invalid scName", + pvName: "fake-pv", + expectedErr: "invalid parameter, pv fake-pv, sc ", + }, + { + name: "no sc", + pvName: "fake-pv", + scName: "fake-storage-class", + expectedErr: "error getting storage class fake-storage-class: storageclasses.storage.k8s.io \"fake-storage-class\" not found", + }, + { + name: "sc without binding mode", + pvName: "fake-pv", + scName: "fake-storage-class", + kubeClientObj: []runtime.Object{scObjWithoutVolumeBind}, + }, + { + name: "sc without immediate binding mode", + pvName: "fake-pv", + scName: "fake-storage-class", + kubeClientObj: []runtime.Object{scObjWithImeediateBind}, + }, + { + name: "get pv fail", + pvName: "fake-pv", + scName: "fake-storage-class", + kubeClientObj: []runtime.Object{scObjWithWffcBind}, + expectedErr: "error getting PV fake-pv: persistentvolumes \"fake-pv\" not found", + }, + { + name: "pv with no affinity", + pvName: "fake-pv", + scName: "fake-storage-class", + kubeClientObj: []runtime.Object{ + scObjWithWffcBind, + pvWithoutNodeAffinity, + }, + }, + { + name: "pv with affinity", + pvName: "fake-pv", + scName: "fake-storage-class", + kubeClientObj: []runtime.Object{ + scObjWithWffcBind, + pvWithNodeAffinity, + }, + expected: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "fake-key", + }, + }, + }, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) + + var kubeClient kubernetes.Interface = fakeKubeClient + + affinity, err := GetVolumeTopology(t.Context(), kubeClient.CoreV1(), kubeClient.StorageV1(), test.pvName, test.scName) + + if test.expectedErr != "" { + assert.EqualError(t, err, test.expectedErr) + } else { + assert.Equal(t, test.expected, affinity) + } + }) + } +} diff --git a/pkg/util/kube/resource_requirements.go b/pkg/util/kube/resource_requirements.go index 782847008..12cf7a79c 100644 --- a/pkg/util/kube/resource_requirements.go +++ b/pkg/util/kube/resource_requirements.go @@ -20,12 +20,34 @@ import ( "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + + "github.com/vmware-tanzu/velero/pkg/constant" ) -// ParseResourceRequirements takes a set of CPU and memory requests and limit string +// ParseCPUAndMemoryResources is a helper function that parses CPU and memory requests and limits, +// using default values for ephemeral storage. +func ParseCPUAndMemoryResources(cpuRequest, memRequest, cpuLimit, memLimit string) (corev1api.ResourceRequirements, error) { + return ParseResourceRequirements( + cpuRequest, + memRequest, + constant.DefaultEphemeralStorageRequest, + cpuLimit, + memLimit, + constant.DefaultEphemeralStorageLimit, + ) +} + +// ParseResourceRequirements takes a set of CPU, memory, ephemeral storage requests and limit string // values and returns a ResourceRequirements struct to be used in a Container. // An error is returned if we cannot parse the request/limit. -func ParseResourceRequirements(cpuRequest, memRequest, cpuLimit, memLimit string) (corev1api.ResourceRequirements, error) { +func ParseResourceRequirements( + cpuRequest, + memRequest, + ephemeralStorageRequest, + cpuLimit, + memLimit, + ephemeralStorageLimit string, +) (corev1api.ResourceRequirements, error) { resources := corev1api.ResourceRequirements{ Requests: corev1api.ResourceList{}, Limits: corev1api.ResourceList{}, @@ -41,6 +63,11 @@ func ParseResourceRequirements(cpuRequest, memRequest, cpuLimit, memLimit string return resources, errors.Wrapf(err, `couldn't parse memory request "%s"`, memRequest) } + parsedEphemeralStorageRequest, err := resource.ParseQuantity(ephemeralStorageRequest) + if err != nil { + return resources, errors.Wrapf(err, `couldn't parse ephemeral storage request "%s"`, ephemeralStorageRequest) + } + parsedCPULimit, err := resource.ParseQuantity(cpuLimit) if err != nil { return resources, errors.Wrapf(err, `couldn't parse CPU limit "%s"`, cpuLimit) @@ -51,6 +78,11 @@ func ParseResourceRequirements(cpuRequest, memRequest, cpuLimit, memLimit string return resources, errors.Wrapf(err, `couldn't parse memory limit "%s"`, memLimit) } + parsedEphemeralStorageLimit, err := resource.ParseQuantity(ephemeralStorageLimit) + if err != nil { + return resources, errors.Wrapf(err, `couldn't parse ephemeral storage limit "%s"`, ephemeralStorageLimit) + } + // A quantity of 0 is treated as unbounded unbounded := resource.MustParse("0") @@ -62,6 +94,10 @@ func ParseResourceRequirements(cpuRequest, memRequest, cpuLimit, memLimit string return resources, errors.WithStack(errors.Errorf(`Memory request "%s" must be less than or equal to Memory limit "%s"`, memRequest, memLimit)) } + if parsedEphemeralStorageLimit != unbounded && parsedEphemeralStorageRequest.Cmp(parsedEphemeralStorageLimit) > 0 { + return resources, errors.WithStack(errors.Errorf(`Ephemeral storage request "%s" must be less than or equal to Ephemeral storage limit "%s"`, ephemeralStorageRequest, ephemeralStorageLimit)) + } + // Only set resources if they are not unbounded if parsedCPURequest != unbounded { resources.Requests[corev1api.ResourceCPU] = parsedCPURequest @@ -69,12 +105,18 @@ func ParseResourceRequirements(cpuRequest, memRequest, cpuLimit, memLimit string if parsedMemRequest != unbounded { resources.Requests[corev1api.ResourceMemory] = parsedMemRequest } + if parsedEphemeralStorageRequest != unbounded { + resources.Requests[corev1api.ResourceEphemeralStorage] = parsedEphemeralStorageRequest + } if parsedCPULimit != unbounded { resources.Limits[corev1api.ResourceCPU] = parsedCPULimit } if parsedMemLimit != unbounded { resources.Limits[corev1api.ResourceMemory] = parsedMemLimit } + if parsedEphemeralStorageLimit != unbounded { + resources.Limits[corev1api.ResourceEphemeralStorage] = parsedEphemeralStorageLimit + } return resources, nil } diff --git a/pkg/util/kube/resource_requirements_test.go b/pkg/util/kube/resource_requirements_test.go index 2b2c96c4a..d015138e7 100644 --- a/pkg/util/kube/resource_requirements_test.go +++ b/pkg/util/kube/resource_requirements_test.go @@ -27,10 +27,12 @@ import ( func TestParseResourceRequirements(t *testing.T) { type args struct { - cpuRequest string - memRequest string - cpuLimit string - memLimit string + cpuRequest string + memRequest string + ephemeralStorageRequest string + cpuLimit string + memLimit string + ephemeralStorageLimit string } tests := []struct { name string @@ -38,43 +40,61 @@ func TestParseResourceRequirements(t *testing.T) { wantErr bool expected *corev1api.ResourceRequirements }{ - {"unbounded quantities", args{"0", "0", "0", "0"}, false, &corev1api.ResourceRequirements{ + {"unbounded quantities", args{"0", "0", "0", "0", "0", "0"}, false, &corev1api.ResourceRequirements{ Requests: corev1api.ResourceList{}, Limits: corev1api.ResourceList{}, }}, - {"valid quantities", args{"100m", "128Mi", "200m", "256Mi"}, false, nil}, - {"CPU request with unbounded limit", args{"100m", "128Mi", "0", "256Mi"}, false, &corev1api.ResourceRequirements{ + {"valid quantities", args{"100m", "128Mi", "5Gi", "200m", "256Mi", "10Gi"}, false, nil}, + {"CPU request with unbounded limit", args{"100m", "128Mi", "5Gi", "0", "256Mi", "10Gi"}, false, &corev1api.ResourceRequirements{ Requests: corev1api.ResourceList{ - corev1api.ResourceCPU: resource.MustParse("100m"), - corev1api.ResourceMemory: resource.MustParse("128Mi"), + corev1api.ResourceCPU: resource.MustParse("100m"), + corev1api.ResourceMemory: resource.MustParse("128Mi"), + corev1api.ResourceEphemeralStorage: resource.MustParse("5Gi"), }, Limits: corev1api.ResourceList{ + corev1api.ResourceMemory: resource.MustParse("256Mi"), + corev1api.ResourceEphemeralStorage: resource.MustParse("10Gi"), + }, + }}, + {"Mem request with unbounded limit", args{"100m", "128Mi", "5Gi", "200m", "0", "10Gi"}, false, &corev1api.ResourceRequirements{ + Requests: corev1api.ResourceList{ + corev1api.ResourceCPU: resource.MustParse("100m"), + corev1api.ResourceMemory: resource.MustParse("128Mi"), + corev1api.ResourceEphemeralStorage: resource.MustParse("5Gi"), + }, + Limits: corev1api.ResourceList{ + corev1api.ResourceCPU: resource.MustParse("200m"), + corev1api.ResourceEphemeralStorage: resource.MustParse("10Gi"), + }, + }}, + {"Ephemeral storage request with unbounded limit", args{"100m", "128Mi", "5Gi", "200m", "256Mi", "0"}, false, &corev1api.ResourceRequirements{ + Requests: corev1api.ResourceList{ + corev1api.ResourceCPU: resource.MustParse("100m"), + corev1api.ResourceMemory: resource.MustParse("128Mi"), + corev1api.ResourceEphemeralStorage: resource.MustParse("5Gi"), + }, + Limits: corev1api.ResourceList{ + corev1api.ResourceCPU: resource.MustParse("200m"), corev1api.ResourceMemory: resource.MustParse("256Mi"), }, }}, - {"Mem request with unbounded limit", args{"100m", "128Mi", "200m", "0"}, false, &corev1api.ResourceRequirements{ + + {"CPU/Mem/EphemeralStorage requests with unbounded limits", args{"100m", "128Mi", "5Gi", "0", "0", "0"}, false, &corev1api.ResourceRequirements{ Requests: corev1api.ResourceList{ - corev1api.ResourceCPU: resource.MustParse("100m"), - corev1api.ResourceMemory: resource.MustParse("128Mi"), - }, - Limits: corev1api.ResourceList{ - corev1api.ResourceCPU: resource.MustParse("200m"), - }, - }}, - {"CPU/Mem requests with unbounded limits", args{"100m", "128Mi", "0", "0"}, false, &corev1api.ResourceRequirements{ - Requests: corev1api.ResourceList{ - corev1api.ResourceCPU: resource.MustParse("100m"), - corev1api.ResourceMemory: resource.MustParse("128Mi"), + corev1api.ResourceCPU: resource.MustParse("100m"), + corev1api.ResourceMemory: resource.MustParse("128Mi"), + corev1api.ResourceEphemeralStorage: resource.MustParse("5Gi"), }, Limits: corev1api.ResourceList{}, }}, - {"invalid quantity", args{"100m", "invalid", "200m", "256Mi"}, true, nil}, - {"CPU request greater than limit", args{"300m", "128Mi", "200m", "256Mi"}, true, nil}, - {"memory request greater than limit", args{"100m", "512Mi", "200m", "256Mi"}, true, nil}, + {"invalid quantity", args{"100m", "invalid", "1Gi", "200m", "256Mi", "valid"}, true, nil}, + {"CPU request greater than limit", args{"300m", "128Mi", "5Gi", "200m", "256Mi", "10Gi"}, true, nil}, + {"memory request greater than limit", args{"100m", "512Mi", "5Gi", "200m", "256Mi", "10Gi"}, true, nil}, + {"ephemeral storage request greater than limit", args{"100m", "128Mi", "10Gi", "200m", "256Mi", "5Gi"}, true, nil}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := ParseResourceRequirements(tt.args.cpuRequest, tt.args.memRequest, tt.args.cpuLimit, tt.args.memLimit) + got, err := ParseResourceRequirements(tt.args.cpuRequest, tt.args.memRequest, tt.args.ephemeralStorageRequest, tt.args.cpuLimit, tt.args.memLimit, tt.args.ephemeralStorageLimit) if tt.wantErr { assert.Error(t, err) return @@ -85,12 +105,14 @@ func TestParseResourceRequirements(t *testing.T) { if tt.expected == nil { expected = corev1api.ResourceRequirements{ Requests: corev1api.ResourceList{ - corev1api.ResourceCPU: resource.MustParse(tt.args.cpuRequest), - corev1api.ResourceMemory: resource.MustParse(tt.args.memRequest), + corev1api.ResourceCPU: resource.MustParse(tt.args.cpuRequest), + corev1api.ResourceMemory: resource.MustParse(tt.args.memRequest), + corev1api.ResourceEphemeralStorage: resource.MustParse(tt.args.ephemeralStorageRequest), }, Limits: corev1api.ResourceList{ - corev1api.ResourceCPU: resource.MustParse(tt.args.cpuLimit), - corev1api.ResourceMemory: resource.MustParse(tt.args.memLimit), + corev1api.ResourceCPU: resource.MustParse(tt.args.cpuLimit), + corev1api.ResourceMemory: resource.MustParse(tt.args.memLimit), + corev1api.ResourceEphemeralStorage: resource.MustParse(tt.args.ephemeralStorageLimit), }, } } else { diff --git a/pkg/util/podvolume/pod_volume_test.go b/pkg/util/podvolume/pod_volume_test.go index a3484c2e3..87a6f3a0f 100644 --- a/pkg/util/podvolume/pod_volume_test.go +++ b/pkg/util/podvolume/pod_volume_test.go @@ -156,7 +156,7 @@ func TestGetVolumesByPod(t *testing.T) { Volumes: []corev1api.Volume{ // PVB Volumes {Name: "pvbPV1"}, {Name: "pvbPV2"}, {Name: "pvbPV3"}, - /// Excluded from PVB because colume mounting default service account token + /// Excluded from PVB because volume mounting default service account token {Name: "default-token-5xq45"}, }, }, diff --git a/pkg/util/volumehelper/volume_policy_helper.go b/pkg/util/volumehelper/volume_policy_helper.go new file mode 100644 index 000000000..95f104994 --- /dev/null +++ b/pkg/util/volumehelper/volume_policy_helper.go @@ -0,0 +1,30 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volumehelper + +import ( + corev1api "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +type VolumeHelper interface { + ShouldPerformSnapshot(obj runtime.Unstructured, groupResource schema.GroupResource) (bool, error) + ShouldPerformFSBackup(volume corev1api.Volume, pod corev1api.Pod) (bool, error) + ShouldPerformCustomAction(obj runtime.Unstructured, groupResource schema.GroupResource, matchParams map[string]any) (bool, error) + GetActionParameters(obj runtime.Unstructured, groupResource schema.GroupResource) (bool, string, map[string]any, error) +} diff --git a/pkg/util/wildcard/expand.go b/pkg/util/wildcard/expand.go index 632e05aa1..8767c8ed3 100644 --- a/pkg/util/wildcard/expand.go +++ b/pkg/util/wildcard/expand.go @@ -31,70 +31,77 @@ func ShouldExpandWildcards(includes []string, excludes []string) bool { } // containsWildcardPattern checks if a pattern contains any wildcard symbols -// Supported patterns: *, ?, [abc], {a,b,c} +// Supported patterns: *, ?, [abc] // Note: . and + are treated as literal characters (not wildcards) // Note: ** and consecutive asterisks are NOT supported (will cause validation error) func containsWildcardPattern(pattern string) bool { - return strings.ContainsAny(pattern, "*?[{") + return strings.ContainsAny(pattern, "*?[") } func validateWildcardPatterns(patterns []string) error { for _, pattern := range patterns { - // Check for invalid regex-only patterns that we don't support - if strings.ContainsAny(pattern, "|()") { - return errors.New("wildcard pattern contains unsupported regex symbols: |, (, )") - } - - // Check for consecutive asterisks (2 or more) - if strings.Contains(pattern, "**") { - return errors.New("wildcard pattern contains consecutive asterisks (only single * allowed)") - } - - // Check for malformed brace patterns - if err := validateBracePatterns(pattern); err != nil { + if err := ValidateNamespaceName(pattern); err != nil { return err } } return nil } +func ValidateNamespaceName(pattern string) error { + // Check for invalid characters that are not supported in glob patterns + if strings.ContainsAny(pattern, "|()!{},") { + return errors.New("wildcard pattern contains unsupported characters: |, (, ), !, {, }, ,") + } + + // Check for consecutive asterisks (2 or more) + if strings.Contains(pattern, "**") { + return errors.New("wildcard pattern contains consecutive asterisks (only single * allowed)") + } + + // Check for malformed brace patterns + if err := validateBracePatterns(pattern); err != nil { + return err + } + + return nil +} + // validateBracePatterns checks for malformed brace patterns like unclosed braces or empty braces +// Also validates bracket patterns [] for character classes func validateBracePatterns(pattern string) error { - depth := 0 + bracketDepth := 0 for i := 0; i < len(pattern); i++ { - if pattern[i] == '{' { - braceStart := i - depth++ + if pattern[i] == '[' { + bracketStart := i + bracketDepth++ - // Scan ahead to find the matching closing brace and validate content - for j := i + 1; j < len(pattern) && depth > 0; j++ { - if pattern[j] == '{' { - depth++ - } else if pattern[j] == '}' { - depth-- - if depth == 0 { - // Found matching closing brace - validate content - content := pattern[braceStart+1 : j] - if strings.Trim(content, ", \t") == "" { - return errors.New("wildcard pattern contains empty brace pattern '{}'") + // Scan ahead to find the matching closing bracket and validate content + for j := i + 1; j < len(pattern) && bracketDepth > 0; j++ { + if pattern[j] == ']' { + bracketDepth-- + if bracketDepth == 0 { + // Found matching closing bracket - validate content + content := pattern[bracketStart+1 : j] + if content == "" { + return errors.New("wildcard pattern contains empty bracket pattern '[]'") } - // Skip to the closing brace + // Skip to the closing bracket i = j break } } } - // If we exited the loop without finding a match (depth > 0), brace is unclosed - if depth > 0 { - return errors.New("wildcard pattern contains unclosed brace '{'") + // If we exited the loop without finding a match (bracketDepth > 0), bracket is unclosed + if bracketDepth > 0 { + return errors.New("wildcard pattern contains unclosed bracket '['") } - // i is now positioned at the closing brace; the outer loop will increment it - } else if pattern[i] == '}' { - // Found a closing brace without a matching opening brace - return errors.New("wildcard pattern contains unmatched closing brace '}'") + // i is now positioned at the closing bracket; the outer loop will increment it + } else if pattern[i] == ']' { + // Found a closing bracket without a matching opening bracket + return errors.New("wildcard pattern contains unmatched closing bracket ']'") } } diff --git a/pkg/util/wildcard/expand_test.go b/pkg/util/wildcard/expand_test.go index f6c7ed434..317020648 100644 --- a/pkg/util/wildcard/expand_test.go +++ b/pkg/util/wildcard/expand_test.go @@ -90,7 +90,7 @@ func TestShouldExpandWildcards(t *testing.T) { name: "brace alternatives wildcard", includes: []string{"ns{prod,staging}"}, excludes: []string{}, - expected: true, // brace alternatives are considered wildcard + expected: false, // brace alternatives are not supported }, { name: "dot is literal - not wildcard", @@ -237,9 +237,9 @@ func TestExpandWildcards(t *testing.T) { activeNamespaces: []string{"app-prod", "app-staging", "app-dev", "db-prod"}, includes: []string{"app-{prod,staging}"}, excludes: []string{}, - expectedIncludes: []string{"app-prod", "app-staging"}, // {prod,staging} matches either + expectedIncludes: nil, expectedExcludes: nil, - expectError: false, + expectError: true, }, { name: "literal dot and plus patterns", @@ -259,33 +259,6 @@ func TestExpandWildcards(t *testing.T) { expectedExcludes: nil, expectError: true, // |, (, ) are not supported }, - { - name: "unclosed brace patterns should error", - activeNamespaces: []string{"app-prod"}, - includes: []string{"app-{prod,staging"}, - excludes: []string{}, - expectedIncludes: nil, - expectedExcludes: nil, - expectError: true, // unclosed brace - }, - { - name: "empty brace patterns should error", - activeNamespaces: []string{"app-prod"}, - includes: []string{"app-{}"}, - excludes: []string{}, - expectedIncludes: nil, - expectedExcludes: nil, - expectError: true, // empty braces - }, - { - name: "unmatched closing brace should error", - activeNamespaces: []string{"app-prod"}, - includes: []string{"app-prod}"}, - excludes: []string{}, - expectedIncludes: nil, - expectedExcludes: nil, - expectError: true, // unmatched closing brace - }, } for _, tt := range tests { @@ -354,13 +327,6 @@ func TestExpandWildcardsPrivate(t *testing.T) { expected: []string{}, // returns empty slice, not nil expectError: false, }, - { - name: "brace patterns work correctly", - patterns: []string{"app-{prod,staging}"}, - activeNamespaces: []string{"app-prod", "app-staging", "app-dev", "app-{prod,staging}"}, - expected: []string{"app-prod", "app-staging"}, // brace patterns do expand - expectError: false, - }, { name: "duplicate matches from multiple patterns", patterns: []string{"app-*", "*-prod"}, @@ -389,20 +355,6 @@ func TestExpandWildcardsPrivate(t *testing.T) { expected: []string{"nsa", "nsb", "nsc"}, // [a-c] matches a to c expectError: false, }, - { - name: "negated character class", - patterns: []string{"ns[!abc]"}, - activeNamespaces: []string{"nsa", "nsb", "nsc", "nsd", "ns1"}, - expected: []string{"nsd", "ns1"}, // [!abc] matches anything except a, b, c - expectError: false, - }, - { - name: "brace alternatives", - patterns: []string{"app-{prod,test}"}, - activeNamespaces: []string{"app-prod", "app-test", "app-staging", "db-prod"}, - expected: []string{"app-prod", "app-test"}, // {prod,test} matches either - expectError: false, - }, { name: "double asterisk should error", patterns: []string{"**"}, @@ -410,13 +362,6 @@ func TestExpandWildcardsPrivate(t *testing.T) { expected: nil, expectError: true, // ** is not allowed }, - { - name: "literal dot and plus", - patterns: []string{"app.prod", "service+"}, - activeNamespaces: []string{"app.prod", "appXprod", "service+", "service"}, - expected: []string{"app.prod", "service+"}, // . and + are literal - expectError: false, - }, { name: "unsupported regex symbols should error", patterns: []string{"ns(1|2)"}, @@ -468,153 +413,101 @@ func TestValidateBracePatterns(t *testing.T) { expectError bool errorMsg string }{ - // Valid patterns + // Valid square bracket patterns { - name: "valid single brace pattern", - pattern: "app-{prod,staging}", + name: "valid square bracket pattern", + pattern: "ns[abc]", expectError: false, }, { - name: "valid brace with single option", - pattern: "app-{prod}", + name: "valid square bracket pattern with range", + pattern: "ns[a-z]", expectError: false, }, { - name: "valid brace with three options", - pattern: "app-{prod,staging,dev}", + name: "valid square bracket pattern with numbers", + pattern: "ns[0-9]", expectError: false, }, { - name: "valid pattern with text before and after brace", - pattern: "prefix-{a,b}-suffix", + name: "valid square bracket pattern with mixed", + pattern: "ns[a-z0-9]", expectError: false, }, { - name: "valid pattern with no braces", - pattern: "app-prod", + name: "valid square bracket pattern with single character", + pattern: "ns[a]", expectError: false, }, { - name: "valid pattern with asterisk", - pattern: "app-*", + name: "valid square bracket pattern with text before and after", + pattern: "prefix-[abc]-suffix", expectError: false, }, + // Unclosed opening brackets { - name: "valid brace with spaces around content", - pattern: "app-{ prod , staging }", - expectError: false, + name: "unclosed opening bracket at end", + pattern: "ns[abc", + expectError: true, + errorMsg: "unclosed bracket", }, { - name: "valid brace with numbers", - pattern: "ns-{1,2,3}", - expectError: false, + name: "unclosed opening bracket at start", + pattern: "[abc", + expectError: true, + errorMsg: "unclosed bracket", }, { - name: "valid brace with hyphens in options", - pattern: "{app-prod,db-staging}", - expectError: false, + name: "unclosed opening bracket in middle", + pattern: "ns[abc-test", + expectError: true, + errorMsg: "unclosed bracket", }, - // Unclosed opening braces + // Unmatched closing brackets { - name: "unclosed opening brace at end", - pattern: "app-{prod,staging", + name: "unmatched closing bracket at end", + pattern: "ns-abc]", expectError: true, - errorMsg: "unclosed brace", + errorMsg: "unmatched closing bracket", }, { - name: "unclosed opening brace at start", - pattern: "{prod,staging", + name: "unmatched closing bracket at start", + pattern: "]ns-abc", expectError: true, - errorMsg: "unclosed brace", + errorMsg: "unmatched closing bracket", }, { - name: "unclosed opening brace in middle", - pattern: "app-{prod-test", + name: "unmatched closing bracket in middle", + pattern: "ns-]abc", expectError: true, - errorMsg: "unclosed brace", + errorMsg: "unmatched closing bracket", }, { - name: "multiple unclosed braces", - pattern: "app-{prod-{staging", + name: "extra closing bracket after valid pair", + pattern: "ns[abc]]", expectError: true, - errorMsg: "unclosed brace", + errorMsg: "unmatched closing bracket", }, - // Unmatched closing braces + // Empty bracket patterns { - name: "unmatched closing brace at end", - pattern: "app-prod}", + name: "completely empty brackets", + pattern: "ns[]", expectError: true, - errorMsg: "unmatched closing brace", + errorMsg: "empty bracket pattern", }, { - name: "unmatched closing brace at start", - pattern: "}app-prod", + name: "empty brackets at start", + pattern: "[]ns", expectError: true, - errorMsg: "unmatched closing brace", + errorMsg: "empty bracket pattern", }, { - name: "unmatched closing brace in middle", - pattern: "app-}prod", + name: "empty brackets standalone", + pattern: "[]", expectError: true, - errorMsg: "unmatched closing brace", - }, - { - name: "extra closing brace after valid pair", - pattern: "app-{prod,staging}}", - expectError: true, - errorMsg: "unmatched closing brace", - }, - - // Empty brace patterns - { - name: "completely empty braces", - pattern: "app-{}", - expectError: true, - errorMsg: "empty brace pattern", - }, - { - name: "braces with only spaces", - pattern: "app-{ }", - expectError: true, - errorMsg: "empty brace pattern", - }, - { - name: "braces with only comma", - pattern: "app-{,}", - expectError: true, - errorMsg: "empty brace pattern", - }, - { - name: "braces with only commas", - pattern: "app-{,,,}", - expectError: true, - errorMsg: "empty brace pattern", - }, - { - name: "braces with commas and spaces", - pattern: "app-{ , , }", - expectError: true, - errorMsg: "empty brace pattern", - }, - { - name: "braces with tabs and commas", - pattern: "app-{\t,\t}", - expectError: true, - errorMsg: "empty brace pattern", - }, - { - name: "empty braces at start", - pattern: "{}app-prod", - expectError: true, - errorMsg: "empty brace pattern", - }, - { - name: "empty braces standalone", - pattern: "{}", - expectError: true, - errorMsg: "empty brace pattern", + errorMsg: "empty bracket pattern", }, // Edge cases @@ -623,58 +516,6 @@ func TestValidateBracePatterns(t *testing.T) { pattern: "", expectError: false, }, - { - name: "pattern with only opening brace", - pattern: "{", - expectError: true, - errorMsg: "unclosed brace", - }, - { - name: "pattern with only closing brace", - pattern: "}", - expectError: true, - errorMsg: "unmatched closing brace", - }, - { - name: "valid brace with special characters inside", - pattern: "app-{prod-1,staging_2,dev.3}", - expectError: false, - }, - { - name: "brace with asterisk inside option", - pattern: "app-{prod*,staging}", - expectError: false, - }, - { - name: "multiple valid brace patterns", - pattern: "{app,db}-{prod,staging}", - expectError: false, - }, - { - name: "brace with single character", - pattern: "app-{a}", - expectError: false, - }, - { - name: "brace with trailing comma but has content", - pattern: "app-{prod,staging,}", - expectError: false, // Has content, so it's valid - }, - { - name: "brace with leading comma but has content", - pattern: "app-{,prod,staging}", - expectError: false, // Has content, so it's valid - }, - { - name: "brace with leading comma but has content", - pattern: "app-{{,prod,staging}", - expectError: true, // unclosed brace - }, - { - name: "brace with leading comma but has content", - pattern: "app-{,prod,staging}}", - expectError: true, // unmatched closing brace - }, } for _, tt := range tests { @@ -723,20 +564,6 @@ func TestExpandWildcardsEdgeCases(t *testing.T) { assert.ElementsMatch(t, []string{"ns-1", "ns_2", "ns.3", "ns@4"}, result) }) - t.Run("complex glob combinations", func(t *testing.T) { - activeNamespaces := []string{"app1-prod", "app2-prod", "app1-test", "db-prod", "service"} - result, err := expandWildcards([]string{"app?-{prod,test}"}, activeNamespaces) - require.NoError(t, err) - assert.ElementsMatch(t, []string{"app1-prod", "app2-prod", "app1-test"}, result) - }) - - t.Run("escaped characters", func(t *testing.T) { - activeNamespaces := []string{"app*", "app-prod", "app?test", "app-test"} - result, err := expandWildcards([]string{"app\\*"}, activeNamespaces) - require.NoError(t, err) - assert.ElementsMatch(t, []string{"app*"}, result) - }) - t.Run("mixed literal and wildcard patterns", func(t *testing.T) { activeNamespaces := []string{"app.prod", "app-prod", "app_prod", "test.ns"} result, err := expandWildcards([]string{"app.prod", "app?prod"}, activeNamespaces) @@ -777,12 +604,8 @@ func TestExpandWildcardsEdgeCases(t *testing.T) { shouldError bool }{ {"unclosed bracket", "ns[abc", true}, - {"unclosed brace", "app-{prod,staging", true}, - {"nested unclosed", "ns[a{bc", true}, {"valid bracket", "ns[abc]", false}, - {"valid brace", "app-{prod,staging}", false}, {"empty bracket", "ns[]", true}, // empty brackets are invalid - {"empty brace", "app-{}", true}, // empty braces are invalid } for _, tt := range tests { diff --git a/site/algolia-crawler.json b/site/algolia-crawler.json deleted file mode 100644 index 06dc083d7..000000000 --- a/site/algolia-crawler.json +++ /dev/null @@ -1,90 +0,0 @@ -new Crawler({ - rateLimit: 8, - maxDepth: 10, - startUrls: ["https://velero.io/docs", "https://velero.io/"], - renderJavaScript: false, - sitemaps: ["https://velero.io/sitemap.xml"], - ignoreCanonicalTo: false, - discoveryPatterns: ["https://velero.io/**"], - schedule: "at 6:39 PM on Friday", - actions: [ - { - indexName: "velero_new", - pathsToMatch: ["https://velero.io/docs**/**"], - recordExtractor: ({ helpers }) => { - return helpers.docsearch({ - recordProps: { - lvl1: ["header h1", "article h1", "main h1", "h1", "head > title"], - content: ["article p, article li", "main p, main li", "p, li"], - lvl0: { - defaultValue: "Documentation", - }, - lvl2: ["article h2", "main h2", "h2"], - lvl3: ["article h3", "main h3", "h3"], - lvl4: ["article h4", "main h4", "h4"], - lvl5: ["article h5", "main h5", "h5"], - lvl6: ["article h6", "main h6", "h6"], - version: "#dropdownMenuButton", - }, - aggregateContent: true, - recordVersion: "v3", - }); - }, - }, - ], - initialIndexSettings: { - velero_new: { - attributesForFaceting: ["type", "lang", "version"], - attributesToRetrieve: [ - "hierarchy", - "content", - "anchor", - "url", - "url_without_anchor", - "type", - "version", - ], - attributesToHighlight: ["hierarchy", "content"], - attributesToSnippet: ["content:10"], - camelCaseAttributes: ["hierarchy", "content"], - searchableAttributes: [ - "unordered(hierarchy.lvl0)", - "unordered(hierarchy.lvl1)", - "unordered(hierarchy.lvl2)", - "unordered(hierarchy.lvl3)", - "unordered(hierarchy.lvl4)", - "unordered(hierarchy.lvl5)", - "unordered(hierarchy.lvl6)", - "content", - ], - distinct: true, - attributeForDistinct: "url", - customRanking: [ - "desc(weight.pageRank)", - "desc(weight.level)", - "asc(weight.position)", - ], - ranking: [ - "words", - "filters", - "typo", - "attribute", - "proximity", - "exact", - "custom", - ], - highlightPreTag: '', - highlightPostTag: "", - minWordSizefor1Typo: 3, - minWordSizefor2Typos: 7, - allowTyposOnNumericTokens: false, - minProximity: 1, - ignorePlurals: true, - advancedSyntax: true, - attributeCriteriaComputedByMinProximity: true, - removeWordsIfNoResults: "allOptional", - }, - }, - appId: "9ASKQJ1HR3", - apiKey: "6392a5916af73b73df2406d3aef5ca45", -}); \ No newline at end of file diff --git a/site/config.yaml b/site/config.yaml index ed80914a4..8eded5b59 100644 --- a/site/config.yaml +++ b/site/config.yaml @@ -12,7 +12,7 @@ params: hero: backgroundColor: med-blue versioning: true - latest: v1.17 + latest: v1.18 versions: - main - v1.18 diff --git a/site/content/docs/main/api-types/backup.md b/site/content/docs/main/api-types/backup.md index 1768f0934..3bad516e3 100644 --- a/site/content/docs/main/api-types/backup.md +++ b/site/content/docs/main/api-types/backup.md @@ -16,6 +16,8 @@ Backup belongs to the API group version `velero.io/v1`. Here is a sample `Backup` object with each of the fields documented: +**Note:** Namespace includes/excludes support glob patterns (`*`, `?`, `[abc]`). See [Namespace Glob Patterns](../namespace-glob-patterns) for more details. + ```yaml # Standard Kubernetes API Version declaration. Required. apiVersion: velero.io/v1 @@ -42,11 +44,12 @@ spec: resourcePolicy: kind: configmap name: resource-policy-configmap - # Array of namespaces to include in the backup. If unspecified, all namespaces are included. - # Optional. + # Array of namespaces to include in the backup. Accepts glob patterns (*, ?, [abc]). + # Note: '*' alone is reserved for empty fields, which means all namespaces. + # If unspecified, all namespaces are included. Optional. includedNamespaces: - '*' - # Array of namespaces to exclude from the backup. Optional. + # Array of namespaces to exclude from the backup. Accepts glob patterns (*, ?, [abc]). Optional. excludedNamespaces: - some-namespace # Array of resources to include in the backup. Resources may be shortcuts (for example 'po' for 'pods') diff --git a/site/content/docs/main/api-types/restore.md b/site/content/docs/main/api-types/restore.md index ec3e19511..1c80a0ee8 100644 --- a/site/content/docs/main/api-types/restore.md +++ b/site/content/docs/main/api-types/restore.md @@ -16,6 +16,8 @@ Restore belongs to the API group version `velero.io/v1`. Here is a sample `Restore` object with each of the fields documented: +**Note:** Namespace includes/excludes support glob patterns (`*`, `?`, `[abc]`). See [Namespace Glob Patterns](../namespace-glob-patterns) for more details. + ```yaml # Standard Kubernetes API Version declaration. Required. apiVersion: velero.io/v1 @@ -45,11 +47,11 @@ spec: writeSparseFiles: true # ParallelFilesDownload is the concurrency number setting for restore parallelFilesDownload: 10 - # Array of namespaces to include in the restore. If unspecified, all namespaces are included. - # Optional. + # Array of namespaces to include in the restore. Accepts glob patterns (*, ?, [abc]). + # If unspecified, all namespaces are included. Optional. includedNamespaces: - '*' - # Array of namespaces to exclude from the restore. Optional. + # Array of namespaces to exclude from the restore. Accepts glob patterns (*, ?, [abc]). Optional. excludedNamespaces: - some-namespace # Array of resources to include in the restore. Resources may be shortcuts (for example 'po' for 'pods') diff --git a/site/content/docs/main/api-types/schedule.md b/site/content/docs/main/api-types/schedule.md index c89fe60d7..ef3df4324 100644 --- a/site/content/docs/main/api-types/schedule.md +++ b/site/content/docs/main/api-types/schedule.md @@ -63,6 +63,10 @@ spec: # CSI VolumeSnapshot status turns to ReadyToUse during creation, before # returning error as timeout. The default value is 10 minute. csiSnapshotTimeout: 10m + # ItemOperationTimeout specifies the time used to wait for + # asynchronous BackupItemAction operations + # The default value is 4 hour. + itemOperationTimeout: 4h # resourcePolicy specifies the referenced resource policies that backup should follow # optional resourcePolicy: diff --git a/site/content/docs/main/data-movement-pod-resource-configuration.md b/site/content/docs/main/data-movement-pod-resource-configuration.md index 3440aa931..a90217ea8 100644 --- a/site/content/docs/main/data-movement-pod-resource-configuration.md +++ b/site/content/docs/main/data-movement-pod-resource-configuration.md @@ -7,7 +7,7 @@ During [CSI Snapshot Data Movement][1], Velero built-in data mover launches data During [fs-backup][2], Velero also launches data mover pods to run the data transfer. The data transfer is a time and resource consuming activity. -Velero by default uses the [BestEffort QoS][2] for the data mover pods, which guarantees the best performance of the data movement activities. On the other hand, it may take lots of cluster resource, i.e., CPU, memory, and how many resources are taken is decided by the concurrency and the scale of data to be moved. +Velero by default uses the [BestEffort QoS][2] for the data mover pods, which guarantees the best performance of the data movement activities. On the other hand, it may take lots of cluster resource, i.e., CPU, memory, ephemeral storage, and how many resources are taken is decided by the concurrency and the scale of data to be moved. If the cluster nodes don't have sufficient resource, Velero also allows you to customize the resources for the data mover pods. Note: If less resources are assigned to data mover pods, the data movement activities may take longer time; or the data mover pods may be OOM killed if the assigned memory resource doesn't meet the requirements. Consequently, the dataUpload/dataDownload may run longer or fail. @@ -25,6 +25,8 @@ Here is a sample of the configMap with ```podResources```: "podResources": { "cpuRequest": "1000m", "cpuLimit": "1000m", + "ephemeralStorageRequest": "5Gi", + "ephemeralStorageLimit": "10Gi", "memoryRequest": "512Mi", "memoryLimit": "1Gi" } diff --git a/site/content/docs/main/namespace-glob-patterns.md b/site/content/docs/main/namespace-glob-patterns.md new file mode 100644 index 000000000..4695124ea --- /dev/null +++ b/site/content/docs/main/namespace-glob-patterns.md @@ -0,0 +1,71 @@ +--- +title: "Namespace Glob Patterns" +layout: docs +--- + +When using `--include-namespaces` and `--exclude-namespaces` flags with backup and restore commands, you can use glob patterns to match multiple namespaces. + +## Supported Patterns + +Velero supports the following glob pattern characters: + +- `*` - Matches any sequence of characters + ```bash + velero backup create my-backup --include-namespaces "app-*" + # Matches: app-prod, app-staging, app-dev, etc. + ``` + +- `?` - Matches exactly one character + ```bash + velero backup create my-backup --include-namespaces "ns?" + # Matches: ns1, ns2, nsa, but NOT ns10 + ``` + +- `[abc]` - Matches any single character in the brackets + ```bash + velero backup create my-backup --include-namespaces "ns[123]" + # Matches: ns1, ns2, ns3 + ``` + +- `[a-z]` - Matches any single character in the range + ```bash + velero backup create my-backup --include-namespaces "ns[a-c]" + # Matches: nsa, nsb, nsc + ``` + +## Unsupported Patterns + +The following patterns are **not supported** and will cause validation errors: + +- `**` - Consecutive asterisks +- `|` - Alternation (regex operator) +- `()` - Grouping (regex operators) +- `!` - Negation +- `{}` - Brace expansion +- `,` - Comma (used in brace expansion) + +## Special Cases + +- `*` alone means "all namespaces" and is not expanded +- Empty brackets `[]` are invalid +- Unmatched or unclosed brackets will cause validation errors + +## Examples + +Combine patterns with include and exclude flags: + +```bash +# Backup all production namespaces except test +velero backup create prod-backup \ + --include-namespaces "*-prod" \ + --exclude-namespaces "test-*" + +# Backup specific numbered namespaces +velero backup create numbered-backup \ + --include-namespaces "app-[0-9]" + +# Restore namespaces matching multiple patterns +velero restore create my-restore \ + --from-backup my-backup \ + --include-namespaces "frontend-*,backend-*" +``` diff --git a/site/content/docs/main/repository-maintenance.md b/site/content/docs/main/repository-maintenance.md index 5ca4f3a8a..e6a45920f 100644 --- a/site/content/docs/main/repository-maintenance.md +++ b/site/content/docs/main/repository-maintenance.md @@ -72,6 +72,8 @@ data: "podResources": { "cpuRequest": "100m", "cpuLimit": "200m", + "ephemeralStorageRequest": "5Gi", + "ephemeralStorageLimit": "10Gi", "memoryRequest": "100Mi", "memoryLimit": "200Mi" }, @@ -99,6 +101,8 @@ data: "podResources": { "cpuRequest": "200m", "cpuLimit": "400m", + "ephemeralStorageRequest": "5Gi", + "ephemeralStorageLimit": "10Gi", "memoryRequest": "200Mi", "memoryLimit": "400Mi" }, diff --git a/site/content/docs/main/resource-filtering.md b/site/content/docs/main/resource-filtering.md index a9e65d157..cbfdb2816 100644 --- a/site/content/docs/main/resource-filtering.md +++ b/site/content/docs/main/resource-filtering.md @@ -17,7 +17,11 @@ Wildcard takes precedence when both a wildcard and specific resource are include ### --include-namespaces -Namespaces to include. Default is `*`, all namespaces. +Namespaces to include. Accepts glob patterns (`*`, `?`, `[abc]`). Default is `*`, all namespaces. + +See [Namespace Glob Patterns](namespace-glob-patterns) for more details on supported patterns. + +Note: `*` alone is reserved for empty fields, which means all namespaces. * Backup a namespace and it's objects. @@ -158,7 +162,9 @@ Wildcard excludes are ignored. ### --exclude-namespaces -Namespaces to exclude. +Namespaces to exclude. Accepts glob patterns (`*`, `?`, `[abc]`). + +See [Namespace Glob Patterns](namespace-glob-patterns.md) for more details on supported patterns. * Exclude kube-system from the cluster backup. diff --git a/site/content/docs/main/supported-configmaps/node-agent-configmap.md b/site/content/docs/main/supported-configmaps/node-agent-configmap.md index f17df3815..1953d34d9 100644 --- a/site/content/docs/main/supported-configmaps/node-agent-configmap.md +++ b/site/content/docs/main/supported-configmaps/node-agent-configmap.md @@ -224,7 +224,7 @@ Configure different node selection rules for specific storage classes: ``` ### Pod Resources (`podResources`) -Configure CPU and memory resources for Data Mover Pods to optimize performance and prevent resource conflict. +Configure CPU, memory and ephemeral storage resources for Data Mover Pods to optimize performance and prevent resource conflict. The configurations work for PodVolumeBackup, PodVolumeRestore, DataUpload, and DataDownload pods. @@ -233,6 +233,8 @@ The configurations work for PodVolumeBackup, PodVolumeRestore, DataUpload, and D "podResources": { "cpuRequest": "1000m", "cpuLimit": "2000m", + "ephemeralStorageRequest": "5Gi", + "ephemeralStorageLimit": "10Gi", "memoryRequest": "1Gi", "memoryLimit": "4Gi" } @@ -535,6 +537,8 @@ Here's a comprehensive example showing how all configuration sections work toget "podResources": { "cpuRequest": "500m", "cpuLimit": "1000m", + "ephemeralStorageRequest": "5Gi", + "ephemeralStorageLimit": "10Gi", "memoryRequest": "1Gi", "memoryLimit": "2Gi" }, diff --git a/site/content/docs/main/upgrade-to-1.17.md b/site/content/docs/main/upgrade-to-1.18.md similarity index 70% rename from site/content/docs/main/upgrade-to-1.17.md rename to site/content/docs/main/upgrade-to-1.18.md index f6738d55c..0bf839fe3 100644 --- a/site/content/docs/main/upgrade-to-1.17.md +++ b/site/content/docs/main/upgrade-to-1.18.md @@ -1,13 +1,13 @@ --- -title: "Upgrading to Velero 1.17" +title: "Upgrading to Velero 1.18" layout: docs --- ## Prerequisites -- Velero [v1.16.x][9] installed. +- Velero [v1.17.x][9] installed. -If you're not yet running at least Velero v1.16, see the following: +If you're not yet running at least Velero v1.17, see the following: - [Upgrading to v1.8][1] - [Upgrading to v1.9][2] @@ -18,13 +18,14 @@ If you're not yet running at least Velero v1.16, see the following: - [Upgrading to v1.14][7] - [Upgrading to v1.15][8] - [Upgrading to v1.16][9] +- [Upgrading to v1.17][10] Before upgrading, check the [Velero compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatibility-matrix) to make sure your version of Kubernetes is supported by the new version of Velero. ## Instructions -### Upgrade from v1.16 -1. Install the Velero v1.17 command-line interface (CLI) by following the [instructions here][0]. +### Upgrade from v1.17 +1. Install the Velero v1.18 command-line interface (CLI) by following the [instructions here][0]. Verify that you've properly installed it by running: @@ -36,7 +37,7 @@ Before upgrading, check the [Velero compatibility matrix](https://github.com/vmw ```bash Client: - Version: v1.17.0 + Version: v1.18.0 Git commit: ``` @@ -46,28 +47,21 @@ Before upgrading, check the [Velero compatibility matrix](https://github.com/vmw velero install --crds-only --dry-run -o yaml | kubectl apply -f - ``` -3. (optional) Update the `uploader-type` to `kopia` if you are using `restic`: - ```bash - kubectl get deploy -n velero -ojson \ - | sed "s/\"--uploader-type=restic\"/\"--uploader-type=kopia\"/g" \ - | kubectl apply -f - - ``` - -4. Update the container image used by the Velero deployment, plugin and (optionally) the node agent daemon set: +3. Update the container image used by the Velero deployment, plugin and (optionally) the node agent daemon set: ```bash # set the container and image of the init container for plugin accordingly, # if you are using other plugin kubectl set image deployment/velero \ - velero=velero/velero:v1.17.0 \ - velero-plugin-for-aws=velero/velero-plugin-for-aws:v1.13.0 \ + velero=velero/velero:v1.18.0 \ + velero-plugin-for-aws=velero/velero-plugin-for-aws:v1.14.0 \ --namespace velero # optional, if using the node agent daemonset kubectl set image daemonset/node-agent \ - node-agent=velero/velero:v1.17.0 \ + node-agent=velero/velero:v1.18.0 \ --namespace velero ``` -5. Confirm that the deployment is up and running with the correct version by running: +4. Confirm that the deployment is up and running with the correct version by running: ```bash velero version @@ -77,11 +71,11 @@ Before upgrading, check the [Velero compatibility matrix](https://github.com/vmw ```bash Client: - Version: v1.17.0 + Version: v1.18.0 Git commit: Server: - Version: v1.17.0 + Version: v1.18.0 ``` [0]: basic-install.md#install-the-cli @@ -93,4 +87,5 @@ Before upgrading, check the [Velero compatibility matrix](https://github.com/vmw [6]: https://velero.io/docs/v1.13/upgrade-to-1.13 [7]: https://velero.io/docs/v1.14/upgrade-to-1.14 [8]: https://velero.io/docs/v1.15/upgrade-to-1.15 -[9]: https://velero.io/docs/v1.16/upgrade-to-1.16 \ No newline at end of file +[9]: https://velero.io/docs/v1.16/upgrade-to-1.16 +[10]: https://velero.io/docs/v1.17/upgrade-to-1.17 \ No newline at end of file diff --git a/site/content/docs/v1.18/api-types/backup.md b/site/content/docs/v1.18/api-types/backup.md index 1768f0934..634264ee1 100644 --- a/site/content/docs/v1.18/api-types/backup.md +++ b/site/content/docs/v1.18/api-types/backup.md @@ -16,6 +16,8 @@ Backup belongs to the API group version `velero.io/v1`. Here is a sample `Backup` object with each of the fields documented: +**Note:** Namespace includes/excludes support glob patterns (`*`, `?`, `[abc]`). See [Namespace Glob Patterns](../namespace-glob-patterns) for more details. + ```yaml # Standard Kubernetes API Version declaration. Required. apiVersion: velero.io/v1 @@ -42,11 +44,11 @@ spec: resourcePolicy: kind: configmap name: resource-policy-configmap - # Array of namespaces to include in the backup. If unspecified, all namespaces are included. - # Optional. + # Array of namespaces to include in the backup. Accepts glob patterns (*, ?, [abc]). + # If unspecified, all namespaces are included. Optional. includedNamespaces: - '*' - # Array of namespaces to exclude from the backup. Optional. + # Array of namespaces to exclude from the backup. Accepts glob patterns (*, ?, [abc]). Optional. excludedNamespaces: - some-namespace # Array of resources to include in the backup. Resources may be shortcuts (for example 'po' for 'pods') diff --git a/site/content/docs/v1.18/api-types/restore.md b/site/content/docs/v1.18/api-types/restore.md index ec3e19511..1c80a0ee8 100644 --- a/site/content/docs/v1.18/api-types/restore.md +++ b/site/content/docs/v1.18/api-types/restore.md @@ -16,6 +16,8 @@ Restore belongs to the API group version `velero.io/v1`. Here is a sample `Restore` object with each of the fields documented: +**Note:** Namespace includes/excludes support glob patterns (`*`, `?`, `[abc]`). See [Namespace Glob Patterns](../namespace-glob-patterns) for more details. + ```yaml # Standard Kubernetes API Version declaration. Required. apiVersion: velero.io/v1 @@ -45,11 +47,11 @@ spec: writeSparseFiles: true # ParallelFilesDownload is the concurrency number setting for restore parallelFilesDownload: 10 - # Array of namespaces to include in the restore. If unspecified, all namespaces are included. - # Optional. + # Array of namespaces to include in the restore. Accepts glob patterns (*, ?, [abc]). + # If unspecified, all namespaces are included. Optional. includedNamespaces: - '*' - # Array of namespaces to exclude from the restore. Optional. + # Array of namespaces to exclude from the restore. Accepts glob patterns (*, ?, [abc]). Optional. excludedNamespaces: - some-namespace # Array of resources to include in the restore. Resources may be shortcuts (for example 'po' for 'pods') diff --git a/site/content/docs/v1.18/namespace-glob-patterns.md b/site/content/docs/v1.18/namespace-glob-patterns.md new file mode 100644 index 000000000..6a2f9f0e3 --- /dev/null +++ b/site/content/docs/v1.18/namespace-glob-patterns.md @@ -0,0 +1,71 @@ +--- +title: "Namespace Glob Patterns" +layout: docs +--- + +When using `--include-namespaces` and `--exclude-namespaces` flags with backup and restore commands, you can use glob patterns to match multiple namespaces. + +## Supported Patterns + +Velero supports the following glob pattern characters: + +- `*` - Matches any sequence of characters + ```bash + velero backup create my-backup --include-namespaces "app-*" + # Matches: app-prod, app-staging, app-dev, etc. + ``` + +- `?` - Matches exactly one character + ```bash + velero backup create my-backup --include-namespaces "ns?" + # Matches: ns1, ns2, nsa, but NOT ns10 + ``` + +- `[abc]` - Matches any single character in the brackets + ```bash + velero backup create my-backup --include-namespaces "ns[123]" + # Matches: ns1, ns2, ns3 + ``` + +- `[a-z]` - Matches any single character in the range + ```bash + velero backup create my-backup --include-namespaces "ns[a-c]" + # Matches: nsa, nsb, nsc + ``` + +## Unsupported Patterns + +The following patterns are **not supported** and will cause validation errors: + +- `**` - Consecutive asterisks +- `|` - Alternation (regex operator) +- `()` - Grouping (regex operators) +- `!` - Negation +- `{}` - Brace expansion +- `,` - Comma (used in brace expansion) + +## Special Cases + +- `*` alone means "all namespaces" and is not expanded +- Empty brackets `[]` are invalid +- Unmatched or unclosed brackets will cause validation errors + +## Examples + +Combine patterns with include and exclude flags: + +```bash +# Backup all production namespaces except test +velero backup create prod-backup \ + --include-namespaces "*-prod" \ + --exclude-namespaces "test-*" + +# Backup specific numbered namespaces +velero backup create numbered-backup \ + --include-namespaces "app-[0-9]" + +# Restore namespaces matching multiple patterns +velero restore create my-restore \ + --from-backup my-backup \ + --include-namespaces "frontend-*,backend-*" +``` diff --git a/site/content/docs/v1.18/resource-filtering.md b/site/content/docs/v1.18/resource-filtering.md index a9e65d157..69b2cb5e1 100644 --- a/site/content/docs/v1.18/resource-filtering.md +++ b/site/content/docs/v1.18/resource-filtering.md @@ -17,7 +17,11 @@ Wildcard takes precedence when both a wildcard and specific resource are include ### --include-namespaces -Namespaces to include. Default is `*`, all namespaces. +Namespaces to include. Accepts glob patterns (`*`, `?`, `[abc]`). Default is `*`, all namespaces. + +See [Namespace Glob Patterns](namespace-glob-patterns) for more details on supported patterns. + +Note: `*` alone is reserved for empty fields, which means all namespaces. * Backup a namespace and it's objects. @@ -158,7 +162,9 @@ Wildcard excludes are ignored. ### --exclude-namespaces -Namespaces to exclude. +Namespaces to exclude. Accepts glob patterns (`*`, `?`, `[abc]`). + +See [Namespace Glob Patterns](namespace-glob-patterns) for more details on supported patterns. * Exclude kube-system from the cluster backup. diff --git a/site/content/docs/v1.18/upgrade-to-1.17.md b/site/content/docs/v1.18/upgrade-to-1.18.md similarity index 70% rename from site/content/docs/v1.18/upgrade-to-1.17.md rename to site/content/docs/v1.18/upgrade-to-1.18.md index f6738d55c..0bf839fe3 100644 --- a/site/content/docs/v1.18/upgrade-to-1.17.md +++ b/site/content/docs/v1.18/upgrade-to-1.18.md @@ -1,13 +1,13 @@ --- -title: "Upgrading to Velero 1.17" +title: "Upgrading to Velero 1.18" layout: docs --- ## Prerequisites -- Velero [v1.16.x][9] installed. +- Velero [v1.17.x][9] installed. -If you're not yet running at least Velero v1.16, see the following: +If you're not yet running at least Velero v1.17, see the following: - [Upgrading to v1.8][1] - [Upgrading to v1.9][2] @@ -18,13 +18,14 @@ If you're not yet running at least Velero v1.16, see the following: - [Upgrading to v1.14][7] - [Upgrading to v1.15][8] - [Upgrading to v1.16][9] +- [Upgrading to v1.17][10] Before upgrading, check the [Velero compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatibility-matrix) to make sure your version of Kubernetes is supported by the new version of Velero. ## Instructions -### Upgrade from v1.16 -1. Install the Velero v1.17 command-line interface (CLI) by following the [instructions here][0]. +### Upgrade from v1.17 +1. Install the Velero v1.18 command-line interface (CLI) by following the [instructions here][0]. Verify that you've properly installed it by running: @@ -36,7 +37,7 @@ Before upgrading, check the [Velero compatibility matrix](https://github.com/vmw ```bash Client: - Version: v1.17.0 + Version: v1.18.0 Git commit: ``` @@ -46,28 +47,21 @@ Before upgrading, check the [Velero compatibility matrix](https://github.com/vmw velero install --crds-only --dry-run -o yaml | kubectl apply -f - ``` -3. (optional) Update the `uploader-type` to `kopia` if you are using `restic`: - ```bash - kubectl get deploy -n velero -ojson \ - | sed "s/\"--uploader-type=restic\"/\"--uploader-type=kopia\"/g" \ - | kubectl apply -f - - ``` - -4. Update the container image used by the Velero deployment, plugin and (optionally) the node agent daemon set: +3. Update the container image used by the Velero deployment, plugin and (optionally) the node agent daemon set: ```bash # set the container and image of the init container for plugin accordingly, # if you are using other plugin kubectl set image deployment/velero \ - velero=velero/velero:v1.17.0 \ - velero-plugin-for-aws=velero/velero-plugin-for-aws:v1.13.0 \ + velero=velero/velero:v1.18.0 \ + velero-plugin-for-aws=velero/velero-plugin-for-aws:v1.14.0 \ --namespace velero # optional, if using the node agent daemonset kubectl set image daemonset/node-agent \ - node-agent=velero/velero:v1.17.0 \ + node-agent=velero/velero:v1.18.0 \ --namespace velero ``` -5. Confirm that the deployment is up and running with the correct version by running: +4. Confirm that the deployment is up and running with the correct version by running: ```bash velero version @@ -77,11 +71,11 @@ Before upgrading, check the [Velero compatibility matrix](https://github.com/vmw ```bash Client: - Version: v1.17.0 + Version: v1.18.0 Git commit: Server: - Version: v1.17.0 + Version: v1.18.0 ``` [0]: basic-install.md#install-the-cli @@ -93,4 +87,5 @@ Before upgrading, check the [Velero compatibility matrix](https://github.com/vmw [6]: https://velero.io/docs/v1.13/upgrade-to-1.13 [7]: https://velero.io/docs/v1.14/upgrade-to-1.14 [8]: https://velero.io/docs/v1.15/upgrade-to-1.15 -[9]: https://velero.io/docs/v1.16/upgrade-to-1.16 \ No newline at end of file +[9]: https://velero.io/docs/v1.16/upgrade-to-1.16 +[10]: https://velero.io/docs/v1.17/upgrade-to-1.17 \ No newline at end of file diff --git a/site/data/docs/main-toc.yml b/site/data/docs/main-toc.yml index c732a9174..271705a1b 100644 --- a/site/data/docs/main-toc.yml +++ b/site/data/docs/main-toc.yml @@ -13,8 +13,8 @@ toc: url: /basic-install - page: Customize Installation url: /customize-installation - - page: Upgrade to 1.17 - url: /upgrade-to-1.17 + - page: Upgrade to 1.18 + url: /upgrade-to-1.18 - page: Supported providers url: /supported-providers - page: Evaluation install @@ -33,6 +33,8 @@ toc: url: /enable-api-group-versions-feature - page: Resource filtering url: /resource-filtering + - page: Namespace glob patterns + url: /namespace-glob-patterns - page: Backup reference url: /backup-reference - page: Backup hooks diff --git a/site/data/docs/v1-18-toc.yml b/site/data/docs/v1-18-toc.yml index c732a9174..271705a1b 100644 --- a/site/data/docs/v1-18-toc.yml +++ b/site/data/docs/v1-18-toc.yml @@ -13,8 +13,8 @@ toc: url: /basic-install - page: Customize Installation url: /customize-installation - - page: Upgrade to 1.17 - url: /upgrade-to-1.17 + - page: Upgrade to 1.18 + url: /upgrade-to-1.18 - page: Supported providers url: /supported-providers - page: Evaluation install @@ -33,6 +33,8 @@ toc: url: /enable-api-group-versions-feature - page: Resource filtering url: /resource-filtering + - page: Namespace glob patterns + url: /namespace-glob-patterns - page: Backup reference url: /backup-reference - page: Backup hooks diff --git a/site/layouts/docs/docs.html b/site/layouts/docs/docs.html index 6d2a3f57f..11e6cf9e9 100644 --- a/site/layouts/docs/docs.html +++ b/site/layouts/docs/docs.html @@ -27,16 +27,6 @@
{{ .Render "versions" }}
-
- -
{{ .Render "nav" }}
@@ -58,16 +48,6 @@ {{ .Render "footer" }}
- - diff --git a/site/layouts/partials/head-docs.html b/site/layouts/partials/head-docs.html index 5ebae8c24..c92837b2f 100644 --- a/site/layouts/partials/head-docs.html +++ b/site/layouts/partials/head-docs.html @@ -8,6 +8,4 @@ {{ $styles := resources.Get "styles.scss" | toCSS $options | resources.Fingerprint }} {{/* TODO {% seo %}*/}} - - diff --git a/test/Makefile b/test/Makefile index 7c99e4d70..0e04239d1 100644 --- a/test/Makefile +++ b/test/Makefile @@ -76,7 +76,7 @@ HAS_VSPHERE_PLUGIN ?= false RESTORE_HELPER_IMAGE ?= #Released version only -UPGRADE_FROM_VELERO_VERSION ?= v1.15.2,v1.16.2 +UPGRADE_FROM_VELERO_VERSION ?= v1.16.2,v1.17.2 # UPGRADE_FROM_VELERO_CLI can has the same format(a list divided by comma) with UPGRADE_FROM_VELERO_VERSION # Upgrade tests will be executed sequently according to the list by UPGRADE_FROM_VELERO_VERSION @@ -85,7 +85,7 @@ UPGRADE_FROM_VELERO_VERSION ?= v1.15.2,v1.16.2 # to the end, nil string will be set if UPGRADE_FROM_VELERO_CLI is shorter than UPGRADE_FROM_VELERO_VERSION UPGRADE_FROM_VELERO_CLI ?= -MIGRATE_FROM_VELERO_VERSION ?= v1.16.2,$(VERSION) +MIGRATE_FROM_VELERO_VERSION ?= v1.17.2,$(VERSION) MIGRATE_FROM_VELERO_CLI ?= VELERO_NAMESPACE ?= velero diff --git a/test/e2e/basic/restore_exec_hooks.go b/test/e2e/basic/restore_exec_hooks.go new file mode 100644 index 000000000..fc1b8ceb8 --- /dev/null +++ b/test/e2e/basic/restore_exec_hooks.go @@ -0,0 +1,150 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package basic + +import ( + "fmt" + "path" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/test" + "github.com/vmware-tanzu/velero/test/util/common" + . "github.com/vmware-tanzu/velero/test/util/k8s" +) + +// RestoreExecHooks tests that a pod with multiple restore exec hooks does not hang +// at the Finalizing phase during restore (Issue #9359 / PR #9366). +type RestoreExecHooks struct { + TestCase + podName string +} + +var RestoreExecHooksTest func() = test.TestFunc(&RestoreExecHooks{}) + +func (r *RestoreExecHooks) Init() error { + Expect(r.TestCase.Init()).To(Succeed()) + r.CaseBaseName = "restore-exec-hooks-" + r.UUIDgen + r.BackupName = "backup-" + r.CaseBaseName + r.RestoreName = "restore-" + r.CaseBaseName + r.podName = "pod-multiple-hooks" + r.NamespacesTotal = 1 + r.NSIncluded = &[]string{} + + for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", r.CaseBaseName, nsNum) + *r.NSIncluded = append(*r.NSIncluded, createNSName) + } + + r.TestMsg = &test.TestMSG{ + Desc: "Restore pod with multiple restore exec hooks", + Text: "Should successfully backup and restore without hanging at Finalizing phase", + FailedMSG: "Failed to successfully backup and restore pod with multiple hooks", + } + + r.BackupArgs = []string{ + "create", "--namespace", r.VeleroCfg.VeleroNamespace, "backup", r.BackupName, + "--include-namespaces", strings.Join(*r.NSIncluded, ","), + "--default-volumes-to-fs-backup", "--wait", + } + + r.RestoreArgs = []string{ + "create", "--namespace", r.VeleroCfg.VeleroNamespace, "restore", r.RestoreName, + "--from-backup", r.BackupName, "--wait", + } + + return nil +} + +func (r *RestoreExecHooks) CreateResources() error { + for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", r.CaseBaseName, nsNum) + + By(fmt.Sprintf("Creating namespace %s", createNSName), func() { + Expect(CreateNamespace(r.Ctx, r.Client, createNSName)). + To(Succeed(), fmt.Sprintf("Failed to create namespace %s", createNSName)) + }) + + // Prepare images and commands adaptively for the target OS + imageAddress := LinuxTestImage + initCommand := `["/bin/sh", "-c", "echo init-hook-done"]` + execCommand1 := `["/bin/sh", "-c", "echo hook1"]` + execCommand2 := `["/bin/sh", "-c", "echo hook2"]` + + if r.VeleroCfg.WorkerOS == common.WorkerOSLinux && r.VeleroCfg.ImageRegistryProxy != "" { + imageAddress = path.Join(r.VeleroCfg.ImageRegistryProxy, LinuxTestImage) + } else if r.VeleroCfg.WorkerOS == common.WorkerOSWindows { + imageAddress = WindowTestImage + initCommand = `["cmd", "/c", "echo init-hook-done"]` + execCommand1 = `["cmd", "/c", "echo hook1"]` + execCommand2 = `["cmd", "/c", "echo hook2"]` + } + + // Inject mixing InitContainer hook and multiple Exec post-restore hooks. + // This guarantees that the loop index 'i' mismatched 'hook.hookIndex' (Issue #9359), + // ensuring the bug is properly reproduced and the fix is verified. + ann := map[string]string{ + // Inject InitContainer Restore Hook + "init.hook.restore.velero.io/container-image": imageAddress, + "init.hook.restore.velero.io/container-name": "test-init-hook", + "init.hook.restore.velero.io/command": initCommand, + + // Inject multiple Exec Restore Hooks + "post.hook.restore.velero.io/test1.command": execCommand1, + "post.hook.restore.velero.io/test1.container": r.podName, + "post.hook.restore.velero.io/test2.command": execCommand2, + "post.hook.restore.velero.io/test2.container": r.podName, + } + + By(fmt.Sprintf("Creating pod %s with multiple restore hooks in namespace %s", r.podName, createNSName), func() { + _, err := CreatePod( + r.Client, + createNSName, + r.podName, + "", // No storage class needed + "", // No PVC needed + []string{}, // No volumes + nil, + ann, + r.VeleroCfg.ImageRegistryProxy, + r.VeleroCfg.WorkerOS, + ) + Expect(err).To(Succeed(), fmt.Sprintf("Failed to create pod with hooks in namespace %s", createNSName)) + }) + + By(fmt.Sprintf("Waiting for pod %s to be ready", r.podName), func() { + err := WaitForPods(r.Ctx, r.Client, createNSName, []string{r.podName}) + Expect(err).To(Succeed(), fmt.Sprintf("Failed to wait for pod %s in namespace %s", r.podName, createNSName)) + }) + } + return nil +} + +func (r *RestoreExecHooks) Verify() error { + for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", r.CaseBaseName, nsNum) + + By(fmt.Sprintf("Verifying pod %s in namespace %s after restore", r.podName, createNSName), func() { + err := WaitForPods(r.Ctx, r.Client, createNSName, []string{r.podName}) + Expect(err).To(Succeed(), fmt.Sprintf("Failed to verify pod %s in namespace %s after restore", r.podName, createNSName)) + }) + } + return nil +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index f0d1c9c2e..3d650dfbe 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -440,6 +440,12 @@ var _ = Describe( StorageClasssChangingTest, ) +var _ = Describe( + "Restore phase does not block at Finalizing when a container has multiple exec hooks", + Label("Basic", "Hooks"), + RestoreExecHooksTest, +) + var _ = Describe( "Backup/restore of 2500 namespaces", Label("Scale", "LongTime"), @@ -494,6 +500,11 @@ var _ = Describe( Label("ResourceFiltering", "IncludeNamespaces", "Restore"), RestoreWithIncludeNamespaces, ) +var _ = Describe( + "Velero test on backup/restore with wildcard namespaces", + Label("ResourceFiltering", "WildcardNamespaces"), + WildcardNamespacesTest, +) var _ = Describe( "Velero test on include resources from the cluster backup", Label("ResourceFiltering", "IncludeResources", "Backup"), diff --git a/test/e2e/nodeagentconfig/node-agent-config.go b/test/e2e/nodeagentconfig/node-agent-config.go index 1b46eed65..dc88d98bd 100644 --- a/test/e2e/nodeagentconfig/node-agent-config.go +++ b/test/e2e/nodeagentconfig/node-agent-config.go @@ -36,7 +36,6 @@ import ( "github.com/vmware-tanzu/velero/pkg/builder" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util/kube" - velerokubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/test" . "github.com/vmware-tanzu/velero/test/e2e/test" k8sutil "github.com/vmware-tanzu/velero/test/util/k8s" @@ -240,9 +239,13 @@ func (n *NodeAgentConfigTestCase) Backup() error { Expect(backupPodList.Items[0].Spec.PriorityClassName).To(Equal(n.nodeAgentConfigs.PriorityClassName)) // In backup, only the second element of LoadAffinity array should be used. - expectedAffinity := velerokubeutil.ToSystemAffinity(n.nodeAgentConfigs.LoadAffinity[1:]) + expectedLabelKey, _, ok := popFromMap(n.nodeAgentConfigs.LoadAffinity[1].NodeSelector.MatchLabels) + Expect(ok).To(BeTrue(), "Expected LoadAffinity's MatchLabels should at least have one key-value pair") - Expect(backupPodList.Items[0].Spec.Affinity).To(Equal(expectedAffinity)) + // From 1.18.1, Velero adds some default affinity in the backup/restore pod, + // so we can't directly compare the whole affinity, + // but we can verify if the expected affinity is contained in the pod affinity. + Expect(backupPodList.Items[0].Spec.Affinity.String()).To(ContainSubstring(expectedLabelKey)) fmt.Println("backupPod content verification completed successfully.") @@ -317,9 +320,13 @@ func (n *NodeAgentConfigTestCase) Restore() error { Expect(restorePodList.Items[0].Spec.PriorityClassName).To(Equal(n.nodeAgentConfigs.PriorityClassName)) // In restore, only the first element of LoadAffinity array should be used. - expectedAffinity := velerokubeutil.ToSystemAffinity(n.nodeAgentConfigs.LoadAffinity[:1]) + expectedLabelKey, _, ok := popFromMap(n.nodeAgentConfigs.LoadAffinity[0].NodeSelector.MatchLabels) + Expect(ok).To(BeTrue(), "Expected LoadAffinity's MatchLabels should at least have one key-value pair") - Expect(restorePodList.Items[0].Spec.Affinity).To(Equal(expectedAffinity)) + // From 1.18.1, Velero adds some default affinity in the backup/restore pod, + // so we can't directly compare the whole affinity, + // but we can verify if the expected affinity is contained in the pod affinity. + Expect(restorePodList.Items[0].Spec.Affinity.String()).To(ContainSubstring(expectedLabelKey)) fmt.Println("restorePod content verification completed successfully.") @@ -345,3 +352,12 @@ func (n *NodeAgentConfigTestCase) Restore() error { return nil } + +func popFromMap[K comparable, V any](m map[K]V) (k K, v V, ok bool) { + for key, val := range m { + delete(m, key) + return key, val, true + } + + return +} diff --git a/test/e2e/repomaintenance/repo_maintenance_config.go b/test/e2e/repomaintenance/repo_maintenance_config.go index 711d9bb89..c0092c4bf 100644 --- a/test/e2e/repomaintenance/repo_maintenance_config.go +++ b/test/e2e/repomaintenance/repo_maintenance_config.go @@ -55,10 +55,12 @@ var GlobalRepoMaintenanceTest func() = TestFunc(&RepoMaintenanceTestCase{ jobConfigs: velerotypes.JobConfigs{ KeepLatestMaintenanceJobs: &keepJobNum, PodResources: &velerokubeutil.PodResources{ - CPURequest: "100m", - MemoryRequest: "100Mi", - CPULimit: "200m", - MemoryLimit: "200Mi", + CPURequest: "100m", + MemoryRequest: "100Mi", + EphemeralStorageRequest: "5Gi", + CPULimit: "200m", + MemoryLimit: "200Mi", + EphemeralStorageLimit: "10Gi", }, PriorityClassName: test.PriorityClassNameForRepoMaintenance, }, @@ -70,10 +72,12 @@ var SpecificRepoMaintenanceTest func() = TestFunc(&RepoMaintenanceTestCase{ jobConfigs: velerotypes.JobConfigs{ KeepLatestMaintenanceJobs: &keepJobNum, PodResources: &velerokubeutil.PodResources{ - CPURequest: "100m", - MemoryRequest: "100Mi", - CPULimit: "200m", - MemoryLimit: "200Mi", + CPURequest: "100m", + MemoryRequest: "100Mi", + EphemeralStorageRequest: "5Gi", + CPULimit: "200m", + MemoryLimit: "200Mi", + EphemeralStorageLimit: "10Gi", }, PriorityClassName: test.PriorityClassNameForRepoMaintenance, }, @@ -230,8 +234,10 @@ func (r *RepoMaintenanceTestCase) Verify() error { resources, err := kube.ParseResourceRequirements( r.jobConfigs.PodResources.CPURequest, r.jobConfigs.PodResources.MemoryRequest, + r.jobConfigs.PodResources.EphemeralStorageRequest, r.jobConfigs.PodResources.CPULimit, r.jobConfigs.PodResources.MemoryLimit, + r.jobConfigs.PodResources.EphemeralStorageLimit, ) if err != nil { return errors.Wrap(err, "failed to parse resource requirements for maintenance job") diff --git a/test/e2e/resource-filtering/wildcard_namespaces.go b/test/e2e/resource-filtering/wildcard_namespaces.go new file mode 100644 index 000000000..c3cd50071 --- /dev/null +++ b/test/e2e/resource-filtering/wildcard_namespaces.go @@ -0,0 +1,143 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filtering + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" + + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/util/k8s" +) + +// WildcardNamespaces tests the inclusion and exclusion of namespaces using wildcards +// introduced in PR #9255 (Issue #1874). It verifies filtering at both Backup and Restore stages. +type WildcardNamespaces struct { + TestCase // Inherit from basic TestCase instead of FilteringCase to customize a single flow + restoredNS []string + excludedByBackupNS []string + excludedByRestoreNS []string +} + +// Register as a single E2E test +var WildcardNamespacesTest func() = TestFunc(&WildcardNamespaces{}) + +func (w *WildcardNamespaces) Init() error { + Expect(w.TestCase.Init()).To(Succeed()) + + w.CaseBaseName = "wildcard-ns-" + w.UUIDgen + w.BackupName = "backup-" + w.CaseBaseName + w.RestoreName = "restore-" + w.CaseBaseName + + // 1. Define namespaces for different filtering lifecycle scenarios + nsIncBoth := w.CaseBaseName + "-inc-both" // Included in both backup and restore + nsExact := w.CaseBaseName + "-exact" // Included exactly without wildcards + nsIncExc := w.CaseBaseName + "-inc-exc" // Included in backup, but excluded during restore + nsBakExc := w.CaseBaseName + "-test-bak" // Excluded during backup + + // Group namespaces for validation + w.restoredNS = []string{nsIncBoth, nsExact} + w.excludedByRestoreNS = []string{nsIncExc} + w.excludedByBackupNS = []string{nsBakExc} + + w.TestMsg = &TestMSG{ + Desc: "Backup and restore with wildcard namespaces", + Text: "Should correctly filter namespaces using wildcards during both backup and restore stages", + FailedMSG: "Failed to properly filter namespaces using wildcards", + } + + // 2. Setup Backup Args + backupIncWildcard1 := fmt.Sprintf("%s-inc-*", w.CaseBaseName) // Matches nsIncBoth, nsIncExc + backupIncWildcard2 := fmt.Sprintf("%s-test-*", w.CaseBaseName) // Matches nsBakExc + backupExcWildcard := fmt.Sprintf("%s-test-bak", w.CaseBaseName) // Excludes nsBakExc + nonExistentWildcard := "non-existent-ns-*" // Tests zero-match boundary condition + + w.BackupArgs = []string{ + "create", "--namespace", w.VeleroCfg.VeleroNamespace, "backup", w.BackupName, + // Use broad wildcards for inclusion to bypass Velero CLI's literal string collision validation + "--include-namespaces", fmt.Sprintf("%s,%s,%s,%s", backupIncWildcard1, backupIncWildcard2, nsExact, nonExistentWildcard), + "--exclude-namespaces", backupExcWildcard, + "--default-volumes-to-fs-backup", "--wait", + } + + // 3. Setup Restore Args + restoreExcWildcard := fmt.Sprintf("%s-*-exc", w.CaseBaseName) // Excludes nsIncExc + + w.RestoreArgs = []string{ + "create", "--namespace", w.VeleroCfg.VeleroNamespace, "restore", w.RestoreName, + "--from-backup", w.BackupName, + "--include-namespaces", fmt.Sprintf("%s,%s,%s", backupIncWildcard1, nsExact, nonExistentWildcard), + "--exclude-namespaces", restoreExcWildcard, + "--wait", + } + + return nil +} + +func (w *WildcardNamespaces) CreateResources() error { + allNamespaces := append(w.restoredNS, w.excludedByRestoreNS...) + allNamespaces = append(allNamespaces, w.excludedByBackupNS...) + + for _, ns := range allNamespaces { + By(fmt.Sprintf("Creating namespace %s", ns), func() { + Expect(CreateNamespace(w.Ctx, w.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns)) + }) + + // Create a ConfigMap in each namespace to verify resource restoration + cmName := "configmap-" + ns + By(fmt.Sprintf("Creating ConfigMap %s in namespace %s", cmName, ns), func() { + _, err := CreateConfigMap(w.Client.ClientGo, ns, cmName, map[string]string{"wildcard-test": "true"}, nil) + Expect(err).To(Succeed(), fmt.Sprintf("Failed to create configmap in namespace %s", ns)) + }) + } + return nil +} + +func (w *WildcardNamespaces) Verify() error { + // 1. Verify namespaces that should be successfully restored + for _, ns := range w.restoredNS { + By(fmt.Sprintf("Checking included namespace %s exists", ns), func() { + _, err := GetNamespace(w.Ctx, w.Client, ns) + Expect(err).To(Succeed(), fmt.Sprintf("Included namespace %s should exist after restore", ns)) + + _, err = GetConfigMap(w.Client.ClientGo, ns, "configmap-"+ns) + Expect(err).To(Succeed(), fmt.Sprintf("ConfigMap in included namespace %s should exist", ns)) + }) + } + + // 2. Verify namespaces excluded during Backup + for _, ns := range w.excludedByBackupNS { + By(fmt.Sprintf("Checking namespace %s excluded by backup does NOT exist", ns), func() { + _, err := GetNamespace(w.Ctx, w.Client, ns) + Expect(err).To(HaveOccurred(), fmt.Sprintf("Namespace %s excluded by backup should NOT exist after restore", ns)) + Expect(apierrors.IsNotFound(err)).To(BeTrue(), "Error should be NotFound") + }) + } + + // 3. Verify namespaces excluded during Restore + for _, ns := range w.excludedByRestoreNS { + By(fmt.Sprintf("Checking namespace %s excluded by restore does NOT exist", ns), func() { + _, err := GetNamespace(w.Ctx, w.Client, ns) + Expect(err).To(HaveOccurred(), fmt.Sprintf("Namespace %s excluded by restore should NOT exist after restore", ns)) + Expect(apierrors.IsNotFound(err)).To(BeTrue(), "Error should be NotFound") + }) + } + return nil +} diff --git a/test/util/velero/install.go b/test/util/velero/install.go index da57ef19e..e2c57392e 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -365,7 +365,7 @@ func VersionNoOlderThan(version string, targetVersion string) (bool, error) { matches := tagRe.FindStringSubmatch(targetVersion) targetMajor := matches[1] targetMinor := matches[2] - if major > targetMajor && minor >= targetMinor { + if major >= targetMajor && minor >= targetMinor { return true, nil } else { return false, nil diff --git a/test/util/velero/install_test.go b/test/util/velero/install_test.go new file mode 100644 index 000000000..8ea51bd40 --- /dev/null +++ b/test/util/velero/install_test.go @@ -0,0 +1,65 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package velero + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_VersionNoOlderThan(t *testing.T) { + type versionTest struct { + caseName string + version string + targetVersion string + result bool + err error + } + tests := []versionTest{ + { + caseName: "branch version compare", + version: "release-1.18", + targetVersion: "v1.16", + result: true, + err: nil, + }, + { + caseName: "tag version compare", + version: "v1.18.0", + targetVersion: "v1.16", + result: true, + err: nil, + }, + { + caseName: "main version compare", + version: "main", + targetVersion: "v1.15", + result: true, + err: nil, + }, + } + + for _, test := range tests { + t.Run(test.caseName, func(t *testing.T) { + res, err := VersionNoOlderThan(test.version, test.targetVersion) + + require.Equal(t, test.err, err) + require.Equal(t, test.result, res) + }) + } +} diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index 96925b0d8..5c9b39a84 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -99,6 +99,15 @@ var ImagesMatrix = map[string]map[string][]string{ "velero": {"velero/velero:v1.16.2"}, "velero-restore-helper": {"velero/velero:v1.16.2"}, }, + "v1.17": { + "aws": {"velero/velero-plugin-for-aws:v1.13.2"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.13.2"}, + "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.13.2"}, + "datamover": {"velero/velero-plugin-for-aws:v1.13.2"}, + "velero": {"velero/velero:v1.17.2"}, + "velero-restore-helper": {"velero/velero:v1.17.2"}, + }, "main": { "aws": {"velero/velero-plugin-for-aws:main"}, "azure": {"velero/velero-plugin-for-microsoft-azure:main"}, @@ -128,16 +137,13 @@ func SetImagesToDefaultValues(config VeleroConfig, version string) (VeleroConfig ret.Plugins = "" - versionWithoutPatch := "main" - if version != "main" { - versionWithoutPatch = semver.MajorMinor(version) - } + versionWithoutPatch := getVersionWithoutPatch(version) // Read migration case needs images from the PluginsMatrix map. images, ok := ImagesMatrix[versionWithoutPatch] if !ok { - return config, fmt.Errorf("fail to read the images for version %s from the ImagesMatrix", - versionWithoutPatch) + fmt.Printf("Cannot read the images for version %s from the ImagesMatrix. Use the original values.\n", versionWithoutPatch) + return config, nil } ret.VeleroImage = images[Velero][0] @@ -164,6 +170,27 @@ func SetImagesToDefaultValues(config VeleroConfig, version string) (VeleroConfig return ret, nil } +func getVersionWithoutPatch(version string) string { + versionWithoutPatch := "" + + mainRe := regexp.MustCompile(`^main$`) + releaseRe := regexp.MustCompile(`^release-(\d+)\.(\d+)(-dev)?$`) + + switch { + case mainRe.MatchString(version): + versionWithoutPatch = "main" + case releaseRe.MatchString(version): + matches := releaseRe.FindStringSubmatch(version) + versionWithoutPatch = fmt.Sprintf("v%s.%s", matches[1], matches[2]) + default: + versionWithoutPatch = semver.MajorMinor(version) + } + + fmt.Println("The version is ", versionWithoutPatch) + + return versionWithoutPatch +} + func getPluginsByVersion(version string, cloudProvider string, needDataMoverPlugin bool) ([]string, error) { var cloudMap map[string][]string arr := strings.Split(version, ".") diff --git a/test/util/velero/velero_utils_test.go b/test/util/velero/velero_utils_test.go new file mode 100644 index 000000000..73df31b6e --- /dev/null +++ b/test/util/velero/velero_utils_test.go @@ -0,0 +1,54 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package velero + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_getVersionWithoutPatch(t *testing.T) { + versionTests := []struct { + caseName string + version string + result string + }{ + { + caseName: "main version", + version: "main", + result: "main", + }, + { + caseName: "release version", + version: "release-1.18-dev", + result: "v1.18", + }, + { + caseName: "tag version", + version: "v1.17.2", + result: "v1.17", + }, + } + + for _, test := range versionTests { + t.Run(test.caseName, func(t *testing.T) { + res := getVersionWithoutPatch(test.version) + require.Equal(t, test.result, res) + }) + } +}