diff --git a/.github/workflows/pr-codespell.yml b/.github/workflows/pr-codespell.yml index 1fff7e920..ddc2acd4e 100644 --- a/.github/workflows/pr-codespell.yml +++ b/.github/workflows/pr-codespell.yml @@ -14,7 +14,7 @@ jobs: uses: codespell-project/actions-codespell@master with: # ignore the config/.../crd.go file as it's generated binary data that is edited elswhere. - skip: .git,*.png,*.jpg,*.woff,*.ttf,*.gif,*.ico,./config/crd/v1beta1/crds/crds.go,./config/crd/v1/crds/crds.go,./go.sum,./LICENSE + skip: .git,*.png,*.jpg,*.woff,*.ttf,*.gif,*.ico,./config/crd/v1beta1/crds/crds.go,./config/crd/v1/crds/crds.go,,./config/crd/v2alpha1/crds/crds.go,./go.sum,./LICENSE ignore_words_list: iam,aks,ist,bridget,ue,shouldnot,atleast check_filenames: true check_hidden: true diff --git a/Dockerfile b/Dockerfile index c628ef524..7f600d0f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -68,7 +68,7 @@ RUN mkdir -p /output/usr/bin && \ # Velero image packing section FROM gcr.io/distroless/base-nossl-debian11:nonroot -LABEL maintainer="Nolan Brubaker " +LABEL maintainer="Xun Jiang " COPY --from=velero-builder /output / diff --git a/Makefile b/Makefile index 6aac273f5..f9229ffc9 100644 --- a/Makefile +++ b/Makefile @@ -229,6 +229,10 @@ endif update: @$(MAKE) shell CMD="-c 'hack/update-all.sh'" +# update-crd is for development purpose only, it is faster than update, so is a shortcut when you want to generate CRD changes only +update-crd: + @$(MAKE) shell CMD="-c 'hack/update-3generated-crd-code.sh'" + build-dirs: @mkdir -p _output/bin/$(GOOS)/$(GOARCH) @mkdir -p .go/src/$(PKG) .go/pkg .go/bin .go/std/$(GOOS)/$(GOARCH) .go/go-build .go/golangci-lint diff --git a/Tiltfile b/Tiltfile index bc7438384..a1e13558b 100644 --- a/Tiltfile +++ b/Tiltfile @@ -12,6 +12,8 @@ k8s_yaml([ 'config/crd/v1/bases/velero.io_schedules.yaml', 'config/crd/v1/bases/velero.io_serverstatusrequests.yaml', 'config/crd/v1/bases/velero.io_volumesnapshotlocations.yaml', + 'config/crd/v2alpha1/bases/velero.io_datauploads.yaml', + 'config/crd/v2alpha1/bases/velero.io_datadownloads.yaml', ]) # default values @@ -60,7 +62,7 @@ RUN wget --output-document /restart.sh --quiet https://raw.githubusercontent.com additional_docker_helper_commands = """ # Install delve to allow debugging -RUN go get github.com/go-delve/delve/cmd/dlv +RUN go install github.com/go-delve/delve/cmd/dlv@latest RUN wget -qO- https://dl.k8s.io/v1.25.2/kubernetes-client-linux-amd64.tar.gz | tar xvz RUN wget -qO- https://get.docker.com | sh diff --git a/changelogs/unreleased/6111-eemcmullan b/changelogs/unreleased/6111-eemcmullan new file mode 100644 index 000000000..608f5a828 --- /dev/null +++ b/changelogs/unreleased/6111-eemcmullan @@ -0,0 +1,2 @@ +Remove any dataSource or dataSourceRef fields from PVCs in PVC BIA for cases of +prior PVC restores with CSI \ No newline at end of file diff --git a/changelogs/unreleased/6176-Lyndon-Li b/changelogs/unreleased/6176-Lyndon-Li new file mode 100644 index 000000000..d9840b919 --- /dev/null +++ b/changelogs/unreleased/6176-Lyndon-Li @@ -0,0 +1 @@ +Add data mover CRD under v2alpha1, include DataUpload CRD and DataDownload CRD \ No newline at end of file diff --git a/changelogs/unreleased/6232-kaovilai b/changelogs/unreleased/6232-kaovilai new file mode 100644 index 000000000..b3d112460 --- /dev/null +++ b/changelogs/unreleased/6232-kaovilai @@ -0,0 +1 @@ +log volumes to backup to help debug why `IsPodRunning` is called. \ No newline at end of file diff --git a/changelogs/unreleased/6268-Lyndon-Li b/changelogs/unreleased/6268-Lyndon-Li new file mode 100644 index 000000000..2aa27069a --- /dev/null +++ b/changelogs/unreleased/6268-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #5123, Kopia repository supports self-cert CA for S3 compatible storage. \ No newline at end of file diff --git a/config/crd/v1/bases/velero.io_backups.yaml b/config/crd/v1/bases/velero.io_backups.yaml index c17ab074f..5314b862c 100644 --- a/config/crd/v1/bases/velero.io_backups.yaml +++ b/config/crd/v1/bases/velero.io_backups.yaml @@ -40,6 +40,11 @@ spec: CSI VolumeSnapshot status turns to ReadyToUse during creation, before returning error as timeout. The default value is 10 minute. type: string + datamover: + description: DataMover specifies the data mover to be used by the + backup. If DataMover is "" or "velero", the built-in data mover + will be used. + type: string defaultVolumesToFsBackup: description: DefaultVolumesToFsBackup specifies whether pod volume file system backup should be used for all volumes by default. @@ -454,6 +459,11 @@ spec: - name type: object x-kubernetes-map-type: atomic + snapshotMoveData: + description: SnapshotMoveData specifies whether snapshot data should + be moved + nullable: true + type: boolean snapshotVolumes: description: SnapshotVolumes specifies whether to take snapshots of any PV's referenced in the set of objects included in the Backup. diff --git a/config/crd/v1/bases/velero.io_schedules.yaml b/config/crd/v1/bases/velero.io_schedules.yaml index 411d12826..a334ee6ce 100644 --- a/config/crd/v1/bases/velero.io_schedules.yaml +++ b/config/crd/v1/bases/velero.io_schedules.yaml @@ -70,6 +70,11 @@ spec: for CSI VolumeSnapshot status turns to ReadyToUse during creation, before returning error as timeout. The default value is 10 minute. type: string + datamover: + description: DataMover specifies the data mover to be used by + the backup. If DataMover is "" or "velero", the built-in data + mover will be used. + type: string defaultVolumesToFsBackup: description: DefaultVolumesToFsBackup specifies whether pod volume file system backup should be used for all volumes by default. @@ -490,6 +495,11 @@ spec: - name type: object x-kubernetes-map-type: atomic + snapshotMoveData: + description: SnapshotMoveData specifies whether snapshot data + should be moved + nullable: true + type: boolean snapshotVolumes: description: SnapshotVolumes specifies whether to take snapshots of any PV's referenced in the set of objects included in the diff --git a/config/crd/v1/crds/crds.go b/config/crd/v1/crds/crds.go index 1db34fa5d..fc60e2201 100644 --- a/config/crd/v1/crds/crds.go +++ b/config/crd/v1/crds/crds.go @@ -30,14 +30,14 @@ import ( var rawCRDs = [][]byte{ []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4VAo\xe46\x0f\xbdϯ \xf6;\xec\xe5\xb3g\xb7\xbd\x14\xbem\xd3\x16\b\x9a\x04A\x12\xe4Nۜ\x19mdI\x95\xa8I\xa7E\xff{A\xc9\xcexl'\xb3Y\xa0\xbaY\xa2\x1e\xc9G>ZEQ\xacЩG\xf2AYS\x01:E\x7f2\x19\xf9\n\xe5\xd3O\xa1Tv\xbd\xff\xbczR\xa6\xad\xe0\"\x06\xb6\xdd\x1d\x05\x1b}C\xbf\xd0F\x19\xc5ʚUG\x8c-2V+\x004\xc62\xcav\x90O\x80\xc6\x1a\xf6Vk\xf2ŖL\xf9\x14k\xaa\xa3\xd2-\xf9\x04>\xb8\xde\x7f*?\xffP~Z\x01\x18쨂\x1a\x9b\xa7\xe8<9\x1b\x14[\xaf(\x94{\xd2\xe4m\xa9\xec*8j\x04}\xebmt\x15\x1c\x0f\xf2\xed\xdes\x8e\xfa\xe7\x04t7\x00\x1dґV\x81\x7f_<\xbeR\x81\x93\x89\xd3ѣ^\n$\x1d\ae\xb6Q\xa3\x9f\x19\x88\x83\xd0XG\x15\xdcH,\x0e\x1bjW\x00}\xa6)\xb6\x02\xb0m\x13w\xa8o\xbd2L\xfe\xc2\xea\xd8\r\x9c\x15\xf05Xs\x8b\xbc\xab\xa0\x1c\xd8-\x1bO\x89\xd8\a\xd5Q`\xec\\\xb2\x1d\b\xfb\xb2\xa5\xfe\x9b\x0f\xe2\xbcE\xa69\x980W\x1ec}88:A9\x12\x01\xa3\xb3\x8c\x18\xd8+\xb3]\x1d\x8d\xf7\x9f3\x15͎:\xacz[\xeb\xc8|\xb9\xbd|\xfc\xf1\xfed\x1b\xc0y\xebȳ\x1aʓר\xfdF\xbb\x00-\x85\xc6+ǩ9>\n`\xb6\x82V\xfa\x8e\x02\xf0\x8e\x06N\xa9\xedc\x00\xbb\x01ީ\x00\x9e\x9c\xa7@&w\xe2\t0\x88\x11\x1a\xb0\xf5Wj\xb8\x84{\xf2\x02\x03ag\xa3n\xa5]\xf7\xe4\x19<5vk\xd4_/\xd8\x01\xd8&\xa7\x1a\x99\xfa\x1e9\xaeTC\x83\x1a\xf6\xa8#\xfd\x1fд\xd0\xe1\x01<\x89\x17\x88f\x84\x97LB\t\xd7\xd6\x13(\xb3\xb1\x15\xec\x98]\xa8\xd6\xeb\xad\xe2Av\x8d\xed\xbah\x14\x1f\xd6IA\xaa\x8el}X\xb7\xb4'\xbd\x0ej[\xa0ov\x8a\xa9\xe1\xe8i\x8dN\x15)t\x93\xa4Wv\xed\xff|/\xd4\xf0\xf1$\xd6Y-\xf3Jby\xa3\x02\xa2\x16P\x01\xb0\xbf\x9a\xb38\x12-[\xc2\xceݯ\xf7\x0f0\xb8NŘ\xb2\x9fx?^\f\xc7\x12\ba\xcal\xc8\xe7\"n\xbc\xed\x12&\x99\xd6Ye8}4Z\x91\x99\xd2\x1fb\xdd)\x96\xba\xff\x11)\xb0Ԫ\x84\x8b4\x8b\xa0&\x88N\xd4Жpi\xe0\x02;\xd2\x17\x18\xe8?/\x800\x1d\n!\xf6\xdbJ0\x1e\xa3S\xe3\xcc\xda\xe8`\x18\x81\xaf\xd4k:\xd6\xee\x1d5R>aP\xae\xaa\x8dj\x926`c=\xe0̾<\x81^\x96\xae\xac<\xfc\xee\xd9z\xdcҕ͘S\xa3\xc5\xd8&w\x86\xe0d\xb2d\x19Ӳ\xe1\f\x1b\x80w\xc8#\xfd2*\xf32\x06\x16\xf3y\xa3\b\xa9\x10(r6h\x1a\xfa-u\x94i\x0egr\xba^\xb8\")\xed\xec3\xd8\r\x93\x19\x83\xf6\xb1.dR\x13\xf8h\xde\x15\xec\xe90?\x13\xe6݉1(\xd3J\x1b\xf4\xd3T\x9c\f\xd4K]ɴ\xe0O\xff\x9b\xe3E&vsw\x05\xfe\xb4~\xf7\xaf\xeb\x9f\xde\x00\b\x96\xe3-lY\xf2T\x16z}\xc4\f\x95\\s\xf9F\x17\x98X\x90{%\xcb\xe2\x16\x9a\x1f\\\x17?\x9cC\xf5\xcfԛ^d\\\x9b\x9f[/\x7f\xe1\xda\xd0\x0fEV*\x96\xd5#\xd1;\xcdž̘\xaa\u07be\x01Љ,\xf0\x16>\xdb!\n\x96`\xfa\x06\xc0cMC\xae<\xc2\xc7w\x0eBr\xc0\x9c9\\\x00d\x81\xe2\xfd\xfd\xe6\xeb\xbf=t^\x03\xa4\xa8\x13\xc5\vCsw\x88\x01\xd7\xc0\xe0+M\v\x94\xa72\x98\x033\xa0\xb0P\xa8Q\x18\r怐\xb0\u0094\nA\xee\xe0\xe7r\x8bJ\xa0A]\x83\x06H\xb2R\x1bT\xa0\r3\b\xcc\x00\x83Bra\x80\v0\xc82K\xad\xb0\x1dQY\x82%r/\xf8\xdfk\xd8\x1a\x8c\xa4A3f\xd0\v@\xf3paP\t\x96\xc1\x91e%^\x13Irv\x02\x85v\x14(E\v\x1e5\xd1k\xf8/\xa9\x10\xb8\xd8\xc9[8\x18S\xe8ۛ\x9b=7բId\x9e\x97\x82\x9b\xd3\r\xc9?ߖF*}\x93\xe2\x11\xb3\x1b\xcd\xf7+\xa6\x92\x037\x98XFް\x82\xaf\buA\vg\x9d\xa7\xffR\t\x80~\xdb\xc1՜\xac0j\xa3\xb8ط~ \xa9\x9f\xe0\x80]\x00N\xbe\\W7\x8b\x86\xd0\xf6\x95\xa5Η\x8f\x0f\x8fm\xd9\xe3\xbaO}\xa2{K \x1b\x16X\x82q\xb1C嘸S2'\x98(R'}$\xba\x19G\xd1'\xbf.\xb797\x96\xef\x7f+Q[!\x97k\xb8#M\x02[\x84\xb2H\xadd\xaea#\xe0\x8e\xe5\x98\xdd1\x8d\xaf\xce\x00Ki\xbd\xb2\x84\x8dcA[\t\xf6\x1b;\xaa\xb5~\xa8t\xd9\b\xbf\x9cBx(0\xe9,\x18ۋ\xefxB\xcb\x02vR5\xfa©\xabu\adx\xc9\xda'\xd1\xfcA\xb0B\x1f\xa4y\xe49\xca\xd2\xf4[\xf4\x10\xba{\xd8\xf4:T\xc8x\xd4H\xad\x94\x1aS\xbbΞ\x197\x16\xbd\x01L\xb0\x80\xe0+i\x98\n\x1ei\x9aR\x83)\x95\xa0U\xfa\x05Yzz\x94\x7f\xd1\biI\u009a(\xa4)_\xc3\x16wRa\x00\xaeB\xdb\xdf6F\xa5,a4\xa1$K\xb3\x86\xc7\x03Z2\xb223^\uee46w?A\xceEip=\x806\xc2`G\x14\x02\xe3f\xa0\x1f\xe5'\xedX5C\xbe\x0f#\xddZD|>\xa09\xa0\x82BV*80\xcb\x1d\xcf\x10\xf4I\x1b\xcc=\xc7+ŷ\xf5\xd4'\xa1\xc82\x0fB\xc3\xf6T\xe1<\x9c\xa7(\xb3\x8cm3\xbc\x05\xa3\xca\xe1p\x8e\f[)3d}%ܧ\xc3\x17Ԇ'3T\xb8\xea\x93\xc1\xf5\n\x10A\xf9\x1fhn\x01:l\x1bY3\xec\t\x81U\u0530\xc6!\xcbZD\xecP\x00\xfe[\xc0\a\xab\xb9\x12\xabO\x86\u0602\xd7\\\x1c3ҖBB&\xc5\x1e\x95\x1b\xcdZ\x85g\x9eevx\x85\xb9Nv\xf6v$\xe3\ty \xde\xd9X\x91\x9f\x14bVcNN\x05:W\xcd2\xcec\xd8\xd8\t\xaf\xc2`\xb3\x03\x8d\xc66\xb9\xfa\xd3յ\xe5g\x00hw\xd4\xee\x18\x1a\x98\u009a\x02\xe1\xf5\x1f\x00\x89yaNC\xeeq\x83y\x80`\x93j\"\x92uL)v\x1aa\\\xedo\x9eǺ\xb1\xee=扪\xd9\xef̾\xfe\xb8\v\x19\x18\x80\xc8\xf5\xf7\xca\xc0\xc5,\xd3\x1431.,\xabl\xf8\xd2ᔦ\x90 0\x1dK3\xeb1q\xe1\xe0\x91\xb7\xdf0\xe6{\xa1\xcbRI\x1e\x13\xddZb\xbcH\xda8\x89\x05}\x83\xef\x98(\a)\x9f\xe6\b\xf1\x9f\xb6M\xe3qCBa8l\xf1\xc0\x8e\\*?\xf5\xc6\x0f\xc0\x17LJ\x13\\\xcb\xcc@\xcaw;T\x16Nq`\x1a\xb5\v\xba\xc6\t2\xeeDBK9\x04\x7f\xecͣa\xa4\x95T\x9a\xf9\x18\xea\xd6\x11\xe8[\xb4걈Z?\x8f,gʏ<-YFF\x94\x89\xc4͇\xd5x\x85\x8c\xf1\x04\x93\a8;\x13]an9\xd1qʥ@\x90\nr\x1b\x89\f\x9b\x86\x8c\x8c{Ʀ\xbde\xd6ϐNDU\x99\xa1\xf6C9Ǯ\xd1\x01ף\xa0k\x8e\xb8(6c[\xcc@c\x86\x89\x91*L\x8e9&\xbb'F\xaf\x8dP1\xa0\xe1\xba\xd1C3\xb1\t\x90@\xa1Ł'\a\xe7\xa6Y\t\"8\x90JԴ\xcaYQd\x01\v\xd0<\x93\x9c\xf7\x83L-\xf4\xe6\x99Y\xf2}x\xa1\xc5\xdf<\x11\xba\xb1yf\xb4d\x97\xb2\xb58\x80\x91\x93\xd3\xfe\xe7$l\xa5\xf6\xcf\x10\xda͠\xebe\x85֒\x94\xa3&\x87\x89<\x97k\xe0\xa6z;\a\xd1\x069\xcd\xf8\xff\xc0\x8cY.\xf1\x9b~ϋJ\xfc$W\xe6 Z\xae\xd4\xc3\xff\x032\x85\x8cŃ\xb7\x15\xd1\f\xf9\xa5\xdd\xeb\x1a\xf8\xaefHz\r;\x9e\x19T=\xce|\xd3z\xb9\x041b\xec\x9d}rf\x92\xc3\xc7\x17\xeby\xe9fG#\x92.\xfd\xce\xce\x7f\xad\xfc\xf9\xaea\x9e\x81\v\x94\x1a\xe5\ns\x97r}$j6oȣz\xff\xf9\x03\xa6S\xe4\x818\xc9\x1bL\xe4}\x0f\xd9\xf6\xd0\xde)\x8f\x9d\x86w}\xea\xf8\xc6%ӯ\x81\xc1\x13\x9e\x9c\xc7\xc2\x04X\xe60;\xd0H\xa43$\x0ee\xf5IȞ\xf0D`|\x9a~\xb6w\xac(\xb8\xe7\tO1\xcdz\x04\xb48q\xed\xb7\x1f,%\xed\v\"\x04eu\xe3\x89\a\xb4\xe5R\xe9\xa2\xf9\xc9A\xbc\"\xa9\x9e\x8a\xf6gL\xb3f[k\xbb\x8a\x18\xfbV;\x16\xd9Up\xe0E\xe4D\xad\x99\xa3T\x82\xdc՛._Y\xc6\xd3z '\xf7\x1b1\xee\rw\x9f\xcf\xd2l\xc45||\xe1\xda\xef{}\x90\xa8?KCo^\x85\x9c\x0e\xf13\x88\xe9:\xd2\xf2\x12Nm[:\xb4wo\"\x84\xdb=\x1b\x17\xe1\xd5\xec\xe1\x1a6\xc2\xc6-\x9e\x1e\xb4\x17熛\xb6\x0f\xdd'/5m\xcf\b)V.\xf5\x12\x1a\xc9\x11;\x12\xa4T\x1d\x8e\fQ\xab\a\x1d\xc9\xf5\x84\x9fGkI\\\x7f\xb7\xbb\x98\xb1\x04\xd3jw\x81\xf6Ę\xc1=O G\xb5\x9f2\x1c\xed\xa7\xb0\xfa=\x0e\x85H\xad랅\x12\x16gګǫ\xee`\xf2\xbb\xfb\xac\xecʍhU1{\xb6\xe9\xc8V\xd8x\xd3\xf9\x19\x91\x89%\xffc\x96\xba,M\xa9X\x81e\xf7\v4\xfe\x02^\fm\xbfC\xccYȜ\xd1\xe6\xc4\xffX3G\x02\xfd\xbfP0\xae\"\xd6\xf0{*JȰ\xd3\xd7g\xb1\xda\xc3\xd8\x11\xb8\x06\xcb\xdf#ˆ\x9b\xac\x81\xc9I\xab[0s\x86\\\xee\x06\x1e\xcb5<\x1f\xa4v6\x956EfAr\rWOx\xba\xba\x1e聫\x8d\xb8r\x06~\xb1\xba\xa9\xbd\x05)\xb2\x13\\Q߫oq\x82\"%1\xb2\xd9\xcb\xea\xa9.\xc2X\xe5\xacXy\xe952\xe7\xc9h?*\x1a\x89u\xb1m\fZy\x10\xb6c])a\xdd\xe3\xa9\xd9F\xc9o!u`\xffw\x04\x95{\xa9\x8d\xcbHv\xdc\xd9%\xd9/p\xb2\xe7\xb3^\xc0v\xaeVE\xaa\xaa\n\xc1\xaa\xcb^\xa2\xd6r[Okf\xb7g\xe03i\x0e\xa8\rȮ\x9a\x95\xef\xf4\xf0\x95۳\xa0AXBN\xc9,\xdcB\xc9\x04\xb5\x9e\x16\xad\b-?\x93\\\xac\x13\x8b\xcc\x05>n\x8b\x7f:\x99Y=\xf1\x8e\xac%\xd2\xc2\x10\xe0\xe3K+\xebi\x95\x86\xfd{N\xf8\x96\xe2\x05\xb4\xd6\xf3\x9c\xf5kY\xa2P\xbcs=\xabe\xe2\x01\xb9\x90B\xedKR\x11\xf1\x9e\xa7\x17\xa4\xef\xc1\xbc\xe7\\lh\x00xwqw\xa0V\xaex\x8e\xc3\x7fW\xf5m\x88^\xbf\xa0\xd5\x1b\xebII\xca\xf8+\xecpn\x98\x1f\xb7\x0ef$H!M;\ra\xe1\x162}\xabaǕ6mDc\x85\xa2\x9cY\xfdͳ4\xe2\x12\x1f\x95:+\xe0\xfa\xd5\xf5l%\xc0\x0e\xf2\xb9\xaa\b\x1a-\x9e\b=\xb4\x99\x84\xc0w\xc0\r\xa0Hd)(mc\x97:\r\xe1X\xe0\x14t4\xc9\xe2\x14\x84}P\x94y\x1c\x01V$u\\L\xe6w\xda\xcd?1\x1eځ\x1e>\v\xd9f\xc6\n\xa7BO\x87mU\x05U\xbb\xb4+g/?\\\xb4\xf9燋\xe6\x9f\x1f.Z\xf5\xfcp\xd1~\xb8hSͦ\xb4\xf5\x1cF\xee\x8c\xd8ȏ\xb3XDlkO\xa18\x01\xdfWa\xf8:\xef\xd8\xca\xccM\xb8W\xa0\x8e?\xba6\\\xb7LI]\xaai\x17H%\xde\xee\xc8\xcbL\xf1\xe67\xd4\xcbW\x83\x9eU/\xbf\x99\xec|\xa1zy\x8fa\xdf\xeb\xbeP\xb5|5\xffe\xd5\xf2\u05feT#GV\xa5\xe7\xdd^|:6do\xb4\x01\xe0߹\xfevP\x1fv\x1e\xe3_\xbd\xda~\x84\xf9\x91\x85\xf1W\x7f\xba\xfa\xfe(\xbd\x98\xb6\xa3\xd4\x1c\x90)0\xa9갫\x8d+\xdb\xc5]\xddB\xba\xefS8\x97Jcl\xc5\xfc\x14\xbd\x86Z\xa6E\xb0\xefu1\x1b\xcc\x7f-\xbc\xad\x88;\u05f8\tt\x99;\xd9\x18\x98\x0f%\x01\xf4I$\a%\x85,\xb5\xcf\x1bX\xe8\xef)}\xe1\xb7B\xa9\f,R\xc1\xbe\x83\x83,\x03\x15\xdb\x13\xb4\x9b\xa9\xdf\x1b\xaf\xda\xf3{\xd4h\xd8\xf1ݺ\xfb\x8b\x91\xbe\x86\x0f\x9e\xb99\x04\xf0|>\xa0\xa0\xddu\xb1o\x17\xe4W\v\xce\x1f\xbb\xee\v\x12H\x05\x82gc\x06\xab>\x9b\xde1M\xbf\x16.I\xb4\xd8\xf2O'8\xe2\xaa\xfcή\xed\xeb\xd6\xee\x8d8\x81K7\xb3\xe3\x8f0\xc4W\xefM\x97\xdb-\xa9\xd9\xebW\xe4\x8d\x02\x9d\xafԋ\xc9M\xcdT\xe5\x9dQ\x8b\x17Y\x87\xfd\xcd[\xef1\xd5vg\xd5\xd8͖*GV\xd6uk\xe6\xa6A.\xa8\xa7\x8b\"\xce|\xed\xdc\xe2\x8a9_\xa169\x8f\xe8:\xb9@\x05\xdc$\xe0\xd1긩\xba\xb7\x99\xbc\xf7\xb0&.\xbe\xdam\x124U\xc2\xcd\u05f8]\xae\x92\xfd\x12Q\xf6\xb8\xaa\x99\xadS\x9b\x8d§\xf1\x9b\xadD[R\x7f6K\xb13k\xcd\xeaZ\xb2\x91q\x97V\x98u+\xc8F\x80\xc6ԕ\x8dԍ\x8d@\x9c\xac&\x8b\xad\x16\x1b\x81=cv'\xa5d\xf2\xc7%Ub\xe1\xabD`\xd6\x1af\xbf\x97\xfc\x9dK\x06\xa9:\xce\xe5\\@\xf3k\xaf\xb9\xe5|\xe5cM;\xab!?\x95\x9b\xc3rg5/3Ë\x8c\xb6\x17\x8f<\r\xc6\xec性\xfab\x88\xdf$\x1d\xd7ܞ\bү_ja^\xf7\\n\xa6\xe1\x19\xb3\fXH\x14\a3O\xdcm8\x89\\\xa1\xb5\x10vy\xfa+/\xfc\xa59\xd7N\xde\xe9Djh\a\xc6\x1c0\xb7P\xaa\xbb3\x16Dk\xd3\xee\xa4\xf3|\xe9\xdd\xdfJT'\x90GT\x8d\x7f1s\x1e\xca-Kmc\xa1JQxm\xe3\xee`\xea\xb9\xd9\xcd\xf2\x84\xf7\xc2\x19\xbc \xd8\x1e\x8e\x04\xc7j\x88\xac\xe6\xb5U\x866j\x18i\x1aN\xc4ʺw\xe0\xf79O5\xf60\xd1\xeb\x06\x1a\xcbC\x8dY#\xff*\xe1\xc6\xf9\x01\xc7\x04\xc8\xd8\xc3Aq\x1bⳇ\x81^+\xf0\x98\v=\xa2}\xae\xb8\xc3>\xafq\xc8g\xc1\xe1\x9e\x05!Ȳ $\x9aL1\x87x^%\x14y\xc5`\xe45\u0091\xf3\x02\x92\x19\x90\xbd\xc391\xc7n\xa2\x8a=\xa2\xf7;c\x8a5\xe6\xb7$\xa7\x8f\xd3D\x1c\xa3\x89ج\x9c\xc34\xe2\xb8̲c2\x114|\xa5P啂\x95\xd7\bW^7`\x99\rYf%g\xe6\xe7e\xc7[\xceN\xdeK\x95\xa2\x9a\xdc\xeb\x88\x15\xcdI\xa1\xec\xc5\x17\xdd1{\x99\xff\xeaN9۪\xe3ʆ\x12\xd6\xf5\xa9\xf7\x04~\xe6\xc2\xef\xa3Z!l\xd9\xfd\xce\x06L㈄\xf3\xff\x8d\x97\xe7\xef\xdct\xbb6\x1a\v\xa6h\x87u{r\xa5\x15z\r\x1fYr\xe8A?\x04㊝T93pUoy\xdd8\xe0\xf6\xef\xab5\xc0'Yoڷo\x92\xd1v뒫\xfa\xb3\x9a\xe6~\xe3\xb0[\x93\xb00q\x02I\xb5\x1e\xe6\xc0U\xba*\x982'WWp]\xe30\x9eǩ\xec\xe6T\xb6eԼ\f/\xbd\rҶ\xba\xfb\x966\xf3NEw+\xb4O\xd1s\xf0\x18?\xca7{\x88\xef\x82x\x8c\xbb +\xa2T\xe0u\xb0(\xe9bY,\xed/x\xf57^\xce,\xe0\x87n\xeb@1Qu\xd9g\x05W\x87\x13\x1dV\"\xef\xbfR4R\xafz\xef\x1f\xf8x\xa3\xca\xf6\xf4\xef\x84\xfb\xf3\xe5ˊ\xb4\x91\x8a\xed\xf1\x17\xe9\xee읣A\xb7u\xe7\xc2f\xef%Te~\x95\x00\x85\xbcg\x7f{p\x0fXS\xbd;\xb89\xd6b\x19Z\x83\x13\xf2fL63\x99\xc7\xc7_\xdc\x04\f\xcfq\xfd\xa1t\xdb\xdfVAh\xb4Ԭ&\xe6:m\xed\x7f\x0f\x01\x15\vt\x05k\x8b?-\xbc\x15R\x850U\x8a-\xc2\xfeع\x81\xb8\"ќ\x88~\r\xf7j\xa5dZLr\xd6:(\xa1cpZ\x97\xb0S\xb2\x92N\xf8^\xf6\xa2\xc21\xffk\xec\x9aj\xba\x9ay\xfe\xa2jw\x83\xb3\xbf\x96\xdeי\x97\x8an\x19\xf4\xb7;ӭ|g\xddU\xbd\xadK)\xeaB\r\xfd\xde\x18\x1b[\x86\x94]\x00\xbd\x91\xbe\xb5M\x90\x86e \xca|Kn^H\xa5\xd4]\xa8\xc8c\xb2\xba\xc3\xd9\xec\t\xc69Rsap?\xc8S\x87\xe6z\xe7˂ϙk\xdd7~\xae\xbaL\x12\xd4zWf٩.I^2\xf1\x00\xccK\x91\xe2\x13\xe3\xd9Ytp\x1dG\x88\xe0\xe66\xaaG\xa3\xd8\xec\xeb Q\xa4\xd5\xe2\x1d\x98\x02\xfbP\xa1\xfe2:x\x16\xf8\xf2$mX>w\xd9\xf8ݰ\a}\x0fA\xa5\xad\x82\xa6\xfa\xc6\xecg\xa6\x1b6\x87ܰ\x06\x9c\xebI^\x9b\x85\x86)\xe0\x11\x05HA\x85\xe6t}\xa5\xfbfG\xbfO\x00j\x1b\x8a\xafd/\x8bL\xb2\xb42pU\xf0\xe5\xbf\xf3\xf0H\xe6[\x1dQ\xbd\xd5\x130\xeb;\xd0\x03D\x18J\xa6\x8b\x86n!e\x06WA\xa0Q\xa6?\xa8k\x13ͻz>Zi\xdd=l\xc6z\x8eJp\xd5 Ŀ\xc1\x8d\xfbߨ\xa4\x863\x8bUQÙ\xcd)\xa8\x8e:\nL\xaeQP\x17\x9f&\xad\xd5\xd9K\x8a\xa9\x91\xf3\x00\xe8DNu\x03\xbd;\x92\x93\xa3\xd6l_\xddN\xfcl\x1d\xb0=\n\xa4\xf8=0\x1b\x9f\x11m\x8ept\xef\xe6u[7,1%\xf3\x03T\x05r\xadVoC\n8\x93{\xf7\xed\x00^}9\xa5\xf2L\x17\xd2\xe4\xa5\xe0*Ɠ\xfdX7\xb4\xb4\xa1]WbD\xf3\xa5\x1b\xcc\xf8\x9e[7\xd02i\xcfԖ\xedq\x95\xc8,CҵC\xbc^s\xb1\xfa\x832_\x90\xe9٩}j\xb7\xf5)~\xc7mw\xb5\x1ds\xb5\xa2\xf4\xe1\x13\xc3\x156_\x12\x1a $i\xe0E\x9e\xab\xa3B\xf0\x9b;CL\xdbm\xab\x05\xe6\xf5\xaaO\x04\xf9O\xf0\\\xfbX(\x1c\xdf\xe6\xec7\xa9\xae!\xe7\xc2\xfe\xc3D\xear\xf0U\xe7E\xf8ӥ\xd33x\xdf\xdb6\xf5yŖ\x1f\x89Ղ\x18\x8b\xd4\xc2g\xd4V\xf0\x19\x87\x81\x85;v\x86)\xed:\x85>4d\x9blĽ\x92{\x85z\xb8\xaaV\xf0W\xc6\r\x17\xfbOR\xddg型\xc6\xdfX\xd4\xf8\x9e)\xc3Y\x96\x9d\x1c>!D\xb9`\x19\xff{\x88;\xed\x1f\xe7\x01\xd5\xea6\xf0[\x04\x1ac?|@kjGÍ\xb0 x\xba\xceɂo\xd6dɹp\xb2KGܶ\xb24\x1d\xe5\xd7(π Wc\xae\xe1\xb34Xm\xbe\xf2.Lk.P\x9b\x15\xeevR\x19\x97\x94_\xad\x80\xef|\xf8\x12ʪ2\x9eQ\xf1\x88\xfbR\x11p\xd3T\xdb5\xeb\x8d2\x13\x8a\xd4\x06]*\x9b\xb3\x93K\xb9\xb1$\xb1\xd11\xdehò\x80F\xfe\xa6\xe2f\x8a\x13\xedz\xc1\xf4/1\x99\xc8M\xbb}\x9d\xae\xaa\xed1\x81s\x94\xa3\xe3\xa8\xce\x1a\x05m3\xd0!E\x14\xf0\xac\xb81\xd6\x02\xb4\xabk\xc0X\x9d\x9fe\xa0\xad\x16\x1c\xb9\x99|\xca\x16\xd1\xef\xd6[،\xef\xf6u\x13\x12u\xe31g\xc3ONZ\xb6l\x89\x04#\xd3rgo\xb8\xae\xfaZV&\a&\xf6V\xa8\x94,\xf7\x87J.Gl\xf9\bܴ\xb4HAA\x1aBW\x95\r\xa6T\xa2\xb5\xfbQ\x1f\xb1j\xd0e\xc9\xd3(\xa6~\xf7\xb6\xfaZލ\xbfk|\xb5S2_y^PAµߑP\\ڀ\xdd\x1c\x82$\a\xf7\x19\"\x7f\xa9/\x89AQ\xa0\x00\xa6=>\x11w1L\xb3u\"\x01\xa9\rS&6\x0ez\xe84\x9e\t\x81\br\x18\xdf\a\xbf\xe3\xe2\ue938\xf3ߢ\xaa\x01_\x83\xe6\xa2\xfaP\x9f\xdb\xcfq\xa2\xa0md\xa4\x90\x92k\xc1Z\x93ALӉ`\xba\xe8\xff\xbe\xc1˱\xb6\x89\x1fc\xbc\u0bfd\xe6\xbdsG\xf4=\xaa\xba\x89\xf7\\\x03\xf4\xf8\x03߹\xf2\x97\xc4b\xfd\xc7\xff\xf7\xf3D\xc7(/\xeb\xed\xa4\x83E\xbeS\xed)\xcd|}\xea>C\xeb\xf9hĮ\xef\xf6v\x91\x93~\xfdy\xf3\xf6_7\x7f~E\x88\xa0\x05ܒ\x1d͞\xaaRoN\xc0A\xc9\r\x93\xaft\t\x99\x05yP\xb2*oI\xf3\x83\xeb\xe2\x87s\xa8\xfe\x05{\xe3\vδ\xf9\xa9\xf5\xf2g\xa6\r\xfeP\xf2JQ^\x8f\x84\xef4\x13\x87\x8aS\x15\u07be\"Dg\xb2\x84[\xf2\xc9\x0eQ\xd2\f\xf2W\x84x\xacqȵG\xf8\xf4\xd6AȎPP\x87\v!\xb2\x04\xf1\xee~\xfb\xe5\xdf\x1e:\xaf\t\xc9Ag\x8a\x95\x06\xe7\xee\x10#L\x13J\xbeഈ\xf2T&\xe6H\rQP*\xd0 \x8c&\xe6\b$\xa3\xa5\xa9\x14\x10\xb9'?U;P\x02\f\xe8\x1a4!\x19\xaf\xb4\x01E\xb4\xa1\x06\b5\x84\x92R2a\b\x13İ\x02\xc8\x1f\xde\xddo\x89\xdc\xfd\x06\x99ф\x8a\x9cP\xadeƨ\x81\x9c\x9c$\xaf\np}\xff\xb8\xa9\xa1\x96J\x96\xa0\f\vtvOKxZo{\xd3{m)\xe0Z\x91\xdcJ\r\xb8ix*B\xee\x89f\xe7c\x8eL7\xd3E9\xea\x00&\xb6\x11\x15\x1e\xf9\ry\x00e\xc1\x10}\x94\x15ϭ\xb0\x9d@Y\x82e\xf2 \xd8?jؚ\x18\x89\x83rj\xc0\v@\xf30a@\t\xcaɉ\xf2\nn\x90$\x05=\x13\x05v\x14R\x89\x16\x93\x96r\x88\xfe؛G\xc3H+\xa98\xf31ԭ#зh᱈Z\x0f\x17-g\xceN,\xaf(G#JE\xe6\xe6Ck\xbcb\xc6x\x82\xc9\x03\x9c\x9d\x89\x0e\x98[Nt\xc2\x11)\xc0\xba\xa0\x85\x8d\xc1\x86McF\xc6=c\xd3\xdeQ\xebgH'\xa2\xaa\xe2\xa0\xfdPαkt\xc0\xcd(\xe8\x9a#.~\xe7t\a\x9ch\xe0\x90\x19\xa9\xe2\xe4\x98c\xb2{R\xf4\xda\b\x15#\x1a\xae\x1b\x104\x13\x9b\x00I0\xa8:\xb2\xec\xe8\xdc4+A\b\x87\xe4\x124\xaerZ\x96\x055\xd9\xf1\xc3W\xeby\xe9&\x97\x93H\x97~g\xe7\xbf\x06\x7f\xbek\x98g\xe0\x12\xdc\x14f\n\n\xb7\xd9\xfc\x88\xd4lޠG\xf5\xee\xd3\xfb\xd8nV\xf7I\x90\xbc\xc1D\xde\xf5\x90m\x0f\xed\x9d\xf2\xd4ixק\x8eo\\\x1a\xe1\x86P\xf2\x04g\xe7\xb1PA,s\xa8\x1dh$\xd2\x19\x12\a\xf3\x19(dOpF0>A1\xdb;U\x14\xdc\xf3\x04\xe7\x94f=\x02Z\x9c\x98\xf6\x89\x17KI\xfb\x02\t\x81\xfb\xd9\xe9\xc4#\x98l\n\xbah~r$]\x91\x84'\xd0\xfe\x82i\xd6lk%ꐱ\xaf\xb5c\x91]\x05GV&NԚ9\xdcJ\x90\xfb:\xdd\xf4\x85r\x96\xd7\x039\xb9ߊqo\xb8\xfb|\x92f+nȇ\xafL\xfb\x8c\xdf{\t\xfa\x934\xf8\xe6E\xc8\xe9\x10\xbf\x80\x98\xae#./\xe1Զ\xa5C;o\x95 \xdc\xeeٺ\b\xaff\x0f\xd3d+l\xdc\xe2\xe9\x81YH7ܴ}\xe8>E\xa511%\xa4X\xbb\xad\x97\xd8H\x8e؉ \xa5\xeapd\x88Z=\xe8\xc8^O\xfcy\xb4\x96\xc4\xf5wyUN3\xc8C^\x05\xb3\x81\xd4\xc0\x81e\xa4\x00u\x982\x1c\xed\xa7\xb4\xfa=\r\x85D\xad랅\x12\x96f\xda\xc3\xe3Uwt\xf3\xbb\xfb\xac\xed\xcaMh\x15\x98=\xdbt$\t8\xdet~Fhb\xd1\xff\x98\xa5.\xcds,Ӡ\xfc~\x81\xc6_\xc0\x8b\xa1\xedw\x889\vYPLN\xfc\xb75s(\xd0\xffCJ\xcaT\xc2\x1a~\x87\xe5\x18\x1c:}\xfd.V{\x18;\x02\xd3\xc4\xf2\xf7D\xf90\xbd\x1c\x99\x9c\xb4\xba\x05\xb83\xe4r?\xf0Xn\xc8\xf3QjgS1)2\v\x92i\xb2z\x82\xb3OƵ\xf5\xc0j+V\xce\xc0/V7\xb5\xb7 \x05?\x93\x15\xf6]}\x8b\x13\x94(\x89\x89;\xae\x9f\xea\xf2\x93uA˵\x97^#\v\x96\x8d\xf6\xc3r\x99T\x17\xdbƠ\xc1\x83\xb0\x1d\xeb\x1a\x11\xeb\x1eO\xcd6I~K\xa9#\x99\xef\x11T\xee\xa56nG\xb2\xe3\xce.\xd9\xfd\"N\xf6\xfc\xae\x17\xa1{W\xa5#U\xa8\xbf\xb0겷Qk\xb9\xad\xa75\xb3\xcb\x19\xf8\x9d4\a\xd4\x06d\xabf\xe5;=\xbcr9\v\x1c\x84f\xe8\x94\xcc\xc2-\x95\xcc@G\xb3\xc5͓\xa0\xe5g6\x17\xeb\x8dE\xea\x02\x1fW\xdc0\xbd\x99\x19\x9etG\xd6\x12ia\b\xf0\xe1kk\xd7\xd3*\r\xfb\xf7\x9c\xf0-ŋ\xe0Z/\nگ\xe2IB\xf1\xce\xf5\f\xcb\xc4\x03r!\x85:T\xa8\"\xd2=O/H߃y/\x98\xd8\xe2\x00\xe4\xed\xd5݁Z\xb9\xc6j9bO\x8f\xe4\xbeoC\xf4\xfa\x85\x18)\xe6\x88=\xa5\xc4\x1d\x7f\x05\x1d\xce\r\xf7ǭ\x83\x99\bRH\xd3ކ\xb0pK\x99\xbf\xd6dϔ6mDS\x85\"^+\x12{\x96F\\\xe2\x83R\x17\x05\\\xbf\xba\x9e\xad\r\xb0\xa3|\x0e\xb5P\xa3\xc5\x13\xb1\a\x93I@؞0C@d\xb2\x12\xb8mc\x97:\x0e\xe1X\xe0\x14t2\xc9\xd2\x14\x84}@TE\x1a\x01\xd6(uLL\xeeﴛ\x7f\xa4,\x96\x81\x1e>\v\xd9f\xc6J\xc6bO\x87m\xa1v\xac]\xd4VЯ\xac\xa8\nB\vK\xfa\xd4pi\xef*\xce:\x1c\xaf\xeb\xce\x10.\x9a\x11#\xed\xa2*9\x98\xd4\x15\xe9*\xcc\xec2\xd1,\x87\xda0{)\x90\x82P\xb2\xa7\x8c\x8f\x94\xbb\f\x9fE\xb4]\x12\xa3xeq\xbd\xe0#m\xf05\x92\"a\x037\xd1ɜ\xd6֥Jw\x15\xef\x15\xa4\xb9gs\x9b\xd9\xc1=+\x15\x93X\xa7we\x0f͋\x18\x15\xe7\x1f.\xda\xe0\xf9\xe1\xa2\xcd3ś\xdfP/\x1f\x06\xbd\xa8^~;\xd9\xf9J\xf5\xf2\x1eþ\xd7}\xa5j\xf90\xffe\xd5\xf27\xbeT\xa3\x00\x1a\xb6\xe7].>\x1f\x1b\xb27\xda\x00\xf0\xef\\\x7f;\xa8\x0f\xbb\x8c\xf1/^m?\xc2\xfc\xc4\xc2\xf8՟V\xdf\x1f\xa5\x17\xd3v\x94\x9a\x032E&\x15\x8e\xf9ڸ\xb2]\xdc\xd5-\xa4\xfb>\x85s\xa94\xa6V\xccO\xd1k\xa8eZ\x04\xfb^\x17\xb3\x81\xe2\xd7\xd2ۊ\xb4\x13\x9d\xdbH\x97\xb93\x9d\x91\xf9\xe0&\x80>\x8b쨤\x90\x95\xf6\xfb\x06\x16\xfa;ܾ\xf0\xa9P,\x03KT\xb0o\xc9QV\x91\x8a\xed\t\xda\xcd\xd4\xef\x8dW\xed\xf9\x1c5\x18zz\xbb\xe9\xfeb\xa4\xaf\xe1#\xcf\xcc\x1c#x>\x1fA`v]\x1c\xda\x05\xf9a\xc1\xf9\x03\xe7}A\"R\x11\xc1\xf8\x98\xc1\xaaO\xe5wLӯ\xa5\xdb$Zl\xf9\xa778Ҫ\xfc.\xae\xed\xeb\xd6\xee\x8d8\x81K\x93\xd9\xe9G\x18ҫ\xf7\xa6\xcb\xed\x96\xd4\xec\xf5+\xf2F\x81\xceW\xea\xa5\xecM\xcdT\xe5]P\x8b\x97X\x87\xfdͩ\xf7\x94j\xbb\x8bj\xecfK\x95\x13+\xeb\xba5s\xd3 \x17\xd4\xd3%\x11g\xbevnqŜ\xafP\x9b\x9cGr\x9d\\\xa4\x02n\x12\xf0hu\xdcT\xdd\xdb̾\xf7\xb0&.\xbd\xdam\x124V\xc2\xcd\u05f8]\xaf\x92\xfd\x1aQ\xf6\xb8\xaa\x99\xadS\x9b\x8d§\xf1\x9b\xadD[R\x7f6K\xb1\vk\xcd\xeaZ\xb2\x91q\x97V\x98u+\xc8F\x80\xa6ԕ\x8dԍ\x8d@\x9c\xac&K\xad\x16\x1b\x81=cv'\xa5d\xf2\xc7%Ub\xf1KTȬ5俗\xfc]J\x06\xa9:\xce\xe5\\@\xf3k\xaf\xb9\xe5|\U0003199d\u0558\x9f\xca\xccq\xb9\xb3ZTܰ\x92cz\xf1\xc4\xf2h\xccn\x8ep\xae/\x86\xf8M\xe2qMw\x99\t\xf9\xf5s-̛\x9e\xcbM5y\x06\xce\t\x8d\x89\xe2`晻\a(\x93k\xb0\x16\xc2.O\x7f兿.\xe8\xc6\xc9;\x9eH\x8de`\xcc\x11\n\ve\xfcړQU>\xedN:\xcf\x17\xdf\xfd\xbd\x02u&x=K\xed_̜\x87r\xcbR\xdbX((\n\xafm\xdc\xedS=7\xbbY\x9e\xe4\x9dp\x06/\n\xb6\x87#±\x1a\x82\u05fc\xb6\xca\xd0F\r#M\xe3\x1b\xb1\xb2\xee\x1d\xf9}\xceSM=L\xf4\xb2\x81\xc6\xf2Pc\xd6ȿH\xb8qy\xc01\x012\xf5pPZB|\xf60\xd0K\x05\x1es\xa1G\xb2ϕv\xd8\xe7%\x0e\xf9,8ܳ \x04Y\x16\x84$\x93)\xe5\x10ϋ\x84\"/\x18\x8c\xbcD8rY@2\x03\xb2w8'\xe5\xd8MR\xb1Gr\xbe3\xa5Xc>%9}\x9c&\xe1\x18MB\xb2r\x0eӄ\xe32ˎ\xc9$\xd0\xf0\x85B\x95\x17\nV^\"\\yـe6d\x99\x95\x9c\x99\x9f\x97\x1do\xb9x\xf3^\xaa\x1c\xd4d\xae#U4'\x85\xb2\x17_t\xc7\xec\xed\xfc\x87;\xe5l\xab\x8e+\x1b۰\xaeO\xbdg\xe4'&|\x1e\xd5\na\xcb\xeew\x120\x8d#\x12\xdf\xffo\xbc<\x7fۨ\xcb\xdah(\xa9\xc2\f\xeb\xee\xecJ+\xf4\x86|\xa0ٱ\a\xfd\x18\x8d+\xf6R\x15ԐU\x9d\xf2z\xe3\x80ۿW\x1bB>\xca:i߾IF\xb3\xa2\xe4g\x1b7D`\xae\xda .\x13\x88\xa8\xf0\x85\xf1\xef%gY\xc4ӊ^.\xe4\x1a\x0f\xae\x84\xc0+\x8f\xb2v껴\r\xe3\x8e\x16:e\xdd\xeb\x15\xf7\x92s\xf9\xbc0\x1c\xa7%\xfb\x0f\xbc\xa5y~\x0f\xe7\xdd\xfd\x16\x9b\x06I\xc1\u06dd\xeb\n\xa1\x1a\xe9\x1dX\x8b\xd9Lgl\xc5o\xf7\x1d\x88\x91J\xbb\xfaO\x94\xd6\xdab\xb3\xb1[\x97\\՟\xd54\xf7[\x87\xdd\x06\x85\x85\x8a3\x91X\xeba\x8eL\xe5\xeb\x92*svu\x0575\x0e\xe3\xfb8\xc1nN\xed\xb6\x8c\x9a\x97\xe1u\xbfQچ[\x7f1\x99w.\xbb\xa9\xd0>E/\xc1c\xfc(\xdf\xec!\xbe+\xe21\ue0ac\x91R\x91\xd7Ѣ\xa4\xab\xedbi\x7f\xb5\xed/\xf2\x04\uf8fbY\x1d\xf2<\xf4\x9aGʉ\x02Dw\xb9\xeb\xd4\xed\xa0x9\xe7e\xba(^\x1f\x14\x86\xf6\xd7w&\xceŷ\x8eL%\xdc\\\x1a\xe0\xea\xf8\xae\x8d]^\xf7_0\xb4\xaaU\x98wv|\xf0\x14\xb6\xae\xfa\x17\xdc\xfd\xe5\xfa5R\xdaHE\x0f\xf0\xb3tW/\xcfѠۺs\xef\xb6wyB\xcdbX\r\xb1P\xc0_\x02\xdd\x03֔\"\x0f\xae\xc1\xb5X.\xbc\xd5\xd7\x18>3\x99\xc7ǟ\xdd\x04\f+`\xf3\xber\xb9|\xab\xed4Xj\x86\x89\xb9N;\xfb\xdfc\xc4^\x10\xbcO\xb6ş\x16\xde\n\xb0\xdc\x19\xcb\xde\x16a\x7f\xea\\$\x1dH4'\xa2_\xe2\xbdZ\xfbK-&9\xd7#*\xa1cpZw\xe9\xe3\xce+\x1eW\xbe\ueb4bc\xce\xe4\xd8m\xe3x\xc3\xf6\xfc}\xe3\xee\"n\xffu\x01_4_)\xbc2\xd1_ҍW\f^t\xe5\xf8\xae\xae\v\xa9\xabN\xf4;cl\xa0\x1c\xd3\xdc\x11\xf4F\xfa\xd6\x06N\x1aʉ\xa8\x8a\x1d\xfa\xac1\x95Rw\xc1\x8a\x95\xc9R\x15\xe7\x80L0Α\x9a\t\x03\x87\xc1\xa6{l\xaew\xbe\xc6\xf9\x92\xb9\xd6}\xd3窫,\x03\xad\xf7\x15\xe7纾z\xc9\xc4c\xd6\xe5J\xa4\xf8H\x19\xbf\x88\x0e\xae\xe3\b\x11\xdc\xdcF\xf5h\x12\x9b}Q'\x88<,ށ)\xb0\x0f\x9e:XF\a\xcf\x02_k\xa5\r-\xe6nN\xbf\x1b\xf6\xc0\xcfZ\xa8\xbcU\x9dU_\xff\xfdLu\xc3\xe6\x98Oـs=\xd1\x05\xb5\xd0 'p\x02A\xa4\xc0\xaay\xbc\x8b\xd3}z\xa5\xdf'\x02\xb5\rŗ\xe5W%\x974\x0f\x06.D\x92\xfes\x1d\x8fh\xbe\xd5\t\xd4k=\x01\xb3\xbe\xd0=B\x84\xa1d\xba\xd0\xee\xd6\xfaF\xb0\x8e\x02M2\xfdQ]\x9bi\xd6\xd5\xf3\xc9J\xeb\xeea;\xd6sT\x82C\x83\x18\xff\x06\x1fN\xf8F%5\x9cY\xaa\x8a\x1a\xcelNAu\xd4Qdr\x8d\x82\xba\xfa4q\xad\xce\u07b8\x8c\x8d\x9c\a\x80Nj\xc2u\xfa\xee|Q\x01Z\xd3C\xb8j\xf9\xd9:`\a\x10\x80\x9b\x11\x91\xd9\xf8\xed\xdd\xe6\x027\xaf,R\xa4D\r\xa1C\x99\x86\xa9\x94h\xa5r\xea\xf3b\r\xba4{\x1a\xc5ԧ\xa2\xc3G\x0f\xdf\xf8\x8b\xd3\xd7{%\x8b\xb5\xe7\x05VW\xdc\xf8\xf4\x8ab\xd2\x06\xec\xe6\x18%9q\xdfT\xf27\x14\xa3\x18\x94%\bB\xb5\xc7'\xe1b\x89i\xb6N\xec\xa6jC\x95I\x8d\x83\x1e:\x8dgB \x84\x1c\xc7\xf7\xc1\xa7\x8f\xdc\x05\x1bw\xfe\x93b5\xe0\x1b\xa2\x99\b\xdf[t\xc9)'\n\xdaFF\nps-Z83\x88i:\x11L\x17\xfd\xdf7x9\xd56\xf1C\x8a\x17\xfc\xa5\u05fcw\x88\n?\xaeU7\xf1\x9ek\x84\x1e\x7f`{W˓Y\xac\xff\xf8\x7f~8\xea\x94\xe4e\xbd\x9et\xb0\xd0w\xaa=\xa5\x99Oi\xdds\xb0\x9e\x8f\x06\xe8\xfan\xaf\x179\xe9\xa7\xcb\xc2\xcekƜ\xe1S\xa0\u05c9\xc4N\x97E\x9b/\x16j^wv\xcf\x14??8\xb7\xc6\xfe\xe6\x9bEbM\x0f!\x12mF\xa6Qǟ\xb3\xd1f+\xd8\f8\x8e|-\xa8\x17\x80^)܌ځ\xc1KT\xa0ykm\xfb\x91\xfc\x9b\xff\r\x00\x00\xff\xff\x9a\xfbL\xe1\xa9x\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4YKs\xe3\xb8\x11\xbe\xebWt\xed\x1e|YQ3\x9bKJ\x97\x94FN\xaa\xa6\xe2\x89]#ǹ\xe4\xb0\x10Д\xb0\x06\x01\x06\x0fi\x94T\xfe{\xaa\x01\xf0!\x92\xb2\xe4\xa9$\x8b\x8bM\xb2\xd1\xe8\xfe\xfa\r\xcd\xe7\xf3\x19\xab\xe5\vZ'\x8d^\x02\xab%~\xf3\xa8\xe9\xc9\x15\xaf\xbfw\x854\x8b\xc3\xc7٫\xd4b\t\xeb༩\xbe\xa23\xc1r\xbc\xc7Rj\xe9\xa5ѳ\n=\x13̳\xe5\f\x80im<\xa3\u05ce\x1e\x01\xb8\xd1\xde\x1a\xa5\xd0\xcew\xa8\x8bװ\xc5m\x90J\xa0\x8d̛\xa3\x0f\x1f\x8a\x8f?\x17\x1ff\x00\x9aU\xb8\x84-㯡v\xdeX\xb6CexbY\x1cP\xa15\x8543W#\xa7\x13vքz\t݇\xc4!\x9f\x9e$\xff\x14\x99m\x12\xb3\x87\xcc,~W\xd2\xf9?_\xa6y\x90\xceG\xbaZ\x05\xcb\xd4%\xb1\"\x89\xdb\x1b\xeb\xff\xd2\x1d=\x87\xadS\xe9\x8bԻ\xa0\x98\xbd\xb0}\x06ษq\tqw\xcd8\x8a\x19@\x86&r\x9b\x03\x13\"\x82\xcdԓ\x95ڣ]\x1b\x15*ݞ%\xd0q+k\x1f\xc1L\xba@V\x06\x1am\xc0y\xe6\x83\x03\x17\xf8\x1e\x98\x83ՁIŶ\n\x17\x7fլ\xf9?\xf2\x03\xf8\xd5\x19\xfd\xc4\xfc~\tE\xdaU\xd4{暯\xc9FO\xbd7\xfeD\n8o\xa5\xdeM\x89\xf4\xc0\x9c\x7faJ\x8a(ɳ\xac\x10\xa4\x03\xbfGP\xccy\xf0\xf4\x82\x9e\x12B@\x10!4\b\xc1\x91\xb9|\x0e\xc0!q\x89\x18MK\xaaFg\x9d\x89M\xa2\xc0ˀK\x92\x9f\xded\xe9{l\x1b\xff.\xb8Ŗ\xa5\xf3\xac\xaa\xcf\xf8\xaevx\x89\xd9\x19\x14\xf7X\xb2\xa0|_U\xb2\x92\xea\xfb\xe5\xb9Z5\xf2B\xa4]g'ޟ\xbdK\xa7n\x8dQ\xc8\x12\x97Du\xf8\x98\xbc\x90\xef\xb1b\xcbLljԫ\xa7\xcf/\xbfۜ\xbd\x86)G\x1a\x04\x05\x19\x8e\xf5l\xb3G\x8b\xf0\x12\xe3/\xd9\xcde\xd5Z\x9e\x00f\xfb+r\xdf\x19\xb1\xb6\xa6F\xebe\x13,i\xf5rQ\xef\xed@\xa6;\x12;Q\x81\xa0$\x84ɏr\xbc\xa0Ț\x82)\xc1\xef\xa5\x03\x8b\xb5E\x87\xda\xf7\xe1m\x05+\x81\xe9,^\x01\x1b\xb4Ćb9(A\xb9\xeb\x80փEnvZ\xfe\xb3\xe5\xed\xc0\x9b\xec\xbc\x1e\x9d\x1f\xf0\x8c\xf1\xa9\x99\"W\r\xf8\x130-\xa0b'\xb0H\xa7@\xd0=~\x91\xc4\x15\xf0\x85\xfc]\xea\xd2,a\xef}햋\xc5N\xfa&\asSUAK\x7fZ\xc4t*\xb7\xc1\x1b\xeb\x16\x02\x0f\xa8\x16N\xee\xe6\xcc\xf2\xbd\xf4\xc8}\xb0\xb8`\xb5\x9cG\xd1uJ\x9a\x95\xf8\xd1\xe6\xac\xed\xee\xced\x1dEmZ1k\xbea\x01ʘ\xc9\v\xd2֤E\a4\xbd\"t\xbe\xfeq\xf3\f\xcd\xd1\xd1\x18C\xf4#\xee\xddFי\x80\x00\x93\xbaD\x9b\x8cXZSE\x9e\xa8Em\xa4\xf6\xf1\x81+\x89z\b\xbf\v\xdbJz\xb2\xfb?\x02:O\xb6*`\x1d\v\x13l\x11B\x1d㾀\xcf\x1a֬B\xb5f\x0e\xff\xe7\x06 \xa4ݜ\x80\xbd\xcd\x04\xfd\x9a:$N\xa8\xf5>4\xb5\xf0\x82\xbd&\xa3xS#?\x8b\x1f\x81NZ\xf2p\xcf<Ƹ\x18\xe0\x9aC\xfcr1m\xd6tp\xd3b\x9c\xa3s_\x8c\xc0ᗁȫ\x96\xf0L\xc6\x1am%],\x8bP\x1a;\xac\x18\xac\xcd\xc0\xfd\xd5d\xaab\xf4\ru\xa8Ƃ\xcc\xe1+2\xf1\xa8\xd5\xe9§\xbfY\xe9\xc7\a]0$\xad$\xe2\xe6\xa4\xf9\x13Zi\xc4\x15\xe5?\r\xc8[\b\xf6\xe6\betk\xedՉr\x90;i>ζ\xcdZ=}n2o\n\xa0\x1co\x19\xab\x02V9rM\t\x1f@HG\r\x80\x8bL\xc7`\xe9\xa0b\x83\xb0\x04oû\xd4\xe7F\x97r7V\xba\xdf\xd3\\\xf2\x98+\xac\aȭ\xe3I\x94\x9a\xc8;jk\x0eR\xa0\x9dS|\xc8R\xf2,I\xb0\xa9r\x95\x12\x95pcM/DYTŢ\xa0\xa8f\xea\x8a\r\xd7-a쀙\xd4Ƀ;\x061\xd9\xd8*\x97T\xedQ\x8b\xb6\x1b9\x93\xc6Ĭ\xe5P\xc0Q\xfa}J\x87j*\xee\xe0\xcdأ\xf5\x8a\xa7\xa9\xd7\x03ٟ\xf7H\x94\xa9\x80\"8\xe4\x16}\xf46T\xe4>\xe4J\x05\xc0\x97\xe0bB\x1d\xe6\x89f\xc5F\xad\xd9\xfd\x8a\xa71\xd0p\u0378\xb9\x85\xb9.\xf2\x1d\xb5\u038d\xc0\x16K\xb4\xa8\xfddR\xa7\x01\xc4j\xf4\x18\xf3\xba0\xdcQJ\xe7X{\xb70\a\xb4\a\x89\xc7\xc5\xd1\xd8W\xa9ws\x02|\x9e#h\x11NJŏ\xf1\xcf\x05\x95\x9f\x1f\xef\x1f\x97\xb0\x12\x02\x8cߣ%\xab\x95A5\x8e\xd6\xebo~\x8a5\xf6'\bR\xfc\xe1\xee{p1u\x8a\x9c\x1b\xb0\xd9D\xef?Q\xa3\x16\x85\"\x886\xc9*\xc6\x02UJ2v\x95\xad\x99r͔#Nu\x98\xfdE\x89\x89*\xc8TF}\xc5q2}#\xcc\x00\xbe\xcd;C\xcd+V\xcf\x135\xf3\xa6\x92|6\xd46\xb6\xc1W\"\xb2i\xbb\xa5\x16\x92S\xdbv\x1eI\xcd8\"κ\xf3\t\x18\x86\xfd\xfa\xa5\xfc1\rSR7W\xcf+\x12?\xf6i\xbb!.%\xb3\\\x11\x1dzj\xb7\x1ch\xa4\x8a\xc9\xec\x18\xe7\x98B\xb8њb\xd7\x1b`mb\xbcsÊ\xf0\xce|\xb2\r\xfc\x15'\x80\x1f\xa9\xf2)\x126\x18\xa7m$Kp\x18S\xf551\xe0zDp\xb6F{\x8b,\xeb\x15\x11\xb6E\x95\xc1z\x05۠\x85\xc2F\xa2\xe3\x1e5\xcd\x13\xb2\x14?\xfcf3\x93b\xce\xd3\b\x84\xe2+\x1e\xe4\xf8Nh\x8c\xee\xc3hG\x13\xf8m8\xd0\xc3/\xcdh\xbd\xb0\x99\xec\x97\t0J\xa9\xa8s\x9c\xc8\x13]\xc70\xbe\xbd\xfc\xb4y\xb8s\xb1\xe1G\xed\xa7\x9a\xc4#Z\x8c\xf3\x15\n\xea\xf9M\xbe\xc5\bΣ\x9dp\x80\xd6z\xd1栌\xde\r\x02'\xad|\xa7A\xfd\\r(cA\xa0\xa7Ҥw\xc0\xf7Lﰻ\xb3\xca\xf2\xbf-)\xb9\xcf\xc0g:\x0f\x91\xfa\x92{\xdcd\xd1g9\xd5ԏ\xee\x8b;\xe2\xe9\xbb\xe2F\xfaƲ\x17\x87\xa2+\xb8\x8f\xe8\x9b*M\xa0\xce}w\x7fܭ\xef\x1f\x86Ǘ\xd37 \xf1ޛ\xf37nA\xe0\xc8\\w\x87\xfe\xdb\xe1PQ\xb7z\xb5\x05\xfe\x92\xa8\xd2ec\xde\x02lk\x82\x7f+2\xef\xa6\x1c:\xff8\xf0\x1e\x19\xe3O\x1eך\f\xa2i,\u0083\xa5\xc1\xb3\xbbC\x8bIa\xaa\xb6\xdc~\x19\xb5\x1a\xfc2\xd3\xff6\xfe\xdd\xe6\x06\xbd&k\xed\xe8e\xaa\x97=\xbbf\x90\xfbo¶\xbdW^¿\xfe=\xfbO\x00\x00\x00\xff\xff\x80.\x12\xd3P\x1c\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4\x96M\x93\xdb6\x0f\xc7\xef\xfe\x14\x98y\x0e\xb9<\x92\xb3\xed\xa5\xa3[\xb3\xc9a\xa7mƳ\x9bɝ&a\x8bY\x8ad\x01\xd0[\xb7\xd3\xef\xde!)\xf9E\xb67\xdbCy\x13\t\x02\x7f\xfe@\x80j\x9af\xa1\xa2\xfd\x8a\xc46\xf8\x0eT\xb4\xf8\x87\xa0\xcf_\xdc>\xffĭ\r\xcb\xdd\xdd\xe2\xd9z\xd3\xc1}b\t\xc3#rH\xa4\xf1#n\xac\xb7b\x83_\f(\xca(Q\xdd\x02@y\x1fD\xe5iΟ\x00:x\xa1\xe0\x1cR\xb3E\xdf>\xa75\xae\x93u\x06\xa98\x9fB\xef\u07b7w?\xb4\xef\x17\x00^\r\u0601A\x87\x82k\xa5\x9fS$\xfc=!\v\xb7;tH\xa1\xb5a\xc1\x11u\xf6\xbf\xa5\x90b\aDž\xba\x7f\x8c]u\x7f,\xae>\x14W\x8f\xd5UYu\x96\xe5\x97[\x16\xbf\xda\xd1*\xbaD\xca]\x17T\f\xd8\xfamr\x8a\xae\x9a,\x00X\x87\x88\x1d|β\xa2\xd2h\x16\x00㱋\xcc\x06\x941\x05\xa4r+\xb2^\x90\xee\x83K\xc3\x04\xb0\x01\x83\xac\xc9F)\xa0\xbe\xf4X\x8e\ba\x03\xd2#\xd4p \x01\xd68*0e\x1f\xc07\x0e~\xa5\xa4\xef\xa0ͼ\xdaj\x9a\x85\x8c\x06\x15\xf5\x87\xf9\xb4\xec\xb3`\x16\xb2~{K\x02\x8b\x92ē\x88\x12\xd7\x06\x0ft\xc2\xf7\\@\xb1oc\xaf\xf8<\xfaSY\xb8\x15\xb9\xda\xec\xee*i\xdd㠺\xd16D\xf4?\xaf\x1e\xbe\xfe\xf8t6\r\xe7Z\xaf\xa4\x16,\x83\x9a\x94fp\x95\x1a\x04\x8f\x10\b\x86@\x13Un\x0fN#\x85\x88$v\xbaZu\x9c\x14\xcf\xc9\xecL»\xac\xb2Z\x81\xc9U\x83\\\xa0\x8d\x97\x00\xcdx\xb0\n\xd32\x10FBF_\xeb\xe8\xcc1d#\xe5!\xac\xbf\xa1\x96\x16\x9e\x90\xb2\x1b\xe0>$gr\xb1\xed\x90\x04\bu\xd8z\xfb\xe7\xc17\xe7s\xe6\xa0N\xc91?\xd3(\x97\xce+\a;\xe5\x12\xfe\x1f\x9470\xa8=\x10\xe6(\x90\xfc\x89\xbfb\xc2-\xfc\x961Y\xbf\t\x1d\xf4\"\x91\xbb\xe5rkej\x1a:\fC\xf2V\xf6\xcbR\xffv\x9d$\x10/\r\xee\xd0-\xd9n\x1bE\xba\xb7\x82Z\x12\xe1RE\xdb\x14\xe9\xbe4\x8ev0\xff\xa3\xb1\xcd\xf0\xbb3\xad\x17\x17\xa4\x8eR\xe8\xafd \x97yM{\xddZOq\x04\x9d\xa72\x9d\xc7OO_`\n]\x921\xa7_\xb8\x1f7\xf21\x05\x19\x98\xf5\x1b\xa4\x9a\xc4\r\x85\xa1\xf8Dob\xb0^ʇv\x16\xfd\x1c?\xa7\xf5`\x85\xa7+\x99s\xd5\xc2}餹\xa8S4Jд\xf0\xe0\xe1^\r\xe8\xee\x15\xe3\x7f\x9e\x80L\x9a\x9b\f\xf6m)8}\x04\xe6ƕ\xda\xc9\xc2Ծo\xe4\xebJ\xd1>E\xd49\x83\x19b\xdem7V\x97\xf2\x80M x\xe9\xad\ue9e2\x9d\xd1=\x14x{\xb6p\xbd\xa0\xf38\xb6\xc9\xf9\xca\xcd\xc3Cɝ%\x9c\xdd\xc2\x06.z\xee\xeb\\J3\xfc\x97dj'\x1e\xd9\xe8D\x84^N\xfa\xb3\xba\xb6\xe9\xad,\x90(\xd0\xc5\xecLԧbT^ze=\x83\xf2\xfbq#H\xaf\x04^\x90r\x19\xe8\x90r\x9fA\x03&]\xf0\x1b\xb1\x9c\xbe%\x91\x82F\xe6\xf6\xc2\xce\n\x0eW4\xbd\x92\x9d<|rN\xad\x1dv \x94\xf0Ff\x15\x91\xda\xcf\xd6ʛ\xf5\x1d\x04\xabls-\a\x87w\xfa\xbbI(\xb8}\x1a.#5\xf0\x19_\xae\xcc>\xf8\x15\x85-!ϯ|^\\Uz\x87\x9f\x817P\xbaz)/&9\xf7;sB\x91%\x90ڞr\xe5\xb4>\xf4\xef\x0e\xfe\xfa{\xf1O\x00\x00\x00\xff\xff\x045\f\xc6i\n\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4VOs۶\x13\xbd\xebS\xec\xcc\xef\x90_gB*i/\x1d\xddZ%\aO\xd3\xd4c\xa5\xbeC\xe0\x8aD\r\x02(v!\xc7\xfd\xf4\x9d\x05H\xfd\xa5d\xf9P\xde\b,\x16\x0fo\xdf>\xa0\xaa\xaa\x99\n\xe6\x11#\x19\xef\x16\xa0\x82\xc1\xef\x8cN\xfe\xa8~\xfa\x99j\xe3\xe7ۏ\xb3'\xe3\x9a\x05,\x13\xb1\xef\x1f\x90|\x8a\x1a?\xe1\xc68\xc3ƻY\x8f\xac\x1a\xc5j1\x03P\xceyV2L\xf2\v\xa0\xbd\xe3\xe8\xad\xc5X\xb5\xe8꧴\xc6u2\xb6\xc1\x98\x93\x8f[o?\xd4\x1f\x7f\xac?\xcc\x00\x9c\xeaq\x01\x8d\x7fv֫&\xe2\xdf\t\x89\xa9ޢ\xc5\xe8k\xe3g\x14PK\xee6\xfa\x14\x16\xb0\x9f(k\x87}\v\xe6OC\x9a\x87\x92&\xcfXC\xfc\xdb\xd4\xec\x173D\x04\x9b\xa2\xb2\xe7 \xf2$\x19\xd7&\xab\xe2\xd9\xf4\f\x80\xb4\x0f\xb8\x80\xaf\x02#(\x8d\xcd\f`8b\x86U\r\xa7\xdb~,\xa9t\x87\xbd*x\x01|@\xf7\xcb\xfd\xdd\xe3O\xab\xa3a\x80\x06IG\x138\x13u\x82\x19\f\x81\x82\x01\x01\xb0߁\x02\xe5@E6\x1b\xa5\x196\xd1\xf7\xb0V\xfa)\x85]V\x00\xbf\xfe\v5\x03\xb1\x8f\xaa\xc5\xf7@Iw\xa0$_\t\x05\xeb[\xd8\x18\x8b\xf5nQ\x88>`d3\xb2\\\xbe\x03\r\x1d\x8c\x9e\x00\x7f'g+QЈx\x90\x80;\x1c\xf9\xc1f\xa0\x03\xfc\x06\xb83\x04\x11CDBW\xe4t\x94\x18$H\xb9\xe1\x045\xac0J\x1a\xa0\xce'ۈ\xe6\xb6\x18\x19\"j\xdf:\xf3\xcf.7\tC\xb2\xa9U<\xcaa\xff\x19\xc7\x18\x9d\xb2\xb0U6\xe1{P\xae\x81^\xbd@\xc4\xccSr\a\xf9r\b\xd5\xf0\xbb\x8f\b\xc6m\xfc\x02:\xe6@\x8b\xf9\xbc5<\xf6\x8e\xf6}\x9f\x9c\xe1\x97yn\x03\xb3N\xec#\xcd\x1bܢ\x9d\x93i+\x15ug\x185\xa7\x88s\x15L\x95\xa1\xbb\xdc?u\xdf\xfc/\x0e\xddF\uf3b0\xf2\x8bȌ8\x1a\xd7\x1eLd\xcd_\xa9\x80\xa8\xbe\b\xa6,-\xa7\xd8\x13-C\xc2\xce\xc3\xe7\xd57\x18\xb7\xce\xc58e\xbf(g\xb7\x90\xf6%\x10\u008c\xdb`,E\xccʓ\x9c\xe8\x9a\xe0\x8d\xe3\xfc\xa3\xadAwJ?\xa5uo\x98F1K\xadjXfC\x815B\n\x8dblj\xb8s\xb0T=ڥ\"\xfc\xcf\v LS%\xc4\xdeV\x82C/<\r.\xac\x1dL\x8cNv\xa1^'\xad\xbe\n\xa8\xa5zB\xa0\xac4\x1b\xa3sk\xc0\xc6GP\xfb\xce\x1f\b\xac\x8f2Own\x06\xa7b\x8b|:z\x82\xe5[\x0e\x92\xed\x9f;ul4\xffǺ\xad\xc5+h\x00R\xdc\xe3\x87\xfa,\xe3e\f0\xa9\xdeI$\xa3\x88\x85\x06\xe1U\xac@L\xea\x10\xd3\xf9\xd6\xf2\xa1K\xfd\xf4\x06\x15\xfc\x9a1\x7f\xf1\xed\xd5\xf9\xa5w,r\xbf\x1a\xf4\xe8m\xeaq\xe5T\xa0ο\x12{\xc7\xd8\xff\x110\x96\x1b\xf3j\xe8x\xf1\xeen\xa9+\x81\xc9^\xdc\xf7\x01\xc5\xef\xf1\xf2I\x87\x80\x9b\xb2܀i\x88\xbc\xe9\xa0\xcb\xd5\xdd[(\xbc\x10~\xb5H\x17\xdav\xfc\xf2\xf5\xfc\xba\x06\xe5\x82\x1f5(Kʝ\x85 \xaf\x9b萑\xf6\xf6\xf9l\xb8\x9b\xcc\b\xf0\xdc\x19\xdd\xe5\x85Y\xc0\xe2\xccD^\x9b\xecso\x87/}o\"N4Q\x95\x9bkbX\xc0\x9f\r_p\xabK\x1bT\x83\x83\xdc\xe4x\xac8\xd1\x1bz\x8dDx\xdeF\x17O2\xd9\x04g\x83$/\xa7急\xe1A>\x8c\xfc\x1b\x00\x00\xff\xff\x02\x83F4\xa5\r\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4YKs\x1c\xb7\x11\xbe\xf3Wt\xc9\a\xc6U\x9aYKI\xa5R{\x93\xc98\xc5ĦX\xa2\xa4\x8b\xcb\a\xec\xa0w\x06\xe6\f\x80\x00\x98\xa56.\xff\xf7T\xe31;\x0f\xec.Ɋ\x9c\xb9\x90\x8bG\xe3C\xbf\xbbQ\x14\xc5\x05\xd3\xe23\x1a+\x94\\\x03\xd3\x02\xbf8\x94\xf4˖\x0f\x7f\xb3\xa5P\xabݛ\x8b\a!\xf9\x1a\xaez\xebT\xf7\x01\xad\xeaM\x85\u05f8\x15R8\xa1\xe4E\x87\x8eq\xe6\xd8\xfa\x02\x80I\xa9\x1c\xa3aK?\x01*%\x9dQm\x8b\xa6\xa8Q\x96\x0f\xfd\x067\xbdh9\x1aO<\x1d\xbd\xfb\xae|\xf3\xb6\xfc\xee\x02@\xb2\x0eנ\x15ߩ\xb6\xefpê\x87^\xdbr\x87-\x1aU\nua5VD\xbb6\xaa\xd7k8L\x84\xbd\xf1܀\xf9N\xf1Ϟ\xcc\xf7\x9e\x8c\x9fi\x85u\xff\xca\xcd\xfe(\xac\xf3+t\xdb\x1b\xd6.A\xf8I+dݷ\xcc,\xa6/\x00l\xa54\xae\xe1\x96`hV!\xbf\x00\x88W\xf4\xb0\n`\x9c{\xa6\xb1\xf6\xce\b\xe9\xd0\\\x11\x85Ĭ\x028\xda\xca\b\xed\x8a\x0e\xe1\xb1A\t\xae\x11\x16\xc2m\xe1\x91Y\x82c\x9c\xbfe\xfe`?Oۭc\x9d\x9e \xb82\xc8\x0e[\x03\x04\xce\x1c\xe6\x00\f\xfc\x04\xb5\x05\xd7 q\xde+\x16\x13R\xc8\xda\x0f\x05I\x80S\xb0A\x0f\x119\xf4:\x83LcUj\xc5K\x99\x88N`\xdd\xceF\xcf\xf1\x86\xd6\xff\xafQM\x00\xdd)\xfe\x02(\xcf:7,\x9e\x9c\xfay\xbc\x9e\x90\x00\x05X\x10\x16X\xdc\x1anq`tr\x90\x1f\xfe~\xff\x11\xd2\xd1^\x18s\xee{\xbe\x1f6ڃ\b\x88aBn1:\x98\xadQ\x9d\xa7\x89\x92k%\xa4\xf3?\xaaV\xa0\x9c\xb3\xdf\xf6\x9bN8\x92\xfb\xbf{\xb4\x8edU\u0095\xcf]\xc8a\xf6\x9a4\x97\x97p#\xe1\x8au\xd8^1\x8b_]\x00\xc4i[\x10c\x9f&\x82q\xda5_\x1c\xb86\x9aHI\xd3\x11y\xcd2\xa1{\x8d\x15I\x8f\x18H;\xc5VD\x0fE\xee\x9c͗\x97\x13\xc2yå/\xeb\x9d\xe6\x8b \x17\\f{\x1269\xf2\xa9\xc9a\x86\x95\v\xa2\x00\xed\xdc\xcb\x0e{Ƒ\xcbF\a[.(\x1c\x11\x03}Rq\xdb\a\xf5q\n\xaa\x86\xc9\x1a\xc3}\x11\xb6=E\xc7\xf2\xf2%v\xbcLIҗIM\xe6\x8e\xe3\xff\x16ܟx9\x9fA?\xe1r\xe3*\xe3\xe4\xe5\x1e\xfa\r\x1a\x89\x0e\xfd\xfd\xb8\xaa,]\xadB\xed\xecJ\xed\xd0\xec\x04>\xae\x1e\x95y\x10\xb2.H5\x8b\xa0\x03v\xe5\v\xe7\xd57\xfeϋ\xef\xe2k\xec\xa7^hR\xfb\x7f\xcd[\xd19v\xf5\xa2K\xa5\x1c\xf6\xe9q\xec\xf2>fV\xf3\xbdd\x16\x8f\x8d\xa8\x9aT\x9cD\x1f{Ę\x04e\xc2<\xb8f&\xf7_]\x95\x89\xa1\xbd!D\xfb\"v\xf7\n&9\xfdo\x85u4\xfe\"\x0e\xf6\xe2I\xe6\xfb\xe9\xe6\xfa\x8fQ\xf0^\xbc\xc8V\x8f$\xe0\xe1\xfbR\x1c`\x15\x1d\xd3EX͜\xeaD5[=퉜I\xe3>L\x16\xa7D3\x93\xdf\x0ek\x9e\x95G:Vg\x12\xb7q3\xf3Tzw\x92_\xd3\xc6\r\xab-0\x83\xc0\xa0c\x9a\xe4\xfc\x80\xfb\"$\x04\x9a\t\x8a\xe6\x14\xb0\x87\xae\b0\xad[\x91\r\xdc1\xecǔ5r\x82\xcarV\xdbcw\xcfJm\xdc\x05:#\x85O\xa3\xa5I\x06g\xfaP\xae\xc9\xd9\xf5\xa4;\xb5D\x8b\xb2\xef\x96P\nxPZ\xb0̸A\xeb\x16\xfaE\x13\xaf\x96y\xc9\ta\x05^\x9e\xe1AlXgJ\x9d(\x8a\x90\x17\x0e\xe5\x8e\xefQ\xe6\xea\x89\xe3\xc5\xc4Q\x88T\xcfS\x96;\x85X\xe4\v\xcf\xd9\x1a*\xc4fCZ\xf1\x8b9#3}\xca49风\x91.\xabq\xdf\x1e\x7fF=\x1e\xda\xfe\x91\xa7\xc1\xfb\xba\xf4\x18@\xa5\xc7K+\xf2JQV?\xe9\xe8\x9d\x11\xef\xd5r\x87o~\x19\x1e\xd5]td\xbd\xa3G\x82xF\xae\xa4\x86\x11\xb9\xb0\xd3\xc7;\xa2\x86ܧ\xdcT\x11l\x99h\x91Cz\t\x9a\xef\xc9P\x1dS\xd9\xe0\x96\x82C0\xbdT\xc8FxCZ\xdb X\xdfU\xba\xb4'h\xf6\x16\xb9\xef\x80d\x98\xb0Lu\xb7\xcat̅.h\x91%*\xfb\xb6e\x9b\x16\xd7\xe0L\xbf\x9c>a\x89\x1dZ\xcb\xeas\xa6\xf8SX\x15\xea\xfb\xb8\x05\xd8F\xf5n(\xf0'\xee\xf1\xd2F\x9dz^\x8f![:OՙQicc\x8a߶~\xcf\xd8\x11\x1c^\t=\xaa\r\xe6S\x84\x97\xf8\x04\x00\xff\xfcu\x0e!\xad\xc9\x19\xd8\xe0\xbdNZ\x18\x9cpʷ\xf8\x98\x19]<ۍ'\xaf\x92\xc9d\xe6~\xf0\xd6\xf0\xac\xfbǃα .\x83F\xb5ɘ\x95c-Ⱦ۠!>l\xf6\x0e\xedԝ\xe7\xba9\xbe\n<\xb0q\xb4?\xc9/P\x8a\x85mŤﺒu9\x05\\Xݲ}\x86p\xba\x88\xcf\xf4ȸ\xc8\x05\x1c\xf49\x19\xb5F㧞ۅ\U00098b95\x18\x9f\x83:Y|&\nŧ\xea\\\f\xbaG\xcd\fY\xba\x7fA\xb8\x9a?q\xbd\x06+|[\x942ϐ\x8a\x86\xa6\x85\xa5\xe0D\xa9\x952\x98q\x99\xb0\f+\x93 2\x85\xffGƏ\xac\x9e,\x06=r>\xa2\x1d[\xeb\xe3\x91~3<\x1b\xad\xe1\xb7\xdf/\xfe\x1b\x00\x00\xff\xffÊ\xc5\x01R\"\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4Y_\x93۶\x11\x7fק\xd8q\x1e\xae\x991\xa9\xd8\xedt:z\xb3\xef\x9aε\xc9Yc\x9d\xfd\x92\xc9\x03D\xacHD$\x80\x02\xa0tj&߽\xb3\x00A\xf1\x9f\xa4\xd3M.\xe1\x8b}\xc0b\xf1\xc3\x0f\xfb\x0f\xab$IfL\x8b\xafh\xacPr\x01L\v|r(\xe9/\x9bn\xffaS\xa1\xe6\xbbw\xb3\xad\x90|\x01\xb7\xb5u\xaa\xfa\x8cV\xd5&\xc3;\xdc\b)\x9cPrV\xa1c\x9c9\xb6\x98\x010)\x95c4l\xe9O\x80LIgTY\xa2Ir\x94\xe9\xb6^\xe3\xba\x16%G\xe3\x95ǭwߥ\xefާ\xdf\xcd\x00$\xabp\x01Z\xf1\x9d*\xeb\n\rZ\xa7\f\xdat\x87%\x1a\x95\n5\xb3\x1a3R\x9e\x1bU\xeb\x05\x1c'\xc2\xe2f\xe3\x00z\xa9\xf8W\xaf\xe7s\xd0\xe3\xa7Ja\xdd\x7f&\xa7\x7f\x10\xd6y\x11]ֆ\x95\x138\xfc\xac\x152\xafKf\xc6\xf33\x00\x9b)\x8d\vx (\x9ae\xc8g\x00\xcd9=\xb4\x04\x18\xe7\x9e9V.\x8d\x90\x0e\xcd-\xa9\x88\x8c%\xc0\xd1fFh\xe7\x99i\xf5\x80ڀ+\x90\xb6\xf4\xac2!\x85\xcc\xfdP\x80\x00N\xc1\x1a\xa1A½2\x80_\xac\x92K\xe6\x8a\x05\xa4D\\\xaa\x15Oe\xd4\xd9\xc8\x04\xce\x1f\x06\xa3\xee@\xe7\xb0\xce\b\x99\x9fB\xf6;\x83\xea\xe1Y*\xfeL$\x8f\x05z\x99\x88\xa6֥b\x1c\rm^0\xc9K\x042Pp\x86I\xbbAs\x02E\\\xf6x\xd0}$_\xa2\xbe\xce\xcc5\xec\\CE\x90\xedm\xff\xb5;tiߥ\xe2\xcd\x02h\x8c\x1a\xacc\xae\xb6`\xeb\xac\x00f\xe1\x01\xf7\xf3{\xb94*7h\xed\x04\f/\x9e\xea\x82\xd9>\x8e\x95\x9fx]\x1c\x1be*\xe6\x16 \xa4\xfb\xfb\xdfNck\x16\xa5N9V~<8\xb4=\xa4\x8f\xc3ဖ\x9c-o\xae\xffO\x81\xbb&HwJ\xf6y\xfd8\x18\x9d\x02\xdbQ\x1a\xe3m\x9a\x19\xf4\xa1\xf6QTh\x1d\xabtO뇼\xaf\x8f3\x17\x06\xc2\xf4\xee]\beY\x81\x15[4\x92J\xa3\xfc\xb0\xbc\xff\xfa\xd7Uo\x18@\x1b\xa5\xd18\x11\xa3k\xf8:ɣ3\n}foHa\x90\x02NY\x03mp\x8a0\x86\xbc\xc1\x10\x9cEX0\xa8\rZ\x94!\x8f\xf4\x14\x03\t1\tj\xfd\vf.\x85\x15\x1aR\x03\xb6Pu\xe9#\xd0\x0e\x8d\x03\x83\x99ʥ\xf8_\xabے\xefѦ%s\u0604\xf8\xe3\xe7c\xb0d%\xecXY\xe3[`\x92C\xc5\x0e`\x90v\x81Zv\xf4y\x11\x9b\u008fd!Bn\xd4\x02\n\xe7\xb4]\xcc\xe7\xb9p1if\xaa\xaaj)\xdca\xee\xf3\x9fX\xd7N\x19;\xe7\xb8\xc3rnE\x9e0\x93\x15\xc2a\xe6j\x83s\xa6E\xe2\xa1K\x9f8ӊ\x7fc\x9a4kozXGN\x17>\x9f\xeb\xce\xdc\x00%;\x10\x16X\xb34\x9c\xe2Ht\fٟ\xff\xb9z\x84\xb8\xb5\xbf\x8c!\xfb\x9e\xf7\xe3B{\xbc\x02\"L\xc8\r\x05]\xbačQ\x95\u05c9\x92k%\xa4\xf3\x7fd\xa5@9\xa4\xdf\xd6\xebJ8\xba\xf7\xff\xd6h\x1d\xddU\n\xb7\xbe\x92\xa0xYk\xb2\\\x9e½\x84[Vay\xcb,\xbe\xfa\x05\x10\xd36!b\x9fw\x05\xdd\"h(\x1cX\xebL\xc4\n\xe6\xc4}\r\xab\x92\x95ƌ\xae\x8f\x18\xa4\xa5b#2\xef\x1b\x14~\x80\x8d\xe4Ӟ\xeaiץoͲm\xadWN\x19\x96\xe3\x0f*\xe8\x1c\n\r\xb0}\x9cZ\x13\xc1\xc9N\xce\v\xca\xc1\x06ɑR\x802.\xde\x17h\xb0\xbbƠVV8e\x0e\xa48d\xcbt\xa4\xe1\xc4E\xf8#+~\xe1\x18\x14\xee\xbdC\x18ܠA\x99a\x8c\x10\xe7*\x99\x89St\x12\xfa\x18\xe2i\xea\xe1L\xf4\x9c\x04\xfcay\x1f#fd\xb8\x81\xee\xc6\xfb^\xa0\x87\xbe\x8d\xc0\x92\xfb\x84ry\xef\x9b\xfbM\xd8\xcc\xc7\x0e\xa7\x80\x81\x16\x18*\xd26\x18\x83\x90\xd6!\xe3\xa06\x93\x1a\xe9m\x00\xe4`\x06\x9b\x15oC\xa4hB\xd21\x84\x13\xf5\xc0(F\t\x0e\xff^}z\x98\xffk\x8a\xf9\xf6\x14\xc0\xb2\f\xad\xf5\xf9\x1a+\x94\xeem\x9b\xb39Za\x90S\xe1\x82iŤؠui\xb3\a\x1a\xfb\xd3\xfb\x9f\xa7\xd9\x03\xf8^\x19\xc0'V\xe9\x12߂\b\x8c\xb7\xe1/ڌ\xb0\x81\x8eV#\xec\x85+\xc40i\xb5\f\x90u5\xc7\xde\xfb\xe3:\xb6EP\xcdqk\x84Rlq\x01o|%x\x84\xf9+9\xd6ooNh\xfdKp\xa07$\xf4&\x80k\xf3]\xd7#\x8f ]\xc1\x1c8#\xf2\x1c\x8f\x85\xe8\xf0\xf3\xc1\x9bBⷠ\f1 UG\x85WL\xb7\x17\xe2\x11\xf2\x11\xe8\x9f\xde\xff|\x12q\x9f/\x10\x92\xe3\x13\xbc\a!\x037Z\xf1oSx\xf4\xd6q\x90\x8e=\xd1NY\xa1,\x9ebV\xc9\xf2\x10\xaa\xfd\x1d\x82U\x15\xc2\x1e\xcb2\t\xf5\x06\x87=;\x10\v\xf1\xe2\xc8\xde\x18hf\xdcYk\x8dU\xc6㧻O\x8b\x80\x8c\f*\xf7\xf1\x8e\xb2\xd3FP\xd5@\xe5B\xc8y\xde\x1aGI3~\xb6\x0e\xe6\xe3\x14d\x05\x939\x86\xf3\"lj\xcaB\xe9\xcdK\xfcx\x9c\xfa\xe37Q\x02\f\x03ǟ\x96D\x9fy8_\xa9>\xe3pݷ\xd6\xd9\xc3m\xeb5\x1a\x89\x0e\xfd\xf9\xb8\xca,\x1d-C\xed\xec\\\xed\xd0\xec\x04\xee\xe7{e\xb6B\xe6\t\x99f\x12l\xc0\xce\xfd\x93y\xfe\x8d\xff\xe7\xc5g\xf1\xaf\xeb\xe7\x1e\xa8\xf7\xe8\x7f\xcdS\xd1>v\xfe\xa2C\xc5Z\xf1\xf9y\xecf\xd5\x140õ\xe4\x16\xfbBdE|\x0441\xf6\x843\t\xaa8y\b\xcdL\x1e^ݔ\x89\xd0\xda\x10\xa2C\xd2\xf4\xb4\x12&9\xfd\xdf\n\xebh\xfcE\f\xd6\xe2Y\xee\xfb\xe5\xfe\xee\x8f1\xf0Z\xbc\xc8WO\x14\xba\xe1{J\x8e\xb0\x92\x8a\xe9$H3\xa7*\x91\r\xa4\xa9\xf6\xbb\xe7D\xfcF\xa0\xb9P\xc5}\xee\t\xc7*t\xa2\x8ale\xae*#\xadd\xda\x16\xca\xdd\xdf]\xc0\xb1j\x05#\x86\xe3u5\xc5c\xd45h\x02]\x87\xc7\xfb\xcb\xc3\xe9@\xd2\a\u0557\x8eȔ\x11\xb9O[\xad\xef\xfbW\x84d\x15\xeb6\xff\xba_Ŵ\x162\xbf\nk\xb7\x97v\x01藎hDy\xa1\x9b\xe7\x8a)\x9c\xbd\x1e\xdf\x18-ʺ\x1aCI`\xab\xb4`\x13\xe3tG#\xfb\xa4\x897\xe3\xba\xe6\f\x13\xc1\x00.pд\x9e&\xdeQ\x8d\xfd\x84\xbaҏ\xd0\xdb\xc5[\xd1t@\xbe֮\xe8\xd9MEr\x1fa2\xfd:\x1c\xc8h\xc5gCҺ.9\x98<:\xd4p\xa2o\xab\x83\xd9^K\xb4{\x9a\xf1\xc3\xda\xf7ۮyZ\x87\x1e_\xc3{\x88\xf0.v\xfe\xe8y\xf3\xe2\xc7u\xa6\xe8\xe9\xd0k\xcf]\xb0\x81\xdb\xf1\n\xdf\xc92\xbc\xf1\tQ\xa1\x7f\xb1\x86\xf6\xe4\x9eٸ\xc9\xd4}CG_X\xea\xb3*\xa9C\xee\v{zwl\x98(\x91C\xfb+\x8bo\xa5[\xdfҹ\x99\xaac\xa3\xa2\xda\"\xf7qc\x02\xf4x]\xec\x92r\xe60!\x15#\tY\x97%[\x97\xb8\x00g\xea\xf1\xf4\x19\xf7\xaa\xd0Z\x96_\xf2\xaf\x1f\x83Tx\xf37K\x80\xadU\xed\xdaG\x7f\xe3h\r\x157\xb6\xb1\x82\xeb\x1a\x0f\x05\xb3\x97\xa0,If\xca\xe2Z\x97?orp&\x94=\xe0~btԵ\xeeN\xdeF\x13\x9a\x98\xfb\xde[\xc7U\x044\x1b]\xe2\xa0\x11\x83B\x95Ѻ\x95\xa3\xa4TWk4D\x84o\x95GFb\xe0\x98\xea\xa2\xf8\xd7בɣ\x86\x18\v\x83\xaa\xe6=\x991雊d\xbfN\x01\x17V\x97\xec0\xa17\x9e\xc4\x17Xd\xbe\xe4GG\x8b\x89^H\xee\xef\xe7\xae\xed\xfe\xb4?\x05L\x97\x7fS?,L\xddB\xf7W\x82\xc1|\xfb\x1b\xc8\xeb\xecp\xa6䳎\x19\xf7ܰ\xb7\xea\t_\x8ax^\xf5t\xbc놮q\xa0\xeao\xf3GƨI\xa2F\x83\x1e9\xef\xe8n:\xa7ݑz\xdd\xfe.\xb0\x80_\x7f\x9b\xfd?\x00\x00\xff\xffg\b\x17r\xc1\x1f\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xdc\x00\x94GI忧\x1a\x00\xbfD\x90\x04e\xeb\xedlx\x13\x054П\xe8/p\xb5Z]\xb1\x92\x7fGm\xb8\x92\xb7\xc0J\x8e?,J\xfae\xd6/\xffi\xd6\\\xdd\x1c>]\xbdp\x99\xdf\xc2}e\xac*\x9eШJg\xf8\vn\xb9\xe4\x96+yU\xa0e9\xb3\xec\xf6\n\x80I\xa9,\xa3׆~\x02dJZ\xad\x84@\xbdڡ\\\xbfT\x1b\xdcT\\\xe4\xa8\x1d\xf0z\xe9ß֟\xfe}\xfd\xa7+\x00\xc9\n\xbc\x05\x8d\xc6*\x8df}@\x81Z\xad\xb9\xba2%f\x04s\xa7UU\xdeB\xfb\x87\x9f\x13\xd6\xf3{}\xf2\xd3\xdd\x1b\xc1\x8d\xfdK\xf7\xed_\xb9\xb1\xee\x9fRT\x9a\x89v1\xf7\xd2p\xb9\xab\x04\xd3\xcd\xeb+\x00\x93\xa9\x12o\xe1\v-S\xb2\f\xf3+\x80\xb0u\xb7\xec*\xec\xfa\xf0Ƀ\xc8\xf6X0\xbf\x1f\x00U\xa2\xbc{|\xf8\xfe\x1fϽ\xd7\x009\x9aL\xf3\xd2:\x02\x84\xbd\x017\xc0\xe0\xbbÍ6\xe0h\rv\xcf,h,5\x1a\x94ր\xdd#\xb0\xb2\x14\xc1\x01\x12x/K~\xaaǢ%4\xbd\"\xea<}~\xfe֕3nN\xa9\xef\xe8\xde\x11\xbe\x96\x05D0.\xb7\xa8=\x13\x9d\xb4\x11L\x94y\xa9\xb8\xb4\xeeG&8\xcaS\xf2\x9bjSpK|\xff\xbdBC\x02\xad\xd6p\xefl\al\x10\xaa2g\x16\xf35\x82S\xe9Ҍ\x0f\x06A\xf0\f\xbb\a\x85'\x8d?d\x98\x1e\xee\b~r\xaapC\xe6\xac\xc6\xf2Q\t\x9e\x1dgI\x13\x9bT\xab[P\xbeZ\x827\xb8g\a\xaet\x04%\xd2H\x1a\xfa\xd2\x1e\xa4\xad1Ud\xcb\x02\x90\xfc<\x84\xa3\xc4\xda+\xf52\xc7\xfb?Ә\xd6jC朷\x06\x95\xc0\xedp\x88n\x10\xf0\af\x95\x8dl\x13 \xaf\xdc\x01\xa24\x94\xca\xd8q\xbe\x8f\xdb\x1e\xf0\xe6`LhaJh\x06\x98\x05SYs\x8e\x10\xed\x99M%\x91\xf6Z\x10\xe7ڱZU~\xec\xe9\xf9֡x\x9c\"\xb0a\x06sPA\xea+\x81&\xac\x95;\xf6\xb7v\xe5z\x14t\x83\xbc\xf74\x04۠\x00\x83\x023\xab\xf4\x90\x92)\xf4\xf4O\x8a\xad\x1c\xa1c\xc4j\xf6ſEl\x02$\x90\x98\xbf\xeey\xb6\xf7N\x00ɦ\x83\x03\xb9B\xe3\f\a9\xaa\xc71$a\x8e\xf7a\x91)\xd3\xd1>3:u\n/fN\xda'\xc1ܶό\xe1\x1d\x18\x96\xf0>zr\xb6\xcf\xffO\xc2\xd6'\xc9\x19B\xfb0\x98\xfa\xbeB\xeb\x82*r\xf6\x1f\xb6\x80Ei\x8f\xd7\xc0m\xfdv\x0e\"\x13\xa2\xb3\xfe?1c\x96K\xfc\xc3\xe9\xccw\x95\xf8I\xae\xccA$\xae4\xcb\xff\x132\xc5\x1d\x16\xcf\xe1\xacHf\xc8_\xbb\xb3\xae\x81o\x1b\x86\xe4װ\xe5¢>\xe1̛\xf4\xe5=\x88\x91r\xde\xd1S0\x9b\xed?\xff \xcfƴy\xa6D\xba\x9cN\xf6.q\x1d#\xf4\x0f\xe6\x19\xb8\xe0\xc2W\xae\xb1\xf0a\xf17G\xcd\xf6\x8d\x8b'\xee\xbe\xfc\x82\xf9\x14y M\xf2\x06\x88ܝl\xb6\xbbt\xf0\xf3S\xd1\b\xaeO\x133\xf9\x84\xc750x\xc1\xa3\xf7X\x98\x04b\x0e\xb3\xceߍFOC\xe2\xb8̋w\x8f\xf1\xe8\xc0\x84T\xca\xec\xecTQ\xf0\xcf\vF\xdc\xfd\xd8\xd3# \xed)\x04\xb8\x9e\x92\xf4\xc2\x11\xc2\x05\xde\xe9\xc4\x03\x97\x16\xabm\xd1\u0a92/R\xbdʕ\x8b.\xcdl\xde\xfet\a\xe9\xe9\xaaw]ޞm\x89\xfeH+ԗ\xd7t\x99\xaa\x0f\xf4\vX\x99d\xb9Y\x14\xa1OI\xc1\x9c]\xf3\xed\xb4#\x7f\xce\xeebj\xfd\x89ɡ\xf8y\xef\xfb`S{\xac\x1e\xe2\xb3:\x0e\xc9\xeb\x1e\xed\x1eu\xdd`\xbbr\xbd\xc41;\xdd\xd6H[ߺi\xba\"\xf9\xa9\xdd3\xdf\fx҆\x15w\xbee%\xc45\t6\xab\x84\xf5-\xb1\xba\x8a\bQR3\xd2F)\x81\xec\xb4=7\xa5\x9a?W\xc3\xef\xf7\xa555\xf4\xba1MՋD0\xf4\xbc\xf4ݧ\xdd\x02q\xbf\x18\xef\xd2P\xf5N\xff\xe1-k\tu\xf6\x99\xea\xfat#\xdf\x14\xbd\x86bӥX+\x83a\\\xe8\xf0\xfc\xb9\xc8g\xb1\xf8Z\x06=\x18u@\xfb\x14\x8cL9iQp\x96\x9b\xc2HW\xecf\xe7K\xf47/\xe8k^\x10C-\x8b\xa2\x92ɔҿ|\x91X\xea\x82\xd1\xd4%\xe2\xa9\xf3\"\xaa\x19\x90'}\xc9)\x1d\xc7Ie\xbc\xe4\x9aMJ\x95m\xber<\xddI\x9c\xd0A\x9cP\r\x9a\xdbiB\xa7\xf0\xb2\x0e\xe1\x04\x1a^(ֺP\xb4u\x89x\xeb\xb2\x11\xd7l\xcc5+93\x7f/\xeb\xec=\xbb\xc8P\x97\xa3\xbf\xa8\x1c\x1f\x95\xb6s\x01\xc2\xe3\xe9\xf8H\t\xb0\x134)\x91\x83\xac\x87\xc6*\r\xe4\xfb\a\xbf\xff<\xa4\xe2պ\xb0\xfe\xe3\xf79|\x9e\x9a\x81ӈ\x90\v[\xc7s\x11\uf10c\x7fJK\xeb\xf9Ԧ\xeb\xa0\x0e\xfe\xafk{v\xec\x15\"\xf4=\x17h\f\xdb\xd5߇zE\x8d\xb0CI$\x9e\xfaZP\xdb:\x1e4\xb8i8!j\xb1\xccV,,\xe0O\xca&\xe9\x1bK\x18\xfaO\xd0\xd1\x10\xb6\x1b\xd5\x1b.-\xee\x06\xe9\xd6ж\xfe\x84̜~\xb2p@\x88_\xbbcC\xb0\xeci\xe0\xef\xc73\xdf\x1d\xe2\xbepg\xb9\xc6)\x0fA\xb9\x95\x17\xf5|\x94{f\xe6\xcc\xe5#\x8di\xeest\x94\xb2\xb1\x94O#{\x8a\xf7\x97\xaf\xe0\v\xbeF\xde\xfe\xea\x84\xde%@⪴\x82\a\xf9\xa8Վb\x92ȟ\xbf1n\xb9\xdc\xfd\xaa\xf4\xa3\xa8v\\6\xbd7\xcb\x06?2m9\x13\xe2\xe8\xf7\x13\x99{_+s\xe4\xbf\xf9\xd9#\x7fL1)\xe0<\x1b\r\xf8am0ťWtw\x9bb\xa3*\xdbՊ\x8f\xa6U\x98x&\xd7A[\xc3\x17e\xb1N\xd2\xf1>Pn`\x83Ʈp\xbbU\xda\xfa\xe0m\xb5\x02\xbe\r\x86:\x16d0.\x9c\xaf\xe1?\xb8H\x0eHSVnN>\xa5\x81\xc9#h\xa7\x15\xceI)\xd8\xd1\xf7&\xb2,\xab\xc8\x0e\xdc\x18\xcbb\aڛ\\[\xe7\xdc\x04i\x1e\xc9c\fZ\xbc\x9a\xf1ͽ̪ؠv\xdd\xdc\xf4\xb7'\x9d\xbb\xe8\xe1MP\xb4@\x01\xee:H\xe7\x9e\x19\x18R\xe7x<=e|\xdc\xff\xca2\xf10\xee\xa8\xf5;\x91\x9b\xc15\x02n\xfa\x10\x8dާ\xe5ƛ\x90\xb8\xa9\xa7\x12ϲ=\x93;\x12\x1f\xad\xaaݾ\x16\xc11K=\x96?\xa9ܗ\xedJ\xa7\xa9\xa6Nv\xdbJ\xcbN\xae&\xa4\xbf\xf3v\xbbS@\xa7I8\xe1g\xea\xb6\x15\xaf\xb5\x19w\xfe\x92FLfb\xde\xce\xc8\xe4\x11\xfaG\xf6Ϛ)\xcc\x1ce6\xdd\x1f\xe8\xdb\xc0\xf9\xc45\x80)bD\xf1m,\xe09\xf86\x93\xd3\xf1m\xbd^ql}\xa9%ȏ\xfb\xd9\xef@\x0eo\xd2ϡ\x85\x9f9\xa6x\x0e\xbf\xc8\xce\x17\xb1;d\x1bP\x92\x83\xe9\xaa\xc0\x83\x9c\x06\xd4n\xdb2Z\x98\x9e\x979\x17t\xf5\x06\xbf͛v\v\x93/\xfd\xf3z\xc1\x87ƍ\xf9\x9c\xe2\x0f\x7f?\x19~\xd2lM\x9eq\v1\xf8\xb0\x11\xe2\xfc+\xdf\xd6\xdf\xe8\xde\b\xfc\xb7\xc1\x88?\xb8i\xfa\x95i\xc9\xe5n\x0e\xf9\xdf°H8\x10 D\x02\x82\b\x12M\x88\xb0( \xa879\xf2\x19\xda&HxCH\x10=N\x06/\x9d \xe7\x1d\"\x87\x95\u009b\xff\v\x00\x00\xff\xff\x97{a\x8a9_\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=Msܸrw\xfd\x8a.\xe5\xb0IJ3^'\x97\x94n\x8e\xecM\xa6\x9ec\xab,=\x9fr\xc1\x90=\x1a\xacH\x80\x0f\x00G\x9e\xbcz\xff=\x85\x06\xc0\xaf!Hp$ew_\x84\x8b-\x0e\xd0ht7\xfa\x03h\x00\xab\xd5\xea\x82U\xfc;*ͥ\xb8\x06Vq\xfcaPؿ\xf4\xfa\xf1\xdf\xf4\x9a\xcbw\x87\xf7\x17\x8f\\\xe4\xd7pSk#\xcbo\xa8e\xad2\xfc\x88;.\xb8\xe1R\\\x94hX\xce\f\xbb\xbe\x00`BH\xc3\xecgm\xff\x04Ȥ0J\x16\x05\xaa\xd5\x03\x8a\xf5c\xbd\xc5m͋\x1c\x15\x01\x0f]\x1f~^\xbf\xff\x97\xf5\xcf\x17\x00\x82\x95x\r:\xdbc^\x17\xa8\xd7\a,P\xc95\x97\x17\xba\xc2\xcc\x02}P\xb2\xae\xae\xa1\xfd\xc15\xf2\x1d:d\xef|{\xfaTpm\xfe\xd4\xfb\xfc\x99kC?UE\xadX\xd1鏾j.\x1eꂩ\xf6\xfb\x05\x80\xced\x85\xd7\xf0\xc5vU\xb1\f\xf3\v\x00\x8f?u\xbd\x02\x96\xe7D\x11V\xdc*.\f\xaa\x1bY\xd4e\xa0\xc4\nrԙ╡\x11\xdf\x19fj\rr\af\x8f\xdd~l\xf9UKq\xcb\xcc\xfe\x1a֚꭫=\xd3\xe1WG\"\a\xc0\x7f2G\x8b\x9b6\x8a\x8b\x87\xb1\xde>\xc0\x8d\x92\x02\xf0G\xa5P[\x94!'\x06\x8a\axڣ\x00#AՂP\xf9w\x96=\xd6\xd5\b\"\x15f\xeb\x01\x9e\x1e\x93\xfe\xc79\\\xee\xf7\b\x05\xd3\x06\f/\x11\x98\xef\x10\x9e\x98&\x1cvR\x81\xd9s=O\x13\v\xa4\x87\xadC\xe7\xf3\xf0\xb3C(g\x06=:\x1dPAxיB\x92\xdb{^\xa26\xac\xec\xc3\xfc\xf0\x80\t\xc0\x88D\x15\xab5\tG\xdb\xfa\xb6\xfb\xc9\x01\xd8JY \x13\x17m\xa5\xc3{'{\xd9\x1eKv\xed+\xcb\nŇ\xdb\xcd\xf7\x7f\xbd\xeb}\x86\x81,yJ\x01\xd7\xc0\xe0;M\fP~\xa6\x82\xd93\x03\n-\xe7Q\x18[\xa3R\xb8\n\xd4\xcd\x1b\x90\x00RA\x85\x8a˜g\x81+\xd4X\xefe]\xe4\xb0EˠuӠR\xb2Bex\x98z\xaet4J\xe7\xeb\x00\xe3\x9f\xec\xa0\\-'\x89\xa8I\xf8\xfc\x84\xc2\xdc\xd3\xc1\xcd\x0f\xae[\xfc\x89I=\xc0`+1\x01r\xfb+ff\rw\xa8,\x98\x80u&\xc5\x01\x95\xa5@&\x1f\x04\xff\x9f\x06\xb6\xb6RoH\x18\rz}\xd0\x16\x9a\xc0\x82\x15p`E\x8dW\xc0D\x0e%;\x82B\xdb\vԢ\x03\x8f\xaa\xe85\xfc\x97T\b\\\xec\xe45썩\xf4\xf5\xbbw\x0f\xdc\x04M\x9aɲ\xac\x057\xc7w\xa4\x14\xf9\xb66R\xe9w9\x1e\xb0x\xa7\xf9Ê\xa9l\xcf\rf\xa6V\xf8\x8eU|E\xa8\vҦ\xeb2\xff\x87\xc0Q\xfdS\x0fד\xf9\xe6\n)\xc2\t\x0eX\x8d\xe8\x04\xc65u\xa3h\tm?Y\xea|\xfbtw\xdf\x15&\xae\x87\xd4'\xbaw$\xace\x81%\x18\x17;\xf43z\xa7dI0Q\xe4\x95\xe4\xc2\xd0\x1fY\xc1Q\fɯ\xebmɍ\xe5\xfb_j\xd4\xc6\xf2j\r7d^\xac\x1c֕\x9d\x81\xf9\x1a6\x02nX\x89\xc5\r\xd3\xf8\xea\f\xb0\x94\xd6+K\xd84\x16t-㰲\xa3Z\xe7\x87`\xde\"\xfc\ns\xfc\xae¬7el;\xbe\xe3\x19M\fҞ\x8d\n\x18hPW\xc6g-\xfdBjj\xf8u\x80\x87\xd3e\xa1W\xd4\xd6~\x98=q\xb85cV\xae\x1c4\xabS\x84\x1crwL\vv(\xe1\xa1\xcc`\xd2\xd7z\xa9\xf6\xed\x04&xU\xb7\x8e\xe0x\xc2U\xfa\t\xcbʪ\x8d\x19\x14\xef}5\x8b\xa2\xa5O\xdexM\xc1\xf0\a5+\xbdv\x85\x13\xe5F\xdd\xed\xd1\xf2\xed\xc0s\xaf\xbdN\xb8\n\x93\x9c\xb5%\xd3\xfcN\xb0J辰6N\xd6f\xac\xd6`\x007w\x9bA\xa3\x0e\xe7-VdÉ\xd1F\xc2\x13㧜v\xc5\xca\xe5\xcd\xdd\x06\xbe[\x97\b\x03Lp\x96\x1cL\xad\x04\xa9\xe3o\xc8\xf2\xe3\xbd\xfc\xb3F\xc8k\xd2J\xc1._E\x00oqg'\xbdB\v\xc36@\xa5\xec\x1cЄ\x9a\xac͚\x1c\x8e\x1cw\xac.\x8cWr\\\xc3\xfb\x9f\xa1\xe4\xa26x\xcaw\x98\xe6\xbd#\x12\x81s\xa3\xd1\xf7\xf2\x17\xed\x18\x99@ҏ\x91\xa6#S\xaa\x929\x1c\xa8^\x8c\xaa\xbc@\xd0Gm\xb0\x84\xad\x87\xd2\xd8j\xe2\n郢\xf0`4l\x8f\x01\xf7\xf1q\x8b\xba(ض\xc0k0\xaa\x1e\xefvj\xea\x8e\xd1\xe6\x1bjó\x04\xca\\\x0eI\xe3Z\x8e\x10F\xd1\x0f\x11\xa2\f(`\x8d<{\xb4\x8e\xa6\xa7\x90\xf5\x16\x8a\xa2C\xdcy\xaa\x00\xfc\xb7\x80\x8f\xd6\xc0e\xd6\xec\\{sƱ \x13*$\x14R<\xa0r=ZW\xe1\x89\x17\x05Mi,\xe5\xa1\xe7du\x8b\xb5-\n\vk$aW[\xb3\xb3\x06+\xfbQ\x19\xe1B\x1bd\xf9\xfa\U000b5607?\xb2\xa2\xce1\xbf)jmP\xdd٠'\x0fAߨ^\x190\xf1\xd3$\x00\xefp\x14\xd6\xe0\x17sOf\x04\xe2\x04\x7f\xe7N\x84QX.\xf5\xe2J)\xd0\x06n\xa5T1\xcfӕS0q2l\x19\x85\x83\xb1 \xac-\xaa.P{T\x9c\x03\xdbꝫ\x96SnI\xa6`[,@c\x81\x99\x91*N\x9e\x14!p%U\x7fF(;\xa2I\xfb\x11Ӭ\x12m\x8b\r\xa9\xf6<\xdb;w\xd3J\x19\xc1\x82\\\xa2&\x8d\xc1\xaa\xaa\x88X\xa1\xb6\xccJ\x86\xeflNi\xb4%A}\f\xe1\xc6\x14I[\x12up[f\xb4q\x9f\xea\x8dؼ\x11\xbd\x87\xa6x\x96\xb0oN\x9a\xbf\xbc\xb0[rs\xd4\xe4\xf4\x91\xd7u\x05܄\xaf)P{~\xa0\xfe;c\xdcy\xb3e3l\xfd\xe2\xb3\xe5E\xb8֠\xf1w\xc242Vw\xdeV-b\xd8\xe7n\xcb+ໆa\xf9\x15\xecxa\x90|\xa99D;\x8e\xce,\xe7^\x92@\xa9\xb6ז\x92\x99l\xff\xa9Y\xc8Mh1\xa0\xd5\x10\x80\xf3\xcbC\fC;k\xf6\xbcZ@\x01\xab\xb0iIF\xee\x9a\xdd\xd0\xef\xac\xe0y\xd3\x19͓\x05\x107\xe2\n\xbeHc\xff\xf9\xf4\x83k\x8b\xa2\xc8\xe1\xa3D\xfdE\x1a\xfa\xf2\xaa$v\x838\x93\xc0\xae1MK\xe1̂\xa5ˢ\xfe[\x1cȄZ\x11m\xd8\xc65l\x84\x8d\xcf\x1c}\x96\xb0i\x8f\x019\x87VYk\xdaO\x15R\xacܒ\x96\xefm\x01\xd0.^\x9eUR\xf58u\xb5\x10\xe2(\x8a\x1e\xbd{k\xad\xdc/';\xd1SEaU\xb0\f\xf3\xb0\xafD\xdb\xde\xcc\xe0\x03ϠD\xf5\x80PY\xbb\x91.T\v4\xb9+gHa\xbak\x11\x8a7\v#\xbb\xb8ceeg}b\xcd\xc0\xe6\xa4\xea\x91=\xee\xe9\xeai\xa3$\xf3N\xfeP\x12\xf5\xbbIY\xcb,\xcbB~\x9d\xfa \x0eI\xe7~\x94\x8c6\x9e\xfej\xcd+\x89\xf7\xdfҬ!\xe3J\xaf\xe1\x03\xa5\xa4\x15\xd8m\x1fV\t;]%\x81\xb4\x98p\rVN\x0e\xac\xb0\xee\x83U\xde\x02\xb0p΄ܝxPi*\xe6i/\xb5\xb3\xf9\xcd\xc6\xd8\xe5#\x1e/\xafN\xb4\xd7\xe5FDW\xed\xfb\xc5\xea\xfc\x13\xa5\xd5x-R\x14G\xb8\xa4\xdf.\xc91[2E\xcep\xde\x16H\xf5\x82\xaa?V\x8f\xf5\x16\x95@\x83zU\xb2j\xe5g\x83\x91et\x8f\xd3\x15J\x1c[\x12F\xd88=x<\xb6q\x93^e\xdd\xff9\n$χJ\xeaHnA\x04\xad[\xa9\x8d[<\xec\xb9\xea#\xab\x8b)\x91\xa3_q\x04\xb63\xa8@\x1b\xa9B*\x93Uك\xc5u+5z^n\xdc>\x91_\xc9t\x80m\x80z\xd9j\x17g\x0f.\xdd^\x95\xfd\xff<̌\x1c-\x82])\x99\xa1\xd6\xf3b\x98huf\x16{\x9b\x85^\xe6\x02\xbf]\x92ZOY\x86\x0ee\x99\x1boI{FP\xf4\xe9Gg\xcdڪ0\xfbw\x8a(\x9f\x83#P6sY\xb2aZ]2\xba7\xaeu\x98\x80\x1e\x98\v\xb6\xd4CM\ni\x99\xcf\xedE\xf2\xf7洔\\l\xa8#x\xffj\x8e\x0e\x043\x80\xe7\x86A7\xa1}ː\xe6Cj\xec\v!9K\xd2>\x8f\xc2\x1egOwA\xd29\x05\xd6\x11\x17\xd2t\x17z|O?i\xd8q\xa5M\x8b\xf0\x02\xa8\\S\x92\xca\xebƧ\xe2\x93Rg\x87\xa7_]\xebΒ\xe4^>\xf9\x94\xc6%Ay \xfe\x9e\x1d\x10\xf8\x0e\xb8\x01\x14\x99\xac\x05-\x96Yua\xbbY\x00\xd11\xd1\x19\x93D\x9b\xd9i,\xea2\x9d +\x92N.fWֺM~a\x9d\xdf\x19yVv\xea˟\x83y\xc9u\x8b\xe9\\Ӥ\fӤ\xb5\x8dy\x9c\x93rH\x97f\x8e&Qui\x96h\x93\x01:\xd1qRn\xe8i\xde\xe7\xd4Pf3B\xe3ٞS`\xc7\xf2@\x13r<'@v\xb3?\x17\xbb\x01\xb3\xd24[ai\xee\xe6\xf8\x8d`\xa1\xcc[\xe7ⷐ\xd9\xe7\x92I\xaa\x9eӜ\x12\xdc}\x1d4\xb1\xd2\x12\xfc\xc41G<\x1e*;\xf7\xfc\fG<\x02r\xb3\x83\xb2.\f\xaf\x8aΕ\\f\x8f\xc7\xe6ʟ_%\x1d\\\xdf\x1e\t\xda\xd7o\x8d\xc8\xc7@\xf6C\n\xa6\xe1\t\x8b\xc2\xfe{B\x85\xcc]\x80\x97\xc9\x15Z+\x15\xdf\b\xf4W\x1d\xf9\xdb\xf3\xaeܲ\x18\x9d\xea'\vXZHᆤ3\xa2\xd8i\xf7\xd8y\xf5\xf4\xed/5\xaa#\xc8\x03\xaa\xc6\x0f\x8a\x8aY{\xda\xd3Ofmc\u00a0|\xbc\x16s\xd70\xf6\x95Q|64*\x00>\bg\x98\x87\xb8\x12,\xabu\xdapjJ\xd9\xda\xe8)\x06B\xc8\x06B\xa4}\x8a\xf7\xbd\xe4\xf8\xe3k\x04W/\x11^%9\"\xaf\x11b\xbdV\x90\xb54\xccZ\x92\xbc\x91t|\xf15\x82\xad%\xe1\xd6\"\x9f1\xfdx\xe2k\x1dK|\x85\xb0\xeb\xec\xc0k\x11\xe9R\x8f\x1d.\x0e\xbf\x12\xc67s\xcc\xf0\xc4GK\x00\x19=^8\x1e\x82%@<9V8\x1b\x84\xa5̃a\x98\xf6\xecC\x82ɉL\x8bv\xd3S\x93\x90\xd26\xba\xe7\x0f\xff%\x1e\xfaK\xdc\x06O\xc1>\xf1p\xdf\xf2C}\x89t>3<\x9b\xec:\xf1\xf0ޢ\x00\xed\xcc\x10m\x12\xe2\xd4a\xbd\xe9 mz\x01nxH\xef\fw\"A\xc2\x12\xaa,?h\xf7\xec\xcd\x18\xa9rT\xb3\xfbZK\xc4yV\x90\aqT\xbf\xff\xc1\x8eN\xb8\x11\xd5\xd6\xea\xee\x99\xc58*\x9b{G2\xf8\x13\x17~\xb7\xde\nn\xc7'\xe9m\xbc\xb5\x0eS|_\xa7\xf5R\xfdU\xe2n\xc7Nc\xc5\x14\xed\xe3o\x8f.)H\xaf\xe1\x13\xcb\xf6M\x0f\x11\x90\xd4\xef\x9ei\xd8IU2\x03\x97\xcdV\xe8;ׁ\xfd\xfbr\r\xf0\x8bl\xd2G:\xf7\x8aE\xa0j^V\xc5\xd1FLp\xd9\x05\xf3<\xc1\x89\nl\xc0\xe7V\x16<\x8bx\x88\xa3Wҹ\x06'\x17\xfaХyY'\v\"\x16\xa3\xd8\xe6\xdc_X8\xb8Tx'\x8bB>\x9d\xb9t\xc1*\xfe\x1f\xf4rG\xda\xda؇\xdb\rU\x0fRE\xaf~4\xd9s\x8d\x8cmqZ\xa1\xb7\x03'ף\vu${\xb5\xf9s\x02\"ݛ\x1f\xfc\f\xaf\xc63i\xb5\xd8\xed\xc6a\xb9&\xc1b\xe2\b\xd2ߌ\xceU\xbe\xaa\x98\x8anꁗ\a}\xd5\xc30\xd8\xf1\xb9\x15\xacI\xb3v\xfa\x0e@\xb7\xf4h\x1e\x9e\x04\xa0\xcd\xdec\xd5\xdfF'Jw\xe8\xf9\x1c\x9c\xa6\x0f.\xcf\x1eY~\x05\x9c\xa6]\xa6\x15Q1\xf2S4\x1d\xef\xc5W\x0f\xb5\xbf6\xdd\xdf\v\x9d\xa0\x14\xee\xfa-Ʈ\xed\xf7\xd7c\a\xd8\x13\xaa\xdfJ\xf4\xedw\x8a\xc6\x1am\xe2'\x80\x8f\xb5\xc2J\xda\xe0&\xd3\b\xc8ؽ\xfb\xf02\xe9u\xdaH\xc5\x1e\xf0\xb3tO#\xa4P\xabߢ\xf7:\x86\xf7uB\xb2\xad\x17Ř\xae\xf3c\x1b\x02ls\xf0Onk\xb7\xd8\xc6f\xfa\x8c\xf4\x1aS$\f\xee\xfe\xfe\xb3\x1b\x90\xe1%\xae?\xd6.\x01ê%\x8d\x96\xd2a\xa0\xae\xd16>y\xf7\xf2\x89\xae;\xef\xbe_\xd0y!\x06)矲*\xcf\x1a͡\xf7B@ ]\x8a\xb0\x7f\x1fo\xd9q1:L\x9cʰ\x92\xbb(,\xa6\xb5\xcc8y%\xb4\xa4L\xa9\xf6\xafwQ\xef\x94w9\xa1^j\x8d_\x9f\x04\xaaoa\xa2ꍈ=P\xd0#\xe1\x9fO\x1aF\x1f'0\x92|\xa1A\xf51\x1fXx\x02i\xf7\x98CX\x1b\xe7\xbay\xc2\xe3\x94t3\xf3?>\xf7\xc7\x15\xf9j\xfcՌU\xf3\x90\xc7E\x02e\xddc\x15)o\xb4\xb8W-2V\x99Zy\x0f,\xab\x15\xddZl\x81\xa0\xbb\xd4\xf7\xbcWZ\xdaW\xaffxپ\x83ծ\x16ξ\xba5¿杕\xe8\xc3#\xce\xcfv\xafb\xad,\xfc\xf3\xd89:\x0f\xe8\x96\xe7\xb97il\x9d昔'45\f\xb7C\xdf\xc5P\x1f?\xf7\xb2\x82/x\xea\xe7\xae\xe0\x93\xb0\x838u\x04\xdc\xe1\x16\xcci\x8dq셪\xc9!\x1e\x9aVt\xb2hD[\xf4\xd5ܠ\xfa \xef\x91^\xfbh\xaa\xb8SDcl\xfdG\xbes\v\xc0\x99\x1d\xd3?\x9dԈ*\xaeI\xa5\x15SX\xa3S\xea\xe4\xa3Fu\xa0\xe75\x82\x90x\x1b\xde\xfdRo\xdb˺\xe1\xaf\x7f\xbb\xf8\xdf\x00\x00\x00\xff\xff\xe1\xf9\x9eA\xc6p\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=Mw$)rw\xfd\n\x9e|\x18\xdbOU=m_\xfctk\xab{\xecz;ӭ\xd7\xd2\xf6\xc9\x17*3J\xc5(\x13r\x81,uy\xdf\xfew\xbf\b \xbf*\xc9$K\x92gfW\\\xba\x95\x05A\x10\x11\xc4\a\x04\xb0Z\xad.x%\xbe\x816B\xc9k\xc6+\x01\xdf-H\xfcˬ\x1f\xffì\x85zwx\x7f\xf1(d~\xcdnjcU\xf9\x15\x8c\xaau\x06\x1fa'\xa4\xb0Bɋ\x12,Ϲ\xe5\xd7\x17\x8cq)\x95\xe5\xf8\xd9\xe0\x9f\x8ceJZ\xad\x8a\x02\xf4\xea\x01\xe4\xfa\xb1\xde¶\x16E\x0e\x9a\x80\x87\xae\x0f?\xae\xdf\xff\xdb\xfa\xc7\v\xc6$/ᚙl\x0fy]\x80Y\x1f\xa0\x00\xad\xd6B]\x98\n2\x04\xfa\xa0U]]\xb3\xf6\a\xd7\xc8w落\xf3\xed\xe9S!\x8c\xfdS\xef\xf3\xcf\xc2X\xfa\xa9*j͋N\x7f\xf4\xd5\b\xf9P\x17\\\xb7\xdf/\x183\x99\xaa\xe0\x9a}Ʈ*\x9eA~\xc1\x98ǟ\xba^1\x9e\xe7D\x11^\xdcj!-\xe8\x1bU\xd4e\xa0Ċ\xe5`2-*K#\xbe\xb3\xdcֆ\xa9\x1d\xb3{\xe8\xf6\x83\xe5W\xa3\xe4-\xb7\xfbk\xb66To]\xed\xb9\t\xbf:\x129\x00\xfe\x93=\"n\xc6j!\x1f\xc6z\xfb\xc0n\xb4\x92\f\xbeW\x1a\f\xa2\xccrb\xa0|`O{\x90\xcc*\xa6kI\xa8\xfc'\xcf\x1e\xebj\x04\x91\n\xb2\xf5\x00O\x8fI\xff\xe3\x1c.\xf7{`\x057\x96YQ\x02\xe3\xbeC\xf6\xc4\r\xe1\xb0S\x9aٽ0\xf34A =l\x1d:?\x0f?;\x84rn\xc1\xa3\xd3\x01\x15\x84w\x9di \xb9\xbd\x17%\x18\xcb\xcb>\xcc\x0f\x0f\x90\x00\x8cHT\xf1ڐp\xb4\xado\xbb\x9f\x1c\x80\xadR\x05py\xd1V:\xbcw\xb2\x97\xed\xa1\xe4\u05fe\xb2\xaa@~\xb8\xdd|\xfb\xf7\xbb\xdeg6\x90%O)&\f\xe3\xec\x1bM\f\xa6\xfdLev\xcf-Ӏ\x9c\ai\xb1F\xa5a\x15\xa8\x9b7 \x19S\x9aU\xa0\x85\xcaE\x16\xb8B\x8d\xcd^\xd5Eζ\x80\fZ7\r*\xad*\xd0V\x84\xa9\xe7JG\xa3t\xbe\x0e0\xfe\x01\a\xe5j9I\x04C\xc2\xe7'\x14\xe4\x9e\x0en~\b\xd3\xe2OL\xea\x01fX\x89K\xa6\xb6\xbfBf\xd7\xec\x0e4\x82\tXgJ\x1e@#\x052\xf5 \xc5\xff6\xb0\rJ\xbd%a\xb4\xe0\xf5A[h\x02K^\xb0\x03/j\xb8b\\\xe6\xac\xe4G\xa6\x01{a\xb5\xec\xc0\xa3*f\xcd~Q\x1a\x98\x90;u\xcd\xf6\xd6V\xe6\xfaݻ\aa\x83&\xcdTY\xd6R\xd8\xe3;R\x8ab[[\xa5ͻ\x1c\x0eP\xbc3\xe2a\xc5u\xb6\x17\x162[kx\xc7+\xb1\"\xd4%i\xd3u\x99\xffS\xe0\xa8\xf9\xa1\x87\xeb\xc9|s\x85\x14\xe1\x04\aP#:\x81qM\xdd(ZB\xe3'\xa4\xce\xd7Ow\xf7]a\x12fH}\xa2{G\xc2Z\x16 \xc1\x84܁\x9f\xd1;\xadJ\x82\t2\xaf\x94\x90\x96\xfe\xc8\n\x01rH~SoKa\x91\xef\x7f\xa9\xc1X\xe4՚ݐyA9\xac+\x9c\x81\xf9\x9am$\xbb\xe1%\x147\xdc\xc0\xab3\x00)mVH\xd84\x16t-㰲\xa3Z\xe7\x87`\xde\"\xfc\ns\xfc\xae\x82\xac7e\xb0\x9d؉\x8c&\x06i\xcfF\x05\f4\xa8+㳖~!55\xfc:\xc0\xc3\xe9\xb2\xd0+\x18\xb4\x1fvO\x1cn\xcd\x18ʕ\x83\x86:E\xaa!wǴ`\x87\x12\x1e\xca\f&}\xad\x97j\xdfN`2\xaf\xea\xd6\x11\x1cO\xb8J?AY\xa1ژA\xf1\xdeWC\x14\x91>y\xe35\x05\xc3\x1fԬ\xf2ڕ\x9d(7\xean\x0fȷ\x83Ƚ\xf6:\xe1*\x9b\xe4,\x96̈;\xc9+\xb3W\x16m\x9c\xaa\xedX\xad\xc1\x00n\xee6\x83F\x1d\xce#VdÉ\xd1V\xb1'.N9\xed\n\xca\xe5\xcd݆}C\x97\b\x02L\xe6,9\xb3\xb5\x96\xa4\x8e\xbf\x02Ϗ\xf7\xea\xcf\x06X^\x93V\nv\xf9*\x02x\v;\x9c\xf4\x1a\x10\x066\x00\xadq\x0e\x18BM\xd5vM\x0eG\x0e;^\x17\xd6+9a\xd8\xfb\x1fY)dm\xe1\x94\xefl\x9a\xf7D$ny\xa9\x0e\xa0\x13h\xf8\x91[\xfe\v\xd6\x1d\x90\x0ea0\x02\xe2\xd9Od\xdc\x1e#\x03\xc5&['\xa9l\xb3\xeb@\x15\x86]^\xe2<\xbbt.\xf1啫[\x8b®\x84\xa4~\"0]\xefO\xa2(B\xff\xe7Q\xc3\x11\xd7\xf1\xd6ܫ\x9f\x8c\x13\xeb\x14\xe2D\x9a\x8e(\x98J\xe5\xec@\xf5b2&\n`\xe6h,\x94\x9eR\x1dυ\x88Kڱ(<\x18öǀ\xfb\xf8\xb8e]\x14|[\xc05\xb3\xba\x1e\xefvJ\x91\x8d\xd1\xe6+\x18+\xb2\x04\xca\\\x0eI\xe3Z\x8e\x10F\xd3\x0f\x11\xa2\f(\x80.\x0f\x7fD\xb7\xdbS\b}\xa7\xa2\xe8\x10w\x9e*\x8c\xfd\x8fd\x1f\xd1\xdcgh\x84\xaf\xbdq\x17P\x90C!\x15+\x94|\x00\xedzD\xc7)H\x98\x06\x94\xb8<\x02\x15-\xad\x86\x02]\x06\xb6\xab\xd1\b\xaf\x19j\x82\xa8\x8c\bi,\xf0|}\xf9Z̃\xefYQ\xe7\x90\xdf\x14\xb5\xb1\xa0\xef0\x04\xccC\b<\xaae\aL\xfc4\t\xc0\xbb_\x85\xc8\x00\xf9\x90\xb9J+\x8a4cDj=\xb1c\x05.\xf0E\xa6zL[\x17\xab\xa3*\fX\xacr\xf9\xaf\x971%\x8a\x12\xd0\xef\xbdߏa\\CC\x8d\x9eF\x8d@l\xf4,\x94\x95=\x8eˑ\xb0PF\x888\xabr\x16\xb0\x97k\xcdǔj\x18N\x13џ\xcf\xde\x18\x88\x01\x83e\xa8\xf6\x1b\xb1x\xd8\xff?\"\x93\xcfb\xab\xa1u,.$\xb2\xb3\x10\xc6\xf6\xb89\f\x88\x1a\xcc0vF\x9ab\xd0\"\xa4\x83\x89ʭü\xdf3\xcdΙ\t1\xd1o$͋\xf3\x9eDŽ\xea\x0fH\xb0\xbdR\x8f)D\xfao\xac\xd7\x06\xca,\xa3%U\xb6\x85=?\b\xa5\xcdp\xb5\x05\xbeCVۨ\x9e\xe0\x96\xe5b\xb7\x03\x8d\xb0h\x81\xb0YO\x9c\"\xd6t\x98\xc0:\n(Za0\xae\x96\xe9\xc8<\xa2Fl(\x14\x8eE\xa12B\x1c\xbdx\xb2\xee\xb98\x88\xbc\xe6\x05\x19z.37>\xde\xe0\x17sOf\x04\xe2\x04\x7f\xe7N\x84Q \x97zQ\xb6\x92\x80\xeeu\xa9t\xcc\xf3t\xe5\x14L\x9c\f[N\xc1q,$m\x8b\xae\v0\x1e\x15\xe7\xc0\xb6z\xe7\xaa\xe5\x94[\xa0*\xf8\x16\nf\xa0\x80\xcc*\x1d'O\x8a\x10\xb8\x92\xaa?#\x94\x1dѤ\xfd hV\x89\xb6\x05\x03̽\xc8\xf6\xce\xddD)#X,W`Hc\xf0\xaa*\"V\xa8-\xb3\x92\xe1;\x9bS\x1amIP\x1fC\xb81EҖD\x1dܖ\x19mܧz#6oD\xef\xa1)\x9f%웓\xe6//\xecHn\x01\x86\x9c>\U000bab98\xb0\xe1k\nԞ\x1fh\xfe\xce\x18w\xdel\xd9\f[\xbf\xf8ly\x11\xae5h\xfc\x9d0\x8d\x8c՝\xb7U\x8b\x18\xf6s\xb7\xe5\x15\x13\xbb\x86a\xf9\x15ۉ\xc2\x02\xf9Rs\x88v\x1c\x9dYν$\x81Rm/\x96\x92\xdbl\xff\xa9Y\xd6Nh1\xa0\xd5\x10\x80\xf3\xcbC\fC\f<\x9d.\n4\xc0$\x90\x9dA\x91\x9b\xd6\xc4xn?\xef\x8aq\xf6\bG\xe7Y\x8d.\x0f\x8d\x15d-o@j\xa0\xcdER#\x8fp$P~\xb70\t\xde\x12Qq\xe5\x11\x8e\xa9U\aDE\xfc\xfc>\x85\xa3.~\xa0Q\xa4L\xa5\xb64D\xf5s\x87Y\x956X\xb6L)\x85\x12(~\xe6\xb0\x1b\x86\xf5\xb6\xc8\x1f\xe1\xf8\x83q\xec\xc3Y\xb3\x17\xd5\x02\n\xa0¦%\x19\xb5k\xf6\x86\xbf\xf1B\xe4Mg4O\x16@\xdc\xc8+\xf6YY\xfc\xe7\xd3wa\x10E\x99\xb3\x8f\n\xccge\xe9˫\x92\xd8\r\xe2L\x02\xbb\xc64-\xa53\vH\x97E\xfd\xb78\x90\tE\x11m\xd8&\f\xdbH\x8c\xcf\x1c}\x96\xb0i\x0f\x019\x87VY\x1b\xda]\x96J\xaeܒ\x96\xefm\x01\xd0.^\x9eUJ\xf78u\xb5\x10\xe2(\x8a\x1e\xbd{\xb4V\ue5d3}\xf9\xa9\xa2\xa1*x\x06y\xd8e\xa3$\x00n\xe1Ad\xac\x04\xfd\x00\xacB\xbb\x91.T\v4\xb9+gHa\xbak\x11\x8a7\v#{\xdace\x85\xb3>\xb1f`sR\xf5Ȏ\xfft\xf5\xb4Q\x92y'\x7f(\x89\xfa\xdd\x14\xb5e\x96e!\xbfN}\x10\x87\xa4s?JN\x1bO\x7fE\xf3J\xe2\xfd\xb74kȅ6k\xf6\x81\x12\xf4\n\xe8\xb6\x0f\xab\x84\x9d\xae\x92@\"&\xc20\x94\x93\x03/\xd0}@\xe5-\x19\x14ΙP\xbb\x13\x0f*M\xc5<\xed\x95q6\xbf\xd9\x18\xbb|\x84\xa3ߜ\xedj\x89ˍ\x8c\xae\xda\xf7\v\xea\xfc\x13\xa5\xd5x-J\x16GvI\xbf]\x92c\xb6d\x8a\x9c\xe1\xbc-\x90\xea\x05U\xbf\xaf\x1e\xeb-h\t\x16̪\xe4\xd5\xca\xcf\x06\xab\xca\xe8\x1e\xa7+\x94F\xb7$\x8c\xc08=x<ظI6C\xf7\x7f\x8e\x02\xc9\xf3\xa1R&\x92i\x11A\xebV\x19\xeb\x16\x0f{\xae\xfa\xc8\xeabJ\xe4\xe8W\x1c\x19\xdfY\xd0\xccX\xa5Cb\x17\xaa\xec\xc1\xe2:J\x8d\x99\x97\x1b\xb7O\xe4W2\x1d`\fP/[\xed\xe2\xec\xc1\xa5۫\xc2\xff\xcf\xc3\xcc\xc8\xd1\"ؕV\x19\x98h6B[\x12\xad\xce\xccbo\xb3\xd0\xcb]\xe0\xb7KR\xeb)\xcbС,s㑴g\x04E\x9f\xbew֬Q\x85\xe1\xdf)\xa2|\x0e\x8e\x8cr\xbb˒\x0f\x93\f\x93ѽq\xad\xc3\x04\xf4\xc0\\\xb0\xa5\x1fjRH\xcb|n/\x92\xbf7\xa7\xa5\x14rC\x1d\xb1\xf7\xaf\xe6\xe8\xb0`\x06b\x19Ice\xc0\x0e߾eH\xf3!5\xf6e!UM\xd1>\x8f\x86\x1egOwA\xd29\xc5\xd0\x11\x97\xcav\x17z|O?\x18\xb6\x13\xda\xd8\x16\xe1\x05P\x85\x99\xc8z\x1a\x1d\xde\x19\xf1\xa9\xfc\xa4\xf5\xd9\xe1\xe9\x17\u05fa\xb3$\xb9WO>\xc1sIP\x1e\x88\xbf\xe7\a`bDŽe 3UKZ,Cu\x81\xdd,\x80\xe8\x98\xe8\x8cI\xa2\xcd\xec4\x96u\x99N\x90\x15I\xa7\x90\xb3+k\xdd&?q\x91\xb6\xb2\xc5\xcec\xab\x9dJ\xa2\x1c+\xfd\xccP\x9fM\xd9\xcd\xe4-\xf9wQ\xd6%\xe3%\xb2eI̹sy\x98!\xed\xd7\xf1\xfa\x89\v\xebOS\xb8M\xd9e\xda4SeU\x80\x85\x90a\x99)iD\x0e\x8d\xfb\xe0\xf9?\x9a\xaf\x1a+\x9c\xed\xb8(j\xbd@G/\xe6\xccҘϫ\xa7\x97\x0f\xe4\xd2\x11Y\x111\x13\x17\xec\x178\xdc\xf3\xf6\xa3\xd2\xcb\\\xe6[\r/\xef\x9aVZ(ʁ\x9d\xf1Nga\x92\xf7\xda\xf7N\xbd\xf0ry\x8c\xb9\xa7\xb3P\t\x937\xf7\xb4)o\xee\xe9\x9b{\xfa\xe6\x9e\x0eʛ{\xfa枾\xb9\xa7\xe3\xe5\xcd=\xed\x947\xf74\xd9~\xa4`\xb8\xa2\x95ۉ\nIX%\xa6o̡=ӗ\xcfR\xf2gA\x96dWo\xc6[\x8e\x9c\x05Zt\x86\xc4t\x8c^\x93n\x8dS2L&w\xa64\xc1\v\x7f\x81\xb36\x01\x81\xb3\xcf\xdal&\x01\xbc\xe0Y\x1b\x8f\xe9p\xed\xfc\x05O\xda\x04Z,?\x84q\xe5ӘJ\xe0aK\xc8\xe5\xa0\xe4\xb1nc^l\x0f\x8f\xd1:\xbfq\xd6\xfdI\xb6\xe6\xf9\"\xf3\xffr~'\"6'\xa7S#P\x85A\xb9\xfacp\xe2,\xdaG\xa9\xed\xfe\x17\x1b]KX\xa7x\xdd5\x03\xddT\xcb~\xca\xeb\x1fG\xb0ϑ\xe4\xd4\xf371\xe7<\xae\xdb:\xc4Խ\xf3\x1e\xbfoZZ(\xbfTޒ\xa5\x9f{ߌ4{\xc6\xc9wn\x8e2\xdbk%Um\xfc\n\x0f\xf6\xf0!sW\x01\x84\x8e\xcc\x12e\xf0\x9e\xedU\x1d9\xe31Cׄ\xcc\xdbx\xbe\xad\xcf\xe0\x00\xcb\x0f\xef\xd7\xfd_\xac\xf2ٷ\x11\xac\x9f\x84ݻ\xfb\x18x\x9e\xa3\xa3\xde9\xe2\x13&\xaf\xbf\x93e(x\x11\x88J3)\n'\x95\x01B߀~\xa9ܒ\xdf\xd9~\xcb\xfc\xc2Sz\x8e\xee\xd2\xcc\xdc&\x97r\xdeK~F>\xee\xb2\xc3R\xb3\xb9\xb7)H\xb3\x94\x8c\xdb\xf1\\\xda\x19\xa8K\xf2lS\xd7\x14\x13rj\xd33i\xd3\xc8\xc3\xe8&\xa5\xd4\xfc\xd9\xe4(45W\xf6u2d\x13\xf3b;ٮ\xb3 \xcf̆M&XZ\xe6kr\xbek'\x8bu\x9eZ\x13Y\xae㹫\xb3 \xc7r[S2V\x93pM\xceSm\xb2O\xe7wF\x9e\x95\x9d\xfa\xf2\xe7`^r\xddb:\xd74)\xc34imc\x1e\xe7\xa4\x1cҥ\x99\xa3IT]\x9a%\xdad\x80Nt\x9c\x94\x1bz\x9a\xf795\x94ٌ\xd0x\xb6\xe7\x14ر<Є\x1c\xcf\t\x90\xdd\xec\xcf\xc5n\xc0\xac4\xcdVX\x9a\xbb9~?Z(\xf3ֹ\xf8-d\xf6\xb9dR\xba\xe74\xa7\x04w_\x06MPZ\x82\x9f8\xe6\x88\xc7Ce瞟\xe1\x88G@nv\xac\xac\v+\xaa\xa2sA\x99\xddñ\xb9\xf2\xe7WE\a\u05f7G\x82\xf6\xe5k#\xf21\x90\xfd\x90\x82\x1b\xf6\x04E\x81\xff\x9eP!s\xd7\x01fj\x05h\xa5\xe2\x1b\x81\xfe\xaa#\x7f\x97\xe0\x95[\x16\xa3S\xfdd\x01K\x844}\x01֤)\x99v\x8f\x9dWO\xdf\xfeR\x83>2\xbas+\xf8AQ1kO{\xfa\xc9l0&\f\xca\xc7k1w)e_\x19\xc5gC\xa3\x02\xd8\a\xe9\f\xf3\x10W\x82\x85Z\xa7\r\xa7\xa6\x94-FO1\x10R5\x10\"\xedS\xbc\xef%\xc7\x1f_#\xb8z\x89\xf0*\xc9\x11y\x8d\x10뵂\xac\xa5a֒䍤㋯\x11l-\t\xb7\x16\xf9\x8c\xe9\xc7\x13_\xebX\xe2+\x84]g\a^\x8bH\x97z\xecpq\xf8\x950\xbe\x99c\x86'>Z\x02\xc8\xe8\xf1\xc2\xf1\x10,\x01\xe2ɱ\xc2\xd9 ,e\x1e\fôg\x1f\x12LNdZ\xb4\x9b\x9e\x9a\x84\x94\xb6\xd1=\x7f\xf8/\xf1\xd0_\xe26x\n\xf6\x89\x87\xfb\x96\x1f\xeaK\xa4\xf3\x99\xe1\xd9d\u05c9\x87\xf7\x16\x05hg\x86h\x93\x10\xa7\x0e\xebM\ai\xd3\vp\xc3Czg\xb8\x13\t\x12\x96Pe\xf9A\xbbgo\xc6(\x9d\x83\x9e\xdd\xd7Z\"γ\x82<\x88\xa3\xfa\xfd\x0fvt\u008d\xa8X\xab\xbbg\x16\xe3\xa8j\xee\x1d\xc9؟\x84\xf4\xbb\xf5(\xb8\x1d\x9f\xa4\xb7\xf1\xd6:L\xf1}\x9d\xd6K\xf5\x17\xab\xbb\x1d;\x03\x15״\x8f\xbf=\xba\xa4 \xb3f\x9fx\xb6oz\x88\x80\xa4~\xf7ܰ\x9d\xd2%\xb7\xec\xb2\xd9\n}\xe7:\xc0\xbf/\u05cc\xfd\xa4\x9a\xf4\x91νb\x11\xa8F\x94UqĈ\x89]v\xc1ti^\xd6ɂ\x88\xc5(\xd8\\\xf8\v\v\a\x97\n\xefTQ\xa8\xa73\x97.x%\xfe\x8b\xde1I[\x1b\xfbp\xbb\xa1\xeaA\xaa\xe8\r\x94&{\xae\x91\xb1-L+\xf4v\xe0\xe4zt\xa1\x8ed\xaf6\x7fN@\xa4W\x04\x82\x9f\xe1\xd5x\xa6P\x8b\xddn\x1c\x96k\x12,.\x8fL\xf9{\xe2\x85\xceW\x15\xd7\xd1M=\xe6\xe5\xc1\\\xf50\fv|n\x05kҬ\x9d\xbe\x8a\xd0-=\x9a\x87\a\x12h\xb3\xf7X\xf5\xb7щ\xd2\x1dz>\a\xa7\xe9\x83˳G\x96_\x01\xa7i\x97iET\x8c\xfc\x14M\xc7{\xf1\xd5C\xe3/\x91\xffE\x1d\xe0ct\x15\xb1\xffd\xc0\xa0\xc9H\x02]\x80:umz\x9b5\x17\xbf\xce\xfa\x052\xe2\x02*\xfe\xe2\xeb\x05\xe3\xf3-\xc6^i\xf0\xf7\x7f\a\xd8\x13\xb6\r\xa7\xec\xed7\n7\x1bu\xe9g\xb8\x0f&\xc3R\xe1\xe0\xaa\xd6\b\xc8\xd83\v/E-\xab4\x7f\x80\x9f\x95{\t#\x85Z\xfd\x16\xbd\xc7P\xbc3\x17\xb2\x89\xfd\\\x8b)s?\xb6!\xc0\xf6\x90\xc1\xc9u\xf4\x88홷\xed[[$\f\xee\xfe\xfeg7 +JX\x7f\xac]\x86\t\xea]\x03H\xe90P\xd7h\x1b\xd7N{\xf5D\xf7\xb9w\x9f\xab\xe8<\b\x04t\xa8\x81\xd2F\xcf\x1a͡\xf7 D ]\x8a\xb0\x7f\x1bo\xd9\xf1\xa1:L\x9cJ!S\xbb(,n\x8c\xca\x04\xb9]\xb4fNg\t^\xef&\xe2)\xf7yB\x7f\xd6\x06\xbe{\xd6.\x87\xce>\xb26¿\xe6Y\x9d\xe8;3.\x90p\x8f\xa0\xad\x10\xfey\xec\x1c\x9d\at\x8d\xf5\xdc\x13DX\xa79\a\xe6\tM\r\xc3\xf5\xd7w1\xd4\xc7\x0f\xf6\xac\xd8g8u\xe4W\xec\x93\xc4A\x9c\xdawwz\arZD\x1d{\x90lr\x88\x87\xa6\x15\x1d\x9d\x1a\xd1\x16}57\xa8>H\xec\xa4\xe7L\x9a*\xee\x98\xd4\x18[\xffY\xec\xdc\nw\x86c\xfa\x97\x93\x1aQ\xc55\xa9\xb4b\nktJ\x9d|4\xa0\x0f\xf4~H\x10\x12oû_\xeam{\x1b9\xfb\xeb\xdf.\xfe/\x00\x00\xff\xff\x80\xea<õr\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4V\xcfo\xeb6\f\xbe\xe7\xaf \xb0û\xcc\xce\xebv\x19r\x1b\xba\x1d\x8am\x0fE\xf3л\"\xd3\tWY\xf2H*]\xf6\xd7\x0f\x92\xec&\xb1\x9d\xb5\x1b0\xdd\"\xf1\xc7Ǐ\xe4\xe7TU\xb52==#\v\x05\xbf\x01\xd3\x13\xfe\xa9\xe8\xd3/\xa9_~\x90\x9a\xc2\xfax\xb7z!\xdfl\xe0>\x8a\x86\xee\t%D\xb6\xf8\x13\xb6\xe4I)\xf8U\x87j\x1a\xa3f\xb3\x020\xde\a5\xe9Z\xd2O\x00\x1b\xbcrp\x0e\xb9ڣ\xaf_\xe2\x0ew\x91\\\x83\x9c\x83\x8f\xa9\x8f\x9f\xeb\xbb\xef\xea\xcf+\x00o:܀ \xa775\x1a\x85\U0004f222R\x1f\xd1!\x87\x9a\xc2Jz\xb4)\xfe\x9eC\xec7p~(\xfeC\xee\x82{\x9bCms\xa8\xa7\x12*\xbf:\x12\xfd\xe5\x96ů4X\xf5.\xb2qˀ\xb2\x81\x1c\x02\xeb\x97s\xd2\nD\xb8\xbc\x90\xdfGgx\xd1y\x05 6\xf4\xb8\x81\xec\xdb\x1b\x8b\xcd\n` $Ǫ\x06.\x8ew%\x9c=`gJ\x12\x80У\xff\xf1\xf1\xe1\xf9\xfb\xed\xd55@\x83b\x99zʹ.T\x06$``@\x01\x1a\xc0X\x8b\"`#3z\x85\x82\x12ȷ\x81\xbb\xdcɷ\xd0\x00f\x17\xa2\x82\x1e\x10\x9e3\xe5Ce\xf5\x9bIϡGV\x1a\xd9\x18\xdc\xceCvq;\xc1\xfa)\x95S\xac\xa0IӅ\x923\r\x94`30\x00\xa1\x05=\x90\x00c\xcf(\xe8u\x8a2\xf3ӂ\xf1\x10v\xbf\xa3\xd5z\xe0AR\xb3\xa2k\xd2P\x1e\x91\x15\x18m\xd8{\xfa\xeb-\xb6$BRRgt\x9c\x93\xf3!\xaf\xc8\xde88\x1a\x17\xf1[0\xbe\x81Μ\x801e\x81\xe8/\xe2e\x13\xa9\xe1\xb7\xc0\x98\xc9\xdc\xc0A\xb5\x97\xcdz\xbd'\x1d\x97ˆ\xae\x8b\x9e\xf4\xb4\xce{B\xbb\xa8\x81e\xdd\xe0\x11\xddZh_\x19\xb6\aR\xb4\x1a\x19צ\xa7*C\xf7y\xc1\xea\xae\xf9\x86\x87u\x94OWX\xf5\x94&K\x94\xc9\xef/\x1e\xf2B\xfcC\a\xd2:\x94\xf9(\xae\xa5\x8a3\xd1\xe9*\xb1\xf3\xf4\xf3\xf6+\x8c\xa9s3\xa6\xecg\xdeώrnA\"\x8c|\x8b\\\x9a\xd8r\xe8rL\xf4M\x1fȗ鲎\xd0O闸\xebHe\x9c\xddԫ\x1a\xee\xb3\xe2\xc0\x0e!\xf6\x8dQljx\xf0po:t\xf7F\xf0\x7fo@bZ\xaaD\xec\xc7Zp)\x96S\xe3\xc2\xda\xc5\xc3(s7\xfa\xb5\xb0\xdd\xdb\x1em\xea`\"1ySK6\xaf\a\xb4\x81\xc1,\xb9\xd4\x1fB\x92=\xfe%\x96AI\n\x9a\x89\xbe\xa4\xfd|\x1fͲ\x9c䗃\x11\x9c^N0=&\x9bi~G-ړuXB\x145\xc1\xf7\xa1\xa4\x83>v\xf3\x9c\x15|\xc1ׅ\xdbG\x0eIY\xb3\xae_\x9f\x1b\xb3\x01\xe5{\xb3'?+wZY\xb1\xca߰K\xa9\xbe\x10\xe8!\x10p\xf4>\xed\xedL!3\x90\xa9\x92\xcflH\xb1[@\xb3\x88\xe7\xc1\xb7!\x7f\xf0MJl\xb4\xec\x13\x0e\xcd\x1e\xf2\x14\\\v\x01o\xf7\xba\x9c\xb9x}\x88\xd0r\xf2\x97\xf4\xbf9'\xb9!\xc6\xc5\xdcUF\xb5\xf8\x902.1\xbe\xbc_\x03\xca\xe8\x9c\xd99܀r\x9c{\x17_\xc3lNө\x19G\xed+u(j\xba\xfe\xbd\x01\x9a9\xa4=y=\xa0\xbf\xb5\r\xf0j\xa6*\x7f\x95\x19v\xa7[\xae\xf7o\xff\x01\xe7+UFw\x03I\xbb+\xa5\x05\xce>D\xcab\xf7\xcaH/\xfe\xf3\x98\x11\xb2\xbd\xb4\x1d5\xe3j5\xc6?\"\xf3\x1anBXl\xf6\xec2\x87o.\xca\x13\rl\xf6c\xc1\x7f\a\x00\x00\xff\xff\xb1J-\xe7\xa6\v\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4VA\x93\xdb6\x0f\xbd\xfbW`&\x87\xbdDr\xf2}\x97\x8e/\x9d̦\x87L\x93f'N\xf7N\x8b\x90\x8d\x9a\"U\x10\xd4\xc6\xfd\xf5\x1d\x90\xd2\xdak\xcb\xc9n\xa7\xd5\xc5c\n\x04\x1f\xde\xc3\x03UU\xd5\xc2\xf4t\x8f\x1c)\xf8\x15\x98\x9e\xf0\x9b\xa0\xd7\x7f\xb1\xde\xff\x14k\n\xcb\xe1\xedbOޮ\xe06E\t\xdd\x17\x8c!q\x83\xef\xb1%OB\xc1/:\x14c\x8d\x98\xd5\x02\xc0x\x1f\xc4\xe8rԿ\x00M\xf0\xc2\xc19\xe4j\x8b\xbeާ\rn\x129\x8b\x9c\x93OG\x0fo\xea\xb7\xff\xab\xdf,\x00\xbc\xe9p\x05Cp\xa9\xc3\xe8M\x1fwA\\hJ\xcez@\x87\x1cj\n\x8b\xd8c\xa3Gl9\xa4~\x05\xc7\x17%\xc5x|\x81~\x9f\xb3\xad\xc7l\x1f\xc7l9\xc0Q\x94_\xbf\x13\xf4\x91\xa2\xe4\xc0\xde%6\xee*\xb2\x1c\x13w\x81\xe5\xb7\xe3\xe9\x15\fѕ7\xe4\xb7\xc9\x19\xbe\xb6\x7f\x01\x10\x9b\xd0\xe3\n\xf2\xf6\xde4h\x17\x00#?9]5Q\xf3\xb6dlvؙr\x0e@\xe8ѿ\xbb\xfbp\xff\xff\xf5\x93e\x00\x8b\xb1a\xea%\xb3<_\"P\x04\x03\x13\x12x\xd8!#\xdcg>!J`\x8c#\xe8Ǥ\x00\x13\xfeX?.\xf6\x1czd\xa1\xa9\xf8\xf2\x9c\xf4\xd7\xc9\xea\x19\xae\x1b\x85^\xa2\xc0jca\x04\xd9\xe1T>ڱZ\b-Ȏ\"0\xf6\x8c\x11\xbd\x1c\x85<>\xa1\x05\xe3!l\xfe\xc0FjX#k\x1a\xd5&9\xab\xfd8 \v06a\xeb\xe9\xaf\xc7\xdc\x11$\xe4C\x9d\x11\x1c5?>\xe4\x05\xd9\x1b\a\x83q\t_\x83\xf1\x16:s\x00F=\x05\x92?ɗCb\r\x9f\x02#\x90o\xc3\nv\"}\\-\x97[\x92\xc9WM\xe8\xba\xe4I\x0e\xcbl\x11\xda$\t\x1c\x97\x16\at\xcbH\xdb\xcap\xb3#\xc1F\x12\xe3\xd2\xf4Te\xe8\xbe\xf8\xa0\xb3\xafxtb\xbcy\x82U\x0e\xdaEQ\x98\xfc\xf6\xe4E6\xc2w\x14P\x0f\x94F([K\x15G\xa2uI\xd9\xf9\xf2\xcb\xfa+LGg1\xce\xd9ϼ\x1f7ƣ\x04J\x18\xf9\x16\xb9\x88\xd8r\xe8rN\xf4\xb6\x0f\xe4%\xffi\x1c\xa1?\xa7?\xa6MG\xa2\xba\xff\x990\x8ajU\xc3m\x1e6\xb0AH\xbd5\x82\xb6\x86\x0f\x1enM\x87\xee\xd6D\xfc\xcf\x05P\xa6c\xa5\xc4>O\x82\xd39y\x1e\\X;5\xd88ޮ\xe85\xef\xe4u\x8f\xcd\x13\x03i\x16jitv\x1b\xf8\x8cW3\xf9|>_\xfd$|\xde\xe0P\x86|K\xdb\xf3U\x00cm\xbe\"\x8c\xbb\xbb\xba\xf7;\x84\xcd\xd4}\x9bO\xd2Fm\x03+\xa2\x81,r5\xd59\"I<\x16L\xe8l\xac/R^\xe1<\x97\xc2hUc\xe3.\x81>E\xf2\x18\x98\xef8C\xbeP~L\x90[\x8f\xbbq\xc6zAo\xf3P\xbf@\x13r\x0fG\xb4\xf0@\xb2+\xe6p\xa7\x97\xd4\xf3T\xd0g\x8f\x87\xb9\xe53\xec_w\xa8\x91e\x9c\"Dl\x18EqDtj^uf\r\xf0)\xc5l/3\x9b\x11tD\x90\x9dv\xef\xf1pI4\xfcH\xdc\xf1\xbe\xff1\xe4\x1b\xbd\x17'\xc0\x8c-2z\x99\xb5\xb8~b\xb0G\xc1\xecr\x1b\x9a\xa8\x06o\xb0\x97\xb8\f\x03\xf2@\xf8\xb0|\b\xbc'\xbf\xad\x94\xf0\xaa4B\\\xe6\xef\x86\xe5\xab\xfcs\xa5䯟\xdf\x7f^\xc1;k!\xc8\x0eYUk\x93\x9b\x1a\xed\xe4\xb6{\x9d'\xeekHd\x7f\xbe\xf9'\xbc\x84\xbe8\xe7\x19ܬs\xf7\x1f\xf4\xe6Π\x94\xa2uQ%0\xe8\xdcT\xb1\xbbQ\xcd2\x1f\xe6\x1aq´\t\xc1\xa1\xb9l=\x9d\xbe\xc4h/!Uz\xc2Kl\x06\xf0\xad:\nUu\xa6\xafJ\xb4\x91\xd0Qs\x16=\xf9\xfc\a\x96\xbc\x1b\xc3t<(\aӶ\xa9m\xcaWL\xfe\xa61[\xbc6\x16f\x14\x99/\xbcz<\xe0Y\x03]\x8c\xa4\xf8\U000917b7\x8d\x91\x9bq\xac7\x89\xb5\xfdǜ3\x9f?\xff\xceX\xefw&\xcex\xf3\x19\xa8\xeft\xe7$\x83\xa3\x16\x9bC\xe3\xb0$\x84\xd0\xce\xf4ދ \xeb\x83>us\x8d\xf8n0\xe4\xcc\xc6\xe1̻߽\xb9\xfa\xf6\xaa\xf8\xb3z^,F\xfdƱ+\x10N%\xf7\xd8e\xe3\xca\xdf\x01\x00\x00\xff\xff\xec\xa0\xe0\xa1k\r\x00\x00"), } diff --git a/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml b/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml new file mode 100644 index 000000000..20d604153 --- /dev/null +++ b/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml @@ -0,0 +1,169 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.0 + name: datadownloads.velero.io +spec: + group: velero.io + names: + kind: DataDownload + listKind: DataDownloadList + plural: datadownloads + singular: datadownload + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: DataDownload status such as New/InProgress + jsonPath: .status.phase + name: Status + type: string + - description: Time duration since this DataDownload was started + jsonPath: .status.startTimestamp + name: Started + type: date + - description: Completed bytes + format: int64 + jsonPath: .status.progress.bytesDone + name: Bytes Done + type: integer + - description: Total bytes + format: int64 + jsonPath: .status.progress.totalBytes + name: Total Bytes + type: integer + - description: Name of the Backup Storage Location where the backup data is stored + jsonPath: .spec.backupStorageLocation + name: Storage Location + type: string + - description: Time duration since this DataDownload was created + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v2alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: DataDownloadSpec is the specification for a DataDownload. + properties: + backupStorageLocation: + description: BackupStorageLocation is the name of the backup storage + location where the backup repository is stored. + type: string + cancel: + description: Cancel indicates request to cancel the ongoing DataDownload. + It can be set when the DataDownload is in InProgress phase + type: boolean + dataMoverConfig: + additionalProperties: + type: string + description: DataMoverConfig is for data-mover-specific configuration + fields. + type: object + datamover: + description: DataMover specifies the data mover to be used by the + backup. If DataMover is "" or "velero", the built-in data mover + will be used. + type: string + operationTimeout: + description: OperationTimeout specifies the time used to wait internal + operations, before returning error as timeout. + type: string + snapshotID: + description: SnapshotID is the ID of the Velero backup snapshot to + be restored from. + type: string + sourceNamespace: + description: SourceNamespace is the original namespace where the volume + is backed up from. It may be different from SourcePVC's namespace + if namespace is remapped during restore. + type: string + targetVolume: + description: TargetVolume is the information of the target PVC and + PV. + properties: + namespace: + description: Namespace is the target namespace + type: string + pv: + description: PV is the name of the target PV that is created by + Velero restore + type: string + pvc: + description: PVC is the name of the target PVC that is created + by Velero restore + type: string + required: + - namespace + - pv + - pvc + type: object + required: + - backupStorageLocation + - operationTimeout + - snapshotID + - sourceNamespace + - targetVolume + type: object + status: + description: DataDownloadStatus is the current status of a DataDownload. + properties: + completionTimestamp: + description: CompletionTimestamp records the time a restore was completed. + Completion time is recorded even on failed restores. The server's + time is used for CompletionTimestamps + format: date-time + nullable: true + type: string + message: + description: Message is a message about the DataDownload's status. + type: string + phase: + description: Phase is the current state of the DataDownload. + enum: + - New + - Accepted + - Prepared + - InProgress + - Canceling + - Canceled + - Completed + - Failed + type: string + progress: + description: Progress holds the total number of bytes of the snapshot + and the current number of restored bytes. This can be used to display + progress information about the restore operation. + properties: + bytesDone: + format: int64 + type: integer + totalBytes: + format: int64 + type: integer + type: object + startTimestamp: + description: StartTimestamp records the time a restore was started. + The server's time is used for StartTimestamps + format: date-time + nullable: true + type: string + type: object + type: object + served: true + storage: true + subresources: {} diff --git a/config/crd/v2alpha1/bases/velero.io_datauploads.yaml b/config/crd/v2alpha1/bases/velero.io_datauploads.yaml new file mode 100644 index 000000000..ea354f1d4 --- /dev/null +++ b/config/crd/v2alpha1/bases/velero.io_datauploads.yaml @@ -0,0 +1,193 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.0 + name: datauploads.velero.io +spec: + group: velero.io + names: + kind: DataUpload + listKind: DataUploadList + plural: datauploads + singular: dataupload + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: DataUpload status such as New/InProgress + jsonPath: .status.phase + name: Status + type: string + - description: Time duration since this DataUpload was started + jsonPath: .status.startTimestamp + name: Started + type: date + - description: Completed bytes + format: int64 + jsonPath: .status.progress.bytesDone + name: Bytes Done + type: integer + - description: Total bytes + format: int64 + jsonPath: .status.progress.totalBytes + name: Total Bytes + type: integer + - description: Name of the Backup Storage Location where this backup should be + stored + jsonPath: .spec.backupStorageLocation + name: Storage Location + type: string + - description: Time duration since this DataUpload was created + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v2alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: DataUploadSpec is the specification for a DataUpload. + properties: + backupStorageLocation: + description: BackupStorageLocation is the name of the backup storage + location where the backup repository is stored. + type: string + cancel: + description: Cancel indicates request to cancel the ongoing DataUpload. + It can be set when the DataUpload is in InProgress phase + type: boolean + csiSnapshot: + description: If SnapshotType is CSI, CSISnapshot provides the information + of the CSI snapshot. + nullable: true + properties: + snapshotClass: + description: StorageClass is the name of the snapshot class that + the volume snapshot is created with + type: string + storageClass: + description: StorageClass is the name of the storage class of + the PVC that the volume snapshot is created from + type: string + volumeSnapshot: + description: VolumeSnapshot is the name of the volume snapshot + to be backed up + type: string + required: + - storageClass + - volumeSnapshot + type: object + dataMoverConfig: + additionalProperties: + type: string + description: DataMoverConfig is for data-mover-specific configuration + fields. + nullable: true + type: object + datamover: + description: DataMover specifies the data mover to be used by the + backup. If DataMover is "" or "velero", the built-in data mover + will be used. + type: string + operationTimeout: + description: OperationTimeout specifies the time used to wait internal + operations, before returning error as timeout. + type: string + snapshotType: + description: SnapshotType is the type of the snapshot to be backed + up. + type: string + sourceNamespace: + description: SourceNamespace is the original namespace where the volume + is backed up from. It is the same namespace for SourcePVC and CSI + namespaced objects. + type: string + sourcePVC: + description: SourcePVC is the name of the PVC which the snapshot is + taken for. + type: string + required: + - backupStorageLocation + - operationTimeout + - snapshotType + - sourceNamespace + - sourcePVC + type: object + status: + description: DataUploadStatus is the current status of a DataUpload. + properties: + completionTimestamp: + description: CompletionTimestamp records the time a backup was completed. + Completion time is recorded even on failed backups. Completion time + is recorded before uploading the backup object. The server's time + is used for CompletionTimestamps + format: date-time + nullable: true + type: string + dataMoverResult: + additionalProperties: + type: string + description: DataMoverResult stores data-mover-specific information + as a result of the DataUpload. + nullable: true + type: object + message: + description: Message is a message about the DataUpload's status. + type: string + path: + description: Path is the full path of the snapshot volume being backed + up. + type: string + phase: + description: Phase is the current state of the DataUpload. + enum: + - New + - Accepted + - Prepared + - InProgress + - Canceling + - Canceled + - Completed + - Failed + type: string + progress: + description: Progress holds the total number of bytes of the volume + and the current number of backed up bytes. This can be used to display + progress information about the backup operation. + properties: + bytesDone: + format: int64 + type: integer + totalBytes: + format: int64 + type: integer + type: object + snapshotID: + description: SnapshotID is the identifier for the snapshot in the + backup repository. + type: string + startTimestamp: + description: StartTimestamp records the time a backup was started. + Separate from CreationTimestamp, since that value changes on restores. + The server's time is used for StartTimestamps + format: date-time + nullable: true + type: string + type: object + type: object + served: true + storage: true + subresources: {} diff --git a/config/crd/v2alpha1/crds/crds.go b/config/crd/v2alpha1/crds/crds.go new file mode 100644 index 000000000..00f9ec3fd --- /dev/null +++ b/config/crd/v2alpha1/crds/crds.go @@ -0,0 +1,60 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by crds_generate.go; DO NOT EDIT. + +package crds + +import ( + "bytes" + "compress/gzip" + "io" + + apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/client-go/kubernetes/scheme" +) + +var rawCRDs = [][]byte{ + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcX\xcdr\xe4\xb8\r\xbe\xfb)P\x93\x83/#yg\x93J\xa5\xfa6\xd3N\xaa\\ٙt\xad\xa7\xfaNI\x90\x9a;\x14\xc9\xf0\xa7\x1d'\x95wO\x81?jI\xcdvۻ\x99\xd5M$\b|\x04\xc0\x0f \xab\xaa\xbaa\x9a\xef\xd1X\xae\xe4\x06\x98\xe6\xf8/\x87\x92\xfel\xfd\xed/\xb6\xe6\xea\xee\xf8\xe1\xe6\x1b\x97\xdd\x06\xb6\xde:5\xfe\x8cVy\xd3\xe2=\xf6\\rǕ\xbc\x19ѱ\x8e9\xb6\xb9\x01`R*\xc7h\xd8\xd2/@\xab\xa43J\b4Հ\xb2\xfe\xe6\x1bl<\x17\x1d\x9a\xa0<\x9b>\xfeP\x7f\xf8\xb1\xfe\xe1\x06@\xb2\x117@\xfa:\xf5$\x85b\x9d\xad\x8f(Ш\x9a\xab\x1b\xab\xb1%ŃQ^o\xe04\x11\x17&\xa3\x11\xf0=s\xec>\xe9\bÂ[\xf7\xf7\xb3\xa9\x9f\xb8uaZ\vo\x98X\xd9\x0e3\x96\xcb\xc1\vf\x96s7\x00\xb6U\x1a7\xf0\x85Lk\xd6\"\x8d\xa5=\x05(\x15\xb0\xae\v^bbg\xb8th\xb6J\xf81{\xa7\x82\x0emk\xb8v\xc1\vsX`\x1dsނ\xf5\xed\x01\x98\x85/\xf8t\xf7 wF\r\x06m\x84\x05\xf0\x8bUr\xc7\xdca\x03u\x14\xaf\xf5\x81YL\xb3ѕ\x8fa\"\r\xb9g\xc2k\x9d\xe1r(!\xf8\xcaG\x84Λ\x10B\xdaw\x8b\xe0\x0e\xdc.\xa1=1K\xf0\x8c\xc3\xee\"\x900O\xea\xacc\xa3^#\x9a-\x8d\x90:\xe6\xb0\x04h\xabF-\xd0a\aͳü\x8d^\x99\x91\xb9\rp\xe9\xfe\xfc\xa7˾HΪ\xc3\xd2{%\x97\x8e\xf9D\xa30\x1b\x8eH(J\x03\x9a\xa2w\x94c\xe2\xb7\x00q\xa4\xe0\xd3l}D\x12\xf5\xceǯB\xa1\x94\x03Ճ; |b\xed7\xaf\xe1\xd1)\xc3\x06\x84\x9fT\x1b\xc3\xf7t@\x83A\xa2\x89\x12\x94\xbd\xc0)v\xca\x14C\xa7\xb1\xad\xa3lR\x96u\xad\xe2\xb74\xf4\x7fϭ\xd6 +\xe6V\xa6\x9a:Hp%\xcb\t\xf6q\xc0rr\xc5\xe9\xe3\x8fL\xe8\x03\xfb\x10\xcfv{\xc0\x91m\x92\xbc\xd2(?\xee\x1e\xf6\x7f|\\\f\x03h\xa34\x1a\xc73\xc5\xc4oƞ\xb3QX\xee\xfb\x96\x14F)\xe8\x886ц\xa0$\xa2\xc0.a\x88\xe1\xe4\x16\fj\x83\x16\xa5\x9b{7\x7f\xaa\a&A5\xbf`\xebjxDCj\xc0\x1e\x94\x17\x1d\xb1\xed\x11\x8d\x03\x83\xad\x1a$\xff\xf7\xa4ۂS\xc1\xa8`\x0e\x13ߝ\xbe@L\x92\t82\xe1\xf1=0\xd9\xc1Ȟ\xc1 Y\x01/g\xfa\x82\x88\xad\xe1\xb32\b\\\xf6j\x03\a\xe7\xb4\xdd\xdc\xdd\r\xdc\xe5\xaaѪq\xf4\x92\xbb\xe7\xbbP\x00x\xe3\x9d2\xf6\xae\xc3#\x8a;ˇ\x8a\x99\xf6\xc0\x1d\xb6\xce\x1b\xbcc\x9aW\x01\xba\f\x95\xa3\x1e\xbb?\x98Tg\xec\xed\x02\xebY\x8e\xc5/\x10\xfe\v\x11 ֧\xc4gii\xdc\xc5\xc9\xd14D\xde\xf9\xf9\xaf\x8f_!\x9b\x0e\xc1X{?\xf8\xfd\xb4ОB@\x0e\xe3\xb2G\x13\x83\xd8\x1b5\x06\x9d(;\xad\xb8t\xe1\xa7\x15\x1c\xe5\xda\xfd\xd67#w\x14\xf7\x7fz\xb4\x8ebU\xc36\x94Rh\x10\xbc\xa6\xfc\xedjx\x90\xb0e#\x8a-\xb3\xf8\xdd\x03@\x9e\xb6\x159\xf6u!\x98w\x01k\xe1\xe8\xb5\xd9D.\xe3\x17\xe25'\x84G\x8d-\x85\x8e\xbcG\xcbx\xcf\x13\xbd\xf5\xca\x00[\xc8\xd6\v\x95\xe5#K_\x91\xe2\xd6B+L\x9fJk209c\xe3ĵ6J\x9e)\x05\x10\x17\xf9٠V\x96;e\x9eO,]\x9fi\xb8\x10\x00\xfaZ&[\x14Wv\xb2\rB\xc0eG\x9e\xc4)\xef\x88\"\xa2\x82\x80I\xc9Aѹ\xb8\xec\xe0\xf8=8ZE\x89j\xd1ўdX\xbe un\x81K8\xb5/0oS\xd6;k\x94\x12\xc8ּG\xb9\xf5Y\x1d\xa9\x81\x92=\x1f\xce\xf78\xef\xb4.\x05\xfe\x8a\xfb\ni83I\xbb\xa0\x9c#$\xd5H\xe3UNH\"ޞ\x0f\xa9\xb6\x15\x8c\xf6\x1cEg/\xc5\xf2\xec|\xe4\r\a+W\xc29\xa1\xcc\xc7#\x95\x97P\xec\x83\x02\n,\xf1\x88\r]\x14M\x16\x10\xc6\x14\xacᡟi\xe4\x16\u07bd\x03e\xe0]\xec\xb4߽\x8f\xe9\xea\xb9p\x15\x973\x1b\x05\x8dO\\\x88l\xf7MYLћ\xaa\xbb\xf2\xee\x8a\x03\xfe\xb1\x12_\xf9\xc1Q\xdb\x11\xf6\xee\x14<1\xee\xa6rW\xc0<\x99\xb6\xef\xa1\xc1\x9e(֠\xf3F\xd2I@c\x88rlP\xa9\xbc{Ӧ\xacd\xda\x1e\x94{\xb8\xbf\xb2\x9d\xc7I0\xb3\xcb\xc3}\xe6\x96}\x88\xc2D1I\x12\x9c*\x05\x94\xa0G\x0e\t\xc5\xe8mhC\x05\x9c\xee5\xd7 /\xa53ne\xf8\xc0\xa9\xad\x90\xd3̉\xf2\x8et\x0f*%\"\xb7a\x7f\u0601\xd7\x118Q\fU\xd7\x06\xa1\xe3}\x8f\x06\xa5\x8b\xf55\x1a\xde\xed\xb7\xb7\xf6d\xa4\xa4\xb3\x9fa\b\x1d\xd6ȴƎ\xdaQ\x8alrԛ\\\xe4\x98\x19\xd0\xed\xc36\xae\xf8\xe7\xebL4;\x87*7\xdd\x1d\xa8\x10\xa4\xe8F\x8d\xb0\xdbo\xa9\x03+lc\xb7?Gx\xb9\xcaAjx/D\xf0\f\xe5Y\xfc\x12\x9e\x97\x1c{\x85N\x01\xf4\xf1\x15\x96w\xfbR!\x9d\xdc\x01\xee\xc0\x1cI\xa4{\x014\xcfE\x9d\x90\xcfG\n\xe7\xaf\xc3۾\n\xf0\xf6E\xc4\xdb5\xe4\vx\x9b\xe7\xdf\f\x99\x8a77\u061d\xa3\xae^\x88\\\x05\xfaX\x1cl__\xa2ʖ\xabrw\xb5\x92YS\xfcj\xfaD\x96\xeb\x89%Ӭf\xe7G\xf2Umh\xb8\x9e\xbf\xb6\x11\x8d\xef1)\xec\xad7\x81\x86\xd2+\r\xdd\xca~U+\xda\xc6\xf7\x8d\xf9U\xf6Z\xfbv\xbe\"\xdc\xf7L7\xabw,'T\xbcO\xe7G\x94R\xffv\xd2\x17\x97\x06z$u\xd8\x01\x1eQ\x02\xb5ڌ\v\xec\xb2N[\xc3W\xea\xc6\xc3\xc5\xe7v}E\n\xfeN\x8aB٥\x9e\xa9\x00\xfa|]~L\xa1\xebNE*\xce$\xa4\x17\x825\x027\xe0\x8c\xbf\xd4?\x16\x0fʈֲ\xe1\x1aQ\x7f\x8eR\U0006a616\x00k\xa8\xa9X\xf7\xb4\xb76\xc5\xfeME#\xf4\xbdW@\xecH\xa6\x94g\x13ϼܒ\xa3\xf4c\x89\r\xbe\xe0Sa\xf4cۢ.1T\x05;\x83\x9a\x99\xe2\xd4\xd9K\xe4|2^0Jd\x95\xe7\x8a:\xa7\xa7\xbe\xc2\xdc\xdfB\x02\xbe\xc9\xd3\t\xdf5g\xe7\x1b\xc9A\x89|\x80\xc2k\x9c\xf4c\x83\x86<\x1e\xde\xfb\xb2\xeb33\x15\x92\x9e\xc9n\x11\xb2\x93\x86\xa9\x0f\v\xaa\xe8\xf4Pe\x88\x97\xa6ܙv\xdcj\xc1J\x85-\xefd\xd12\x9c\x922\x1f\xf4\x89R\xdf\xda#L\xaf\xa3\xe5\xc2Wz\xe2,Ea\xfeX\xb9\x9a\x9f^=\xbf\x8f\x85\x17.Q\xcbW\xe8km\xecB\xf8\x1a\xa9\xa6\a\xf0\x12\xa5\xce\xd9\xf1\x9c\v\x97f~O\x1a,:\xeal0 \xeff\xba\xd3S\xc6|\xc47\xd3\x03\xdd\x06\xfe\xf3ߛ\xff\x05\x00\x00\xff\xff\xd8T?\xb3K\x1a\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcY\xcbsۼ\x11\xbf\xfb\xaf\xd8I\x0f\xb9X\xf4\x97\xaf\x9dNG\xb7Dng4\xfd\x92j\"\xd7w\x88\\\x91\x88A\x80\xc5C\xae\xdb\xe9\xff\xdeY<(> \xd1N\x93ꐉ\x01\xecb\x9f\xbf\xdd\x05W\xab\xd5\r\xeb\xf8#jÕ\\\x03\xeb8\xfeӢ\xa4\xbfL\xf1\xf4'Spuw\xfap\xf3\xc4e\xb5\x86\x8d3V\xb5_\xd1(\xa7K\xbc\xc7#\x97\xdcr%oZ\xb4\xacb\x96\xado\x00\x98\x94\xca2Z6\xf4'@\xa9\xa4\xd5J\bԫ\x1ae\xf1\xe4\x0exp\\T\xa8=\xf3t\xf5\xe9\x97\xe2ï\xc5/7\x00\x92\xb5\xb8\x06\xe2\xe7:\xa1Xe\x8a\x13\nԪ\xe0\xea\xc6tX\x12\xdbZ+\u05ed\xe1\xbc\x11\xc8\xe2\x95A\xdc{f\xd9\xdf=\a\xbf(\xb8\xb1\x7f\x9dl\xfcƍ\xf5\x9b\x9dp\x9a\x89ѭ~\xddpY;\xc1\xf4p\xe7\x06\xc0\x94\xaa\xc35|\xa1+;V\"\xadEM\xbc\b+`U\xe5m\xc3\xc4NsiQo\x94pm\xb2\xc9\n*4\xa5\xe6\x9d\xf5\xba\x9f\x05\x02c\x99u\x06\x8c+\x1b`\x06\xbe\xe0\xf3\xddV\ued2a5\x9a \x12\xc07\xa3\xe4\x8e\xd9f\rE8^t\r3\x18w\x83\xf9\xf6~#.\xd9\x17\x92\xd6X\xcde\x9d\xbb\xff\x81\xb7\b\x95\xd3\xdem\xa4s\x89`\x1bn\x86\x82=3C\xc2i\x8b\xd5E1\xfc>13\x96\xb5\xddT\x9e\x01i\x10\xa8b\x16s\xe2lT\xdb\t\xb4X\xc1\xe1\xc5bR\xe2\xa8t\xcb\xec\x1a\xb8\xb4\x7f\xfc\xc3eKDS\x15\x9e\xf4^ɱY>\xd1*\f\x96\x83$\xe4\xa1\x1au\xd66\xca2\xf1\xbf\bb\x89\xc1\xa7\x01}\x90$\xf0\x1d\xae/\x8aB\xe1\x06\xea\b\xb6A\xf8\xc4\xca'\xd7\xc1\xde*\xcdj\x84\xdfT\x19\x9c\xf7ܠ\x8e\xce;\x84#\xa6QNTpH\x1a\x03\x18\xabt\u058b\x1d\x96E\xa0\x8a|\x13ۉ+\xc7w\xfe\xe0 +5\xb2l\x90%\x94)\xfc\t\xaed>\xd2>֘\x8f\xb2\xb0}\xfa\x95\x89\xaea\x1fBz\x97\r\xb6l\x1dϫ\x0e\xe5\xc7\xdd\xf6\xf1\xf7\xfb\xd12@\xa7U\x87\xda\xf2\x84/\xe17\x00\xce\xc1*\x8c\xb5~O\f\xc3)\xa8\b1\xd1x\xffE\xb4\xc0*\xca\x10\xfc\xca\rh\xec4\x1a\x94vh\xdb\xf4SG`\x12\xd4\xe1\x1b\x96\xb6\x80=jb\x93<\\*yBmAc\xa9j\xc9\xff\xd5\xf36`\x95\xbfT0\x8b\x11\xf0\xce?\x8fN\x92\t81\xe1\xf0\x16\x98\xac\xa0e/\xa0\x91n\x01'\a\xfc\xfc\x11S\xc0g\xa5\x11\xb8<\xaa54\xd6vf}wWs\x9b\nF\xa9\xda\xd6In_\xee<\xf6\xf3\x83\xb3J\x9b\xbb\nO(\xee\f\xafWL\x97\r\xb7XZ\xa7\xf1\x8eu|\xe5E\x97\xbeh\x14m\xf5;\x1dK\x8cy?\x92u\x16a\xe1\xe7\xd1\xfe\x8a\a\b\xf4\x81\x1b`\x914hq64-\x91u\xbe\xfey\xff\x00\xe9j\uf329\xf5\xbd\xddτ\xe6\xec\x022\x18\x97G\xd4\xc1\x89G\xadZ\xcf\x13e\xd5).\xad\xff\xa3\x14\x1c\xe5\xd4\xfc\xc6\x1dZn\xc9\xef\xffph,\xf9\xaa\x80\x8d\xaf\xa2p@p\x1d\xc5oU\xc0V\u0086\xb5(6\xcc\xe0Ow\x00YڬȰ\xafs\xc1\xb0\x01\x98\x1e\x0eV\x1bl\xa4\x1a~\xc1_g8\xd8wX\x92\xe3\xc8vDď<\xa2\xdcQi`\x83\x93ň]>]\xe9\x97\x05\xb7顉<\x9fr4I,9\x80䄷\xe1\xe4\x8c)\x80\x98\x82tO\xa3\xb1S\x86[\xa5_\x88q\xc0\xe7b\xc6\xe1\x82\xf1\xe9W2Y\xa2X\xd0d\xe3\x0f\x01\x97\x15\xd9\x11\xfb\x98#x\b\f\xbcLJ֊r\xe2\x92y\xc3ok\x89\x86BԠ%\x8d\xa4'\x1e\x8097\xc0%\x9c\xbb\x17\x18v)S\xad\x0eJ\tdS\xbc+\r\xdfK֙F\xd9\x05ݶGH'\x1f^:\xa4\xcb7\xfb\xed-\xfd\x93\xd6).N\xbc\x8a\x00L\xc9Cu|\x0e\xb2\x10\x80\x96\x0em\xf6[0\x91|n\x04\xe9\x84`\a\x81k\xb0\xda\xcd\x15\xbb\x1c\x86\xf4Kl7\x82\x99쁉\x821\x00\xfd\xf1\\\xf4%~P\xfa\x13\xb6aS\xa4\xe9\rN出\xd1\x01\x11\xef\xeb.\xab\x13\x8d<\xf2\xc8\xeb\xf9\xdd\xc3\xd9\xe8Z\x8a\\UmV3\x06W\x92ũD\x90$\xab\x96\xd6W\xa9~P\x97t\xe4ulC3\x97\x1e9\x8aʼ9\xd9\x17\xec\xe1\x85X\x80\xb0^\x89T\xec\"R\x11=x\x061 \x9c\xf1\xa3\x11mf\x14\b%\xa5 Ds!ҽo\xaaJ\xe4ܾSWn\t\xc3\xff69>\xb1\x83\xa5\x01\xc2\xebn\x15<3n\xfb\xd65\a\xe0\x89\x97\xb9\x85\x03\x1e\xa9]\xd2h\x9d\x96T\xd9Pkj \x8cg\xa9\\\x06گ(e\x06efA\xa1iE\xf2Z\xd0\xff\xa7\x98=L\xf4\x8c2\xae{\x9b\x84\xbe\x83\xed\x1f'\x96\x84\x1c\x9fNr*\xcdkNc\x81\xecw\xcemK\x00\x87\x8c\xa4q\xec\xf4p\xe5\xf1\xb6\xa0F!\xf5p\x04\x80gv\x94\xa1\xe1r\x02p\x1a;6\xfbm\x86gOQ\xc5\xfc\xcad\xe7\xa25v\x8f\x9bWفD\xc9\xe05-?7\xbcl\xc6~\x9b\x8d\b^\x16\xf6\x84\xbeE}\x83\x98y\xa0^\xe5\x1b\xd6əi\x96M\xb6\x87\xf1:\xdd\x1a\xbb>\xbb\xbb{ܼ\xaa\xa9\xf7\xaf\x1e\xafk\xeb\xc3\x13W\xb4r\xe9\xb4Fi\xd3\xc3\x17M\xb8\xdf\xd1ؗ\xe1\xc9h\xf8(\xb0\xd4\f\xcf)\xfc䬫\x01ڰԠ\xfb\x87\x89\xf4,\x95k\x87\xcf\xec\x02\xa5\x9f\xe4\x89\x1bV\x80'\x94@S\vゐ۳4Ŕ&\x9fN=\x97\x88b\xe1\r2ͬQ\xbc\xf4\"\xf0@\xc1\xe9G\xd2\xf7\xe6\nO\x0f\xa2\x94~\x19#\xcc#:\xbdw\xd1 \xba\xca2}Um\xcc&g\xdf+|E\xe3D\xa6@\xfc\xc4^!\\\x19\xa6-\x93\xed\x15\xae\xcf\b\xcc\x00\x03\x1d\x98D\x98\xb862}\x7f\x03Ѣ1\xac^\xc2\xf1\xcf\xe1Tx\xe9\x88$\xc0\x0eTGǢ\xbd71\xd9\xde\x04\xa3\x1d\xb3͂\x04;f\x9b\x94\xd6G'\x84\xa7\x99պ\xd8\xfa\x1e\x90b\xf8G\x95\x1a\xf5\u0093D\xea\x12+n:\xc1^2\x8c\x93\"\xc3\x1c\x1fdK\xc2\xd5TZ\xe7\xf1p}\xba\xef\xbf>\xe4\xa7\xc5\xdc'\x84\x9c\x0f\x86\x1f\x03&\xfb\xfdW\x85\x9fs\xc3\x158J\x99\xbc\xbd\x7fe\xfb\xbb\xbdOY\xc7+\x94\x96:z\xed\xcbϸ\x97\x92W\a\x9a\xc1\x1b\xd9\xdbڿ\xd17\xa9%\x89G\x87\x17\xfa\x81\xf85,\xd7\r\xec)\xc5\tX\xfcC\xf0f\xfa\xbd\xe2\xb6\xff\xfc\xc1l|\x8d.\x1b&kJ\bI%ŗ\xa4\x1c\xe3Y\x81\x1f\x95\xf3\xb1\xf8\xff\xcfJ\x9e\r\x97٢\x97\xbc\x1a\xf0\x8eo\x10\xc3\x15w\xe8_\xff\xd7\xf0\xef\xff\xdc\xfc7\x00\x00\xff\xffgK\fV\xa3\x1e\x00\x00"), +} + +var CRDs = crds() + +func crds() []*apiextv1.CustomResourceDefinition { + apiextinstall.Install(scheme.Scheme) + decode := scheme.Codecs.UniversalDeserializer().Decode + var objs []*apiextv1.CustomResourceDefinition + for _, crd := range rawCRDs { + gzr, err := gzip.NewReader(bytes.NewReader(crd)) + if err != nil { + panic(err) + } + bytes, err := io.ReadAll(gzr) + if err != nil { + panic(err) + } + gzr.Close() + + obj, _, err := decode(bytes, nil, nil) + if err != nil { + panic(err) + } + objs = append(objs, obj.(*apiextv1.CustomResourceDefinition)) + } + return objs +} diff --git a/config/crd/v2alpha1/crds/doc.go b/config/crd/v2alpha1/crds/doc.go new file mode 100644 index 000000000..9eed410f6 --- /dev/null +++ b/config/crd/v2alpha1/crds/doc.go @@ -0,0 +1,4 @@ +// Package crds embeds the controller-tools generated CRD manifests +package crds + +//go:generate go run ../../../../hack/crd-gen/v1/main.go diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 1c71f4ec3..8d235284e 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -82,6 +82,46 @@ rules: - get - patch - update +- apiGroups: + - velero.io + resources: + - datadownloads + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - velero.io + resources: + - datadownloads/status + verbs: + - get + - patch + - update +- apiGroups: + - velero.io + resources: + - datauploads + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - velero.io + resources: + - datauploads/status + verbs: + - get + - patch + - update - apiGroups: - velero.io resources: diff --git a/design/CLI/PoC/base/deployment.yaml b/design/CLI/PoC/base/deployment.yaml index 36cf4d462..20f8ff4e0 100644 --- a/design/CLI/PoC/base/deployment.yaml +++ b/design/CLI/PoC/base/deployment.yaml @@ -57,7 +57,7 @@ spec: - emptyDir: {} name: scratch --- -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: diff --git a/hack/crd-gen/v1/main.go b/hack/crd-gen/v1/main.go index e61763531..5f45b04e0 100644 --- a/hack/crd-gen/v1/main.go +++ b/hack/crd-gen/v1/main.go @@ -14,8 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// This code embeds the CRD manifests in config/crd/v1/bases in -// config/crd/v1/crds/crds.go. +// This code embeds the CRD manifests in ../bases in ../crds/crds.go package main diff --git a/hack/update-1fmt.sh b/hack/update-1fmt.sh index a9d020c30..fef575451 100755 --- a/hack/update-1fmt.sh +++ b/hack/update-1fmt.sh @@ -35,13 +35,11 @@ fi files="$(find . -type f -name '*.go' -not -path './.go/*' -not -path './vendor/*' -not -path './site/*' -not -path '*/generated/*' -not -name 'zz_generated*' -not -path '*/mocks/*')" echo "${ACTION} gofmt" -for file in ${files}; do - output=$(gofmt "${MODE}" -s "${file}") - if [[ -n "${output}" ]]; then - VERIFY_FMT_FAILED=1 - echo "${output}" - fi -done +output=$(printf '%s\n' "${files}" | xargs gofmt "${MODE}" -s) +if [[ -n "${output}" ]]; then + VERIFY_FMT_FAILED=1 + echo "${output}" +fi if [[ -n "${VERIFY_FMT_FAILED:-}" ]]; then echo "${ACTION} gofmt - failed! Please run 'make update'." else @@ -49,13 +47,11 @@ else fi echo "${ACTION} goimports" -for file in ${files}; do - output=$(goimports "${MODE}" -local github.com/vmware-tanzu/velero "${file}") - if [[ -n "${output}" ]]; then - VERIFY_IMPORTS_FAILED=1 - echo "${output}" - fi -done +output=$(printf '%s\n' "${files}" | xargs goimports "${MODE}" -local github.com/vmware-tanzu/velero) +if [[ -n "${output}" ]]; then + VERIFY_IMPORTS_FAILED=1 + echo "${output}" +fi if [[ -n "${VERIFY_IMPORTS_FAILED:-}" ]]; then echo "${ACTION} goimports - failed! Please run 'make update'." else diff --git a/hack/update-3generated-crd-code.sh b/hack/update-3generated-crd-code.sh index 9f9242619..6bd185cc3 100755 --- a/hack/update-3generated-crd-code.sh +++ b/hack/update-3generated-crd-code.sh @@ -40,20 +40,38 @@ ${GOPATH}/src/k8s.io/code-generator/generate-groups.sh \ all \ github.com/vmware-tanzu/velero/pkg/generated \ github.com/vmware-tanzu/velero/pkg/apis \ - "velero:v1" \ + "velero:v1,v2alpha1" \ --go-header-file ./hack/boilerplate.go.txt \ --output-base ../../.. \ $@ # Generate apiextensions.k8s.io/v1 -# Generate manifests e.g. CRD, RBAC etc. + +# Generate CRD for v1. controller-gen \ crd:crdVersions=v1 \ paths=./pkg/apis/velero/v1/... \ - rbac:roleName=velero-perms \ paths=./pkg/controller/... \ output:crd:artifacts:config=config/crd/v1/bases \ object \ paths=./pkg/apis/velero/v1/... +# Generate CRD for v2alpha1. +controller-gen \ + crd:crdVersions=v1 \ + paths=./pkg/apis/velero/v2alpha1/... \ + paths=./pkg/controller/... \ + output:crd:artifacts:config=config/crd/v2alpha1/bases \ + object \ + paths=./pkg/apis/velero/v2alpha1/... + +# Generate RBAC. +controller-gen \ + paths=./pkg/apis/velero/v1/... \ + paths=./pkg/apis/velero/v2alpha1/... \ + paths=./pkg/controller/... \ + rbac:roleName=velero-perms + go generate ./config/crd/v1/crds + +go generate ./config/crd/v2alpha1/crds diff --git a/hack/verify-generated-crd-code.sh b/hack/verify-generated-crd-code.sh index 387acd74e..1ae3cef94 100755 --- a/hack/verify-generated-crd-code.sh +++ b/hack/verify-generated-crd-code.sh @@ -19,7 +19,7 @@ HACK_DIR=$(dirname "${BASH_SOURCE}") ${HACK_DIR}/update-3generated-crd-code.sh # ensure no changes to generated CRDs -if ! git diff --exit-code config/crd/v1/crds/crds.go >/dev/null; then +if [! git diff --exit-code config/crd/v1/crds/crds.go config/crd/v2alpha1/crds/crds.go >/dev/null]; then # revert changes to state before running CRD generation to stay consistent # with code-generator `--verify-only` option which discards generated changes git checkout config/crd diff --git a/pkg/apis/velero/v1/pod_volume_operation_progress.go b/pkg/apis/velero/shared/data_move_operation_progress.go similarity index 80% rename from pkg/apis/velero/v1/pod_volume_operation_progress.go rename to pkg/apis/velero/shared/data_move_operation_progress.go index e5b3344c7..f92b3e533 100644 --- a/pkg/apis/velero/v1/pod_volume_operation_progress.go +++ b/pkg/apis/velero/shared/data_move_operation_progress.go @@ -14,11 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1 +package shared -// PodVolumeOperationProgress represents the progress of a -// PodVolumeBackup/Restore operation -type PodVolumeOperationProgress struct { +// DataMoveOperationProgress represents the progress of a +// data movement operation + +// +k8s:deepcopy-gen=true +type DataMoveOperationProgress struct { // +optional TotalBytes int64 `json:"totalBytes,omitempty"` diff --git a/pkg/apis/velero/v1/backup_types.go b/pkg/apis/velero/v1/backup_types.go index f38f35320..e25e1463d 100644 --- a/pkg/apis/velero/v1/backup_types.go +++ b/pkg/apis/velero/v1/backup_types.go @@ -165,6 +165,16 @@ type BackupSpec struct { // ResourcePolicy specifies the referenced resource policies that backup should follow // +optional ResourcePolicy *v1.TypedLocalObjectReference `json:"resourcePolicy,omitempty"` + + // SnapshotMoveData specifies whether snapshot data should be moved + // +optional + // +nullable + SnapshotMoveData *bool `json:"snapshotMoveData,omitempty"` + + // DataMover specifies the data mover to be used by the backup. + // If DataMover is "" or "velero", the built-in data mover will be used. + // +optional + DataMover string `json:"datamover,omitempty"` } // BackupHooks contains custom behaviors that should be executed at different phases of the backup. diff --git a/pkg/apis/velero/v1/pod_volume_backup_types.go b/pkg/apis/velero/v1/pod_volume_backup_types.go index d34e09f6c..a9a2ad4e8 100644 --- a/pkg/apis/velero/v1/pod_volume_backup_types.go +++ b/pkg/apis/velero/v1/pod_volume_backup_types.go @@ -19,6 +19,8 @@ package v1 import ( corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" ) // PodVolumeBackupSpec is the specification for a PodVolumeBackup. @@ -100,7 +102,7 @@ type PodVolumeBackupStatus struct { // number of backed up bytes. This can be used to display progress information // about the backup operation. // +optional - Progress PodVolumeOperationProgress `json:"progress,omitempty"` + Progress shared.DataMoveOperationProgress `json:"progress,omitempty"` } // TODO(2.0) After converting all resources to use the runttime-controller client, diff --git a/pkg/apis/velero/v1/pod_volume_restore_type.go b/pkg/apis/velero/v1/pod_volume_restore_type.go index 72c3b891c..84ffc770f 100644 --- a/pkg/apis/velero/v1/pod_volume_restore_type.go +++ b/pkg/apis/velero/v1/pod_volume_restore_type.go @@ -19,6 +19,8 @@ package v1 import ( corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" ) // PodVolumeRestoreSpec is the specification for a PodVolumeRestore. @@ -86,7 +88,7 @@ type PodVolumeRestoreStatus struct { // number of restored bytes. This can be used to display progress information // about the restore operation. // +optional - Progress PodVolumeOperationProgress `json:"progress,omitempty"` + Progress shared.DataMoveOperationProgress `json:"progress,omitempty"` } // TODO(2.0) After converting all resources to use the runtime-controller client, the genclient and k8s:deepcopy markers will no longer be needed and should be removed. diff --git a/pkg/apis/velero/v1/zz_generated.deepcopy.go b/pkg/apis/velero/v1/zz_generated.deepcopy.go index 2d00769c7..a64379ebd 100644 --- a/pkg/apis/velero/v1/zz_generated.deepcopy.go +++ b/pkg/apis/velero/v1/zz_generated.deepcopy.go @@ -376,6 +376,11 @@ func (in *BackupSpec) DeepCopyInto(out *BackupSpec) { *out = new(corev1.TypedLocalObjectReference) (*in).DeepCopyInto(*out) } + if in.SnapshotMoveData != nil { + in, out := &in.SnapshotMoveData, &out.SnapshotMoveData + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupSpec. @@ -977,21 +982,6 @@ func (in *PodVolumeBackupStatus) DeepCopy() *PodVolumeBackupStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *PodVolumeOperationProgress) DeepCopyInto(out *PodVolumeOperationProgress) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodVolumeOperationProgress. -func (in *PodVolumeOperationProgress) DeepCopy() *PodVolumeOperationProgress { - if in == nil { - return nil - } - out := new(PodVolumeOperationProgress) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodVolumeRestore) DeepCopyInto(out *PodVolumeRestore) { *out = *in diff --git a/pkg/apis/velero/v2alpha1/data_download_types.go b/pkg/apis/velero/v2alpha1/data_download_types.go new file mode 100644 index 000000000..b880393ae --- /dev/null +++ b/pkg/apis/velero/v2alpha1/data_download_types.go @@ -0,0 +1,156 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v2alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" +) + +// DataDownloadSpec is the specification for a DataDownload. +type DataDownloadSpec struct { + // TargetVolume is the information of the target PVC and PV. + TargetVolume TargetVolumeSpec `json:"targetVolume"` + + // BackupStorageLocation is the name of the backup storage location + // where the backup repository is stored. + BackupStorageLocation string `json:"backupStorageLocation"` + + // DataMover specifies the data mover to be used by the backup. + // If DataMover is "" or "velero", the built-in data mover will be used. + // +optional + DataMover string `json:"datamover,omitempty"` + + // SnapshotID is the ID of the Velero backup snapshot to be restored from. + SnapshotID string `json:"snapshotID"` + + // SourceNamespace is the original namespace where the volume is backed up from. + // It may be different from SourcePVC's namespace if namespace is remapped during restore. + SourceNamespace string `json:"sourceNamespace"` + + // DataMoverConfig is for data-mover-specific configuration fields. + // +optional + DataMoverConfig map[string]string `json:"dataMoverConfig,omitempty"` + + // Cancel indicates request to cancel the ongoing DataDownload. It can be set + // when the DataDownload is in InProgress phase + Cancel bool `json:"cancel,omitempty"` + + // OperationTimeout specifies the time used to wait internal operations, + // before returning error as timeout. + OperationTimeout metav1.Duration `json:"operationTimeout"` +} + +// TargetPVCSpec is the specification for a target PVC. +type TargetVolumeSpec struct { + // PVC is the name of the target PVC that is created by Velero restore + PVC string `json:"pvc"` + + // PV is the name of the target PV that is created by Velero restore + PV string `json:"pv"` + + // Namespace is the target namespace + Namespace string `json:"namespace"` +} + +// DataDownloadPhase represents the lifecycle phase of a DataDownload. +// +kubebuilder:validation:Enum=New;Accepted;Prepared;InProgress;Canceling;Canceled;Completed;Failed +type DataDownloadPhase string + +const ( + DataDownloadPhaseNew DataDownloadPhase = "New" + DataDownloadPhaseAccepted DataDownloadPhase = "Accepted" + DataDownloadPhasePrepared DataDownloadPhase = "Prepared" + DataDownloadPhaseInProgress DataDownloadPhase = "InProgress" + DataDownloadPhaseCanceling DataDownloadPhase = "Canceling" + DataDownloadPhaseCanceled DataDownloadPhase = "Canceled" + DataDownloadPhaseCompleted DataDownloadPhase = "Completed" + DataDownloadPhaseFailed DataDownloadPhase = "Failed" +) + +// DataDownloadStatus is the current status of a DataDownload. +type DataDownloadStatus struct { + // Phase is the current state of the DataDownload. + // +optional + Phase DataDownloadPhase `json:"phase,omitempty"` + + // Message is a message about the DataDownload's status. + // +optional + Message string `json:"message,omitempty"` + + // StartTimestamp records the time a restore was started. + // The server's time is used for StartTimestamps + // +optional + // +nullable + StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"` + + // CompletionTimestamp records the time a restore was completed. + // Completion time is recorded even on failed restores. + // The server's time is used for CompletionTimestamps + // +optional + // +nullable + CompletionTimestamp *metav1.Time `json:"completionTimestamp,omitempty"` + + // Progress holds the total number of bytes of the snapshot and the current + // number of restored bytes. This can be used to display progress information + // about the restore operation. + // +optional + Progress shared.DataMoveOperationProgress `json:"progress,omitempty"` +} + +// TODO(2.0) After converting all resources to use the runtime-controller client, the genclient and k8s:deepcopy markers will no longer be needed and should be removed. +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:generate=true +// +kubebuilder:object:root=true +// +kubebuilder:storageversion +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="DataDownload status such as New/InProgress" +// +kubebuilder:printcolumn:name="Started",type="date",JSONPath=".status.startTimestamp",description="Time duration since this DataDownload was started" +// +kubebuilder:printcolumn:name="Bytes Done",type="integer",format="int64",JSONPath=".status.progress.bytesDone",description="Completed bytes" +// +kubebuilder:printcolumn:name="Total Bytes",type="integer",format="int64",JSONPath=".status.progress.totalBytes",description="Total bytes" +// +kubebuilder:printcolumn:name="Storage Location",type="string",JSONPath=".spec.backupStorageLocation",description="Name of the Backup Storage Location where the backup data is stored" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since this DataDownload was created" + +type DataDownload struct { + metav1.TypeMeta `json:",inline"` + + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // +optional + Spec DataDownloadSpec `json:"spec,omitempty"` + + // +optional + Status DataDownloadStatus `json:"status,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:generate=true +// +kubebuilder:object:root=true +// +kubebuilder:rbac:groups=velero.io,resources=datadownloads,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=velero.io,resources=datadownloads/status,verbs=get;update;patch + +// DataDownloadList is a list of DataDownloads. +type DataDownloadList struct { + metav1.TypeMeta `json:",inline"` + + // +optional + metav1.ListMeta `json:"metadata,omitempty"` + + Items []DataDownload `json:"items"` +} diff --git a/pkg/apis/velero/v2alpha1/data_upload_types.go b/pkg/apis/velero/v2alpha1/data_upload_types.go new file mode 100644 index 000000000..e12d7478f --- /dev/null +++ b/pkg/apis/velero/v2alpha1/data_upload_types.go @@ -0,0 +1,209 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v2alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" +) + +// DataUploadSpec is the specification for a DataUpload. +type DataUploadSpec struct { + // SnapshotType is the type of the snapshot to be backed up. + SnapshotType SnapshotType `json:"snapshotType"` + + // If SnapshotType is CSI, CSISnapshot provides the information of the CSI snapshot. + // +optional + // +nullable + CSISnapshot *CSISnapshotSpec `json:"csiSnapshot"` + + // SourcePVC is the name of the PVC which the snapshot is taken for. + SourcePVC string `json:"sourcePVC"` + + // DataMover specifies the data mover to be used by the backup. + // If DataMover is "" or "velero", the built-in data mover will be used. + // +optional + DataMover string `json:"datamover,omitempty"` + + // BackupStorageLocation is the name of the backup storage location + // where the backup repository is stored. + BackupStorageLocation string `json:"backupStorageLocation"` + + // SourceNamespace is the original namespace where the volume is backed up from. + // It is the same namespace for SourcePVC and CSI namespaced objects. + SourceNamespace string `json:"sourceNamespace"` + + // DataMoverConfig is for data-mover-specific configuration fields. + // +optional + // +nullable + DataMoverConfig *map[string]string `json:"dataMoverConfig,omitempty"` + + // Cancel indicates request to cancel the ongoing DataUpload. It can be set + // when the DataUpload is in InProgress phase + Cancel bool `json:"cancel,omitempty"` + + // OperationTimeout specifies the time used to wait internal operations, + // before returning error as timeout. + OperationTimeout metav1.Duration `json:"operationTimeout"` +} + +type SnapshotType string + +const ( + SnapshotTypeCSI SnapshotType = "CSI" +) + +// CSISnapshotSpec is the specification for a CSI snapshot. +type CSISnapshotSpec struct { + // VolumeSnapshot is the name of the volume snapshot to be backed up + VolumeSnapshot string `json:"volumeSnapshot"` + + // StorageClass is the name of the storage class of the PVC that the volume snapshot is created from + StorageClass string `json:"storageClass"` + + // StorageClass is the name of the snapshot class that the volume snapshot is created with + // +optional + SnapshotClass string `json:"snapshotClass"` +} + +// DataUploadPhase represents the lifecycle phase of a DataUpload. +// +kubebuilder:validation:Enum=New;Accepted;Prepared;InProgress;Canceling;Canceled;Completed;Failed +type DataUploadPhase string + +const ( + DataUploadPhaseNew DataUploadPhase = "New" + DataUploadPhaseAccepted DataUploadPhase = "Accepted" + DataUploadPhasePrepared DataUploadPhase = "Prepared" + DataUploadPhaseInProgress DataUploadPhase = "InProgress" + DataUploadPhaseCanceling DataUploadPhase = "Canceling" + DataUploadPhaseCanceled DataUploadPhase = "Canceled" + DataUploadPhaseCompleted DataUploadPhase = "Completed" + DataUploadPhaseFailed DataUploadPhase = "Failed" +) + +// DataUploadStatus is the current status of a DataUpload. +type DataUploadStatus struct { + // Phase is the current state of the DataUpload. + // +optional + Phase DataUploadPhase `json:"phase,omitempty"` + + // Path is the full path of the snapshot volume being backed up. + // +optional + Path string `json:"path,omitempty"` + + // SnapshotID is the identifier for the snapshot in the backup repository. + // +optional + SnapshotID string `json:"snapshotID,omitempty"` + + // DataMoverResult stores data-mover-specific information as a result of the DataUpload. + // +optional + // +nullable + DataMoverResult *map[string]string `json:"dataMoverResult,omitempty"` + + // Message is a message about the DataUpload's status. + // +optional + Message string `json:"message,omitempty"` + + // StartTimestamp records the time a backup was started. + // Separate from CreationTimestamp, since that value changes + // on restores. + // The server's time is used for StartTimestamps + // +optional + // +nullable + StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"` + + // CompletionTimestamp records the time a backup was completed. + // Completion time is recorded even on failed backups. + // Completion time is recorded before uploading the backup object. + // The server's time is used for CompletionTimestamps + // +optional + // +nullable + CompletionTimestamp *metav1.Time `json:"completionTimestamp,omitempty"` + + // Progress holds the total number of bytes of the volume and the current + // number of backed up bytes. This can be used to display progress information + // about the backup operation. + // +optional + Progress shared.DataMoveOperationProgress `json:"progress,omitempty"` +} + +// TODO(2.0) After converting all resources to use the runttime-controller client, +// the genclient and k8s:deepcopy markers will no longer be needed and should be removed. +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true +// +kubebuilder:object:generate=true +// +kubebuilder:storageversion +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="DataUpload status such as New/InProgress" +// +kubebuilder:printcolumn:name="Started",type="date",JSONPath=".status.startTimestamp",description="Time duration since this DataUpload was started" +// +kubebuilder:printcolumn:name="Bytes Done",type="integer",format="int64",JSONPath=".status.progress.bytesDone",description="Completed bytes" +// +kubebuilder:printcolumn:name="Total Bytes",type="integer",format="int64",JSONPath=".status.progress.totalBytes",description="Total bytes" +// +kubebuilder:printcolumn:name="Storage Location",type="string",JSONPath=".spec.backupStorageLocation",description="Name of the Backup Storage Location where this backup should be stored" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since this DataUpload was created" + +type DataUpload struct { + metav1.TypeMeta `json:",inline"` + + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // +optional + Spec DataUploadSpec `json:"spec,omitempty"` + + // +optional + Status DataUploadStatus `json:"status,omitempty"` +} + +// TODO(2.0) After converting all resources to use the runtime-controller client, +// the k8s:deepcopy marker will no longer be needed and should be removed. +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true +// +kubebuilder:rbac:groups=velero.io,resources=datauploads,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=velero.io,resources=datauploads/status,verbs=get;update;patch + +// DataUploadList is a list of DataUploads. +type DataUploadList struct { + metav1.TypeMeta `json:",inline"` + + // +optional + metav1.ListMeta `json:"metadata,omitempty"` + + Items []DataUpload `json:"items"` +} + +// DataUploadResult represents the SnasphotBackup result to be used by DataDownload. +type DataUploadResult struct { + // BackupStorageLocation is the name of the backup storage location + // where the backup repository is stored. + BackupStorageLocation string `json:"backupStorageLocation"` + + // DataMover specifies the data mover used by the DataUpload + // +optional + DataMover string `json:"datamover,omitempty"` + + // SnapshotID is the identifier for the snapshot in the backup repository. + SnapshotID string `json:"snapshotID,omitempty"` + + // SourceNamespace is the original namespace where the volume is backed up from. + SourceNamespace string `json:"sourceNamespace"` + + // DataMoverResult stores data-mover-specific information as a result of the DataUpload. + // +optional + // +nullable + DataMoverResult *map[string]string `json:"dataMoverResult,omitempty"` +} diff --git a/pkg/apis/velero/v2alpha1/doc.go b/pkg/apis/velero/v2alpha1/doc.go new file mode 100644 index 000000000..9bab0a40c --- /dev/null +++ b/pkg/apis/velero/v2alpha1/doc.go @@ -0,0 +1,21 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:deepcopy-gen=package + +// Package v2alpha1 is the v2alpha1 version of the API. +// +groupName=velero.io +package v2alpha1 diff --git a/pkg/apis/velero/v2alpha1/groupversion_info.go b/pkg/apis/velero/v2alpha1/groupversion_info.go new file mode 100644 index 000000000..645c95653 --- /dev/null +++ b/pkg/apis/velero/v2alpha1/groupversion_info.go @@ -0,0 +1,36 @@ +/* +Copyright 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v2alpha1 contains API Schema definitions for the velero v2alpha1 API group +// +kubebuilder:object:generate=true +// +groupName=velero.io +package v2alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var ( + // SchemeGroupVersion is group version used to register these objects + SchemeGroupVersion = schema.GroupVersion{Group: "velero.io", Version: "v2alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/pkg/apis/velero/v2alpha1/register.go b/pkg/apis/velero/v2alpha1/register.go new file mode 100644 index 000000000..31b471281 --- /dev/null +++ b/pkg/apis/velero/v2alpha1/register.go @@ -0,0 +1,60 @@ +/* +Copyright 2017 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v2alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// Resource gets a Velero GroupResource for a specified resource +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} + +type typeInfo struct { + PluralName string + ItemType runtime.Object + ItemListType runtime.Object +} + +func newTypeInfo(pluralName string, itemType, itemListType runtime.Object) typeInfo { + return typeInfo{ + PluralName: pluralName, + ItemType: itemType, + ItemListType: itemListType, + } +} + +// CustomResources returns a map of all custom resources within the Velero +// API group, keyed on Kind. +func CustomResources() map[string]typeInfo { + return map[string]typeInfo{ + "DataUpload": newTypeInfo("datauploads", &DataUpload{}, &DataUploadList{}), + "DataDownload": newTypeInfo("datadownloads", &DataDownload{}, &DataDownloadList{}), + } +} + +func addKnownTypes(scheme *runtime.Scheme) error { + for _, typeInfo := range CustomResources() { + scheme.AddKnownTypes(SchemeGroupVersion, typeInfo.ItemType, typeInfo.ItemListType) + } + + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil +} diff --git a/pkg/apis/velero/v2alpha1/zz_generated.deepcopy.go b/pkg/apis/velero/v2alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..9a9afaa6d --- /dev/null +++ b/pkg/apis/velero/v2alpha1/zz_generated.deepcopy.go @@ -0,0 +1,299 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +// Code generated by controller-gen. DO NOT EDIT. + +package v2alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CSISnapshotSpec) DeepCopyInto(out *CSISnapshotSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSISnapshotSpec. +func (in *CSISnapshotSpec) DeepCopy() *CSISnapshotSpec { + if in == nil { + return nil + } + out := new(CSISnapshotSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataDownload) DeepCopyInto(out *DataDownload) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataDownload. +func (in *DataDownload) DeepCopy() *DataDownload { + if in == nil { + return nil + } + out := new(DataDownload) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DataDownload) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataDownloadList) DeepCopyInto(out *DataDownloadList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]DataDownload, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataDownloadList. +func (in *DataDownloadList) DeepCopy() *DataDownloadList { + if in == nil { + return nil + } + out := new(DataDownloadList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DataDownloadList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataDownloadSpec) DeepCopyInto(out *DataDownloadSpec) { + *out = *in + out.TargetVolume = in.TargetVolume + if in.DataMoverConfig != nil { + in, out := &in.DataMoverConfig, &out.DataMoverConfig + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + out.OperationTimeout = in.OperationTimeout +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataDownloadSpec. +func (in *DataDownloadSpec) DeepCopy() *DataDownloadSpec { + if in == nil { + return nil + } + out := new(DataDownloadSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataDownloadStatus) DeepCopyInto(out *DataDownloadStatus) { + *out = *in + if in.StartTimestamp != nil { + in, out := &in.StartTimestamp, &out.StartTimestamp + *out = (*in).DeepCopy() + } + if in.CompletionTimestamp != nil { + in, out := &in.CompletionTimestamp, &out.CompletionTimestamp + *out = (*in).DeepCopy() + } + out.Progress = in.Progress +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataDownloadStatus. +func (in *DataDownloadStatus) DeepCopy() *DataDownloadStatus { + if in == nil { + return nil + } + out := new(DataDownloadStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataUpload) DeepCopyInto(out *DataUpload) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataUpload. +func (in *DataUpload) DeepCopy() *DataUpload { + if in == nil { + return nil + } + out := new(DataUpload) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DataUpload) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataUploadList) DeepCopyInto(out *DataUploadList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]DataUpload, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataUploadList. +func (in *DataUploadList) DeepCopy() *DataUploadList { + if in == nil { + return nil + } + out := new(DataUploadList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DataUploadList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataUploadResult) DeepCopyInto(out *DataUploadResult) { + *out = *in + if in.DataMoverResult != nil { + in, out := &in.DataMoverResult, &out.DataMoverResult + *out = new(map[string]string) + if **in != nil { + in, out := *in, *out + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataUploadResult. +func (in *DataUploadResult) DeepCopy() *DataUploadResult { + if in == nil { + return nil + } + out := new(DataUploadResult) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataUploadSpec) DeepCopyInto(out *DataUploadSpec) { + *out = *in + if in.CSISnapshot != nil { + in, out := &in.CSISnapshot, &out.CSISnapshot + *out = new(CSISnapshotSpec) + **out = **in + } + if in.DataMoverConfig != nil { + in, out := &in.DataMoverConfig, &out.DataMoverConfig + *out = new(map[string]string) + if **in != nil { + in, out := *in, *out + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + } + out.OperationTimeout = in.OperationTimeout +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataUploadSpec. +func (in *DataUploadSpec) DeepCopy() *DataUploadSpec { + if in == nil { + return nil + } + out := new(DataUploadSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataUploadStatus) DeepCopyInto(out *DataUploadStatus) { + *out = *in + if in.DataMoverResult != nil { + in, out := &in.DataMoverResult, &out.DataMoverResult + *out = new(map[string]string) + if **in != nil { + in, out := *in, *out + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + } + if in.StartTimestamp != nil { + in, out := &in.StartTimestamp, &out.StartTimestamp + *out = (*in).DeepCopy() + } + if in.CompletionTimestamp != nil { + in, out := &in.CompletionTimestamp, &out.CompletionTimestamp + *out = (*in).DeepCopy() + } + out.Progress = in.Progress +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataUploadStatus. +func (in *DataUploadStatus) DeepCopy() *DataUploadStatus { + if in == nil { + return nil + } + out := new(DataUploadStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TargetVolumeSpec) DeepCopyInto(out *TargetVolumeSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetVolumeSpec. +func (in *TargetVolumeSpec) DeepCopy() *TargetVolumeSpec { + if in == nil { + return nil + } + out := new(TargetVolumeSpec) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/backup/backup_pv_action.go b/pkg/backup/backup_pv_action.go index 6eb8b9309..56c4fff8a 100644 --- a/pkg/backup/backup_pv_action.go +++ b/pkg/backup/backup_pv_action.go @@ -20,6 +20,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -61,5 +62,18 @@ func (a *PVCAction) Execute(item runtime.Unstructured, backup *v1.Backup) (runti GroupResource: kuberesource.PersistentVolumes, Name: pvc.Spec.VolumeName, } - return item, []velero.ResourceIdentifier{pv}, nil + // remove dataSource if exists from prior restored CSI volumes + if pvc.Spec.DataSource != nil { + pvc.Spec.DataSource = nil + } + if pvc.Spec.DataSourceRef != nil { + pvc.Spec.DataSourceRef = nil + } + + pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pvc) + if err != nil { + return nil, nil, errors.Wrap(err, "unable to convert pvc to unstructured item") + } + + return &unstructured.Unstructured{Object: pvcMap}, []velero.ResourceIdentifier{pv}, nil } diff --git a/pkg/client/factory.go b/pkg/client/factory.go index 58ea5994b..36a6a0459 100644 --- a/pkg/client/factory.go +++ b/pkg/client/factory.go @@ -32,6 +32,7 @@ import ( "k8s.io/client-go/rest" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" clientset "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned" ) @@ -156,6 +157,9 @@ func (f *factory) KubebuilderClient() (kbclient.Client, error) { if err := velerov1api.AddToScheme(scheme); err != nil { return nil, err } + if err := velerov2alpha1api.AddToScheme(scheme); err != nil { + return nil, err + } if err := k8scheme.AddToScheme(scheme); err != nil { return nil, err } diff --git a/pkg/cmd/cli/nodeagent/server.go b/pkg/cmd/cli/nodeagent/server.go index 81183751d..08fadb9f3 100644 --- a/pkg/cmd/cli/nodeagent/server.go +++ b/pkg/cmd/cli/nodeagent/server.go @@ -45,6 +45,7 @@ import ( "github.com/vmware-tanzu/velero/internal/credentials" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/buildinfo" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" @@ -125,6 +126,10 @@ func newNodeAgentServer(logger logrus.FieldLogger, factory client.Factory, metri cancelFunc() return nil, err } + if err := velerov2alpha1api.AddToScheme(scheme); err != nil { + cancelFunc() + return nil, err + } if err := v1.AddToScheme(scheme); err != nil { cancelFunc() return nil, err diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 904dee663..ec108536e 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -55,6 +55,7 @@ import ( "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/internal/storage" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/buildinfo" "github.com/vmware-tanzu/velero/pkg/client" @@ -317,6 +318,10 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s cancelFunc() return nil, err } + if err := velerov2alpha1api.AddToScheme(scheme); err != nil { + cancelFunc() + return nil, err + } if err := corev1api.AddToScheme(scheme); err != nil { cancelFunc() return nil, err diff --git a/pkg/cmd/util/output/backup_describer.go b/pkg/cmd/util/output/backup_describer.go index 983c0e6b0..7fa210c18 100644 --- a/pkg/cmd/util/output/backup_describer.go +++ b/pkg/cmd/util/output/backup_describer.go @@ -32,6 +32,7 @@ import ( "github.com/fatih/color" kbclient "sigs.k8s.io/controller-runtime/pkg/client" + veleroapishared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" "github.com/vmware-tanzu/velero/pkg/features" @@ -597,7 +598,7 @@ type volumesByPod struct { // Add adds a pod volume with the specified pod namespace, name // and volume to the appropriate group. -func (v *volumesByPod) Add(namespace, name, volume, phase string, progress velerov1api.PodVolumeOperationProgress) { +func (v *volumesByPod) Add(namespace, name, volume, phase string, progress veleroapishared.DataMoveOperationProgress) { if v.volumesByPodMap == nil { v.volumesByPodMap = make(map[string]*podVolumeGroup) } diff --git a/pkg/controller/pod_volume_backup_controller.go b/pkg/controller/pod_volume_backup_controller.go index 61efe7c10..f87b1c5a4 100644 --- a/pkg/controller/pod_volume_backup_controller.go +++ b/pkg/controller/pod_volume_backup_controller.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/credentials" + veleroapishared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/podvolume" @@ -303,7 +304,7 @@ func (r *PodVolumeBackupReconciler) NewBackupProgressUpdater(ctx context.Context // UpdateProgress which implement ProgressUpdater interface to update pvb progress status func (b *BackupProgressUpdater) UpdateProgress(p *uploader.Progress) { original := b.PodVolumeBackup.DeepCopy() - b.PodVolumeBackup.Status.Progress = velerov1api.PodVolumeOperationProgress{TotalBytes: p.TotalBytes, BytesDone: p.BytesDone} + b.PodVolumeBackup.Status.Progress = veleroapishared.DataMoveOperationProgress{TotalBytes: p.TotalBytes, BytesDone: p.BytesDone} if b.Cli == nil { b.Log.Errorf("failed to update backup pod %s volume %s progress with uninitailize client", b.PodVolumeBackup.Spec.Pod.Name, b.PodVolumeBackup.Spec.Volume) return diff --git a/pkg/controller/pod_volume_restore_controller.go b/pkg/controller/pod_volume_restore_controller.go index 5a8e380fc..ab6db871c 100644 --- a/pkg/controller/pod_volume_restore_controller.go +++ b/pkg/controller/pod_volume_restore_controller.go @@ -38,6 +38,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" "github.com/vmware-tanzu/velero/internal/credentials" + veleroapishared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/podvolume" "github.com/vmware-tanzu/velero/pkg/repository" @@ -321,7 +322,7 @@ func (c *PodVolumeRestoreReconciler) NewRestoreProgressUpdater(ctx context.Conte // UpdateProgress which implement ProgressUpdater interface to update pvr progress status func (c *RestoreProgressUpdater) UpdateProgress(p *uploader.Progress) { original := c.PodVolumeRestore.DeepCopy() - c.PodVolumeRestore.Status.Progress = velerov1api.PodVolumeOperationProgress{TotalBytes: p.TotalBytes, BytesDone: p.BytesDone} + c.PodVolumeRestore.Status.Progress = veleroapishared.DataMoveOperationProgress{TotalBytes: p.TotalBytes, BytesDone: p.BytesDone} if c.Cli == nil { c.Log.Errorf("failed to update restore pod %s volume %s progress with uninitailize client", c.PodVolumeRestore.Spec.Pod.Name, c.PodVolumeRestore.Spec.Volume) return diff --git a/pkg/controller/suite_test.go b/pkg/controller/suite_test.go index f68ab84a0..44c858bcf 100644 --- a/pkg/controller/suite_test.go +++ b/pkg/controller/suite_test.go @@ -36,6 +36,7 @@ import ( . "github.com/onsi/gomega" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" @@ -98,6 +99,9 @@ func newTestEnvironment() *testEnvironment { err := velerov1api.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = velerov2alpha1api.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + env = &envtest.Environment{ CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, } diff --git a/pkg/generated/clientset/versioned/clientset.go b/pkg/generated/clientset/versioned/clientset.go index c98ad605e..881dee994 100644 --- a/pkg/generated/clientset/versioned/clientset.go +++ b/pkg/generated/clientset/versioned/clientset.go @@ -22,6 +22,7 @@ import ( "fmt" velerov1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" + velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v2alpha1" discovery "k8s.io/client-go/discovery" rest "k8s.io/client-go/rest" flowcontrol "k8s.io/client-go/util/flowcontrol" @@ -30,13 +31,15 @@ import ( type Interface interface { Discovery() discovery.DiscoveryInterface VeleroV1() velerov1.VeleroV1Interface + VeleroV2alpha1() velerov2alpha1.VeleroV2alpha1Interface } // Clientset contains the clients for groups. Each group has exactly one // version included in a Clientset. type Clientset struct { *discovery.DiscoveryClient - veleroV1 *velerov1.VeleroV1Client + veleroV1 *velerov1.VeleroV1Client + veleroV2alpha1 *velerov2alpha1.VeleroV2alpha1Client } // VeleroV1 retrieves the VeleroV1Client @@ -44,6 +47,11 @@ func (c *Clientset) VeleroV1() velerov1.VeleroV1Interface { return c.veleroV1 } +// VeleroV2alpha1 retrieves the VeleroV2alpha1Client +func (c *Clientset) VeleroV2alpha1() velerov2alpha1.VeleroV2alpha1Interface { + return c.veleroV2alpha1 +} + // Discovery retrieves the DiscoveryClient func (c *Clientset) Discovery() discovery.DiscoveryInterface { if c == nil { @@ -69,6 +77,10 @@ func NewForConfig(c *rest.Config) (*Clientset, error) { if err != nil { return nil, err } + cs.veleroV2alpha1, err = velerov2alpha1.NewForConfig(&configShallowCopy) + if err != nil { + return nil, err + } cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(&configShallowCopy) if err != nil { @@ -82,6 +94,7 @@ func NewForConfig(c *rest.Config) (*Clientset, error) { func NewForConfigOrDie(c *rest.Config) *Clientset { var cs Clientset cs.veleroV1 = velerov1.NewForConfigOrDie(c) + cs.veleroV2alpha1 = velerov2alpha1.NewForConfigOrDie(c) cs.DiscoveryClient = discovery.NewDiscoveryClientForConfigOrDie(c) return &cs @@ -91,6 +104,7 @@ func NewForConfigOrDie(c *rest.Config) *Clientset { func New(c rest.Interface) *Clientset { var cs Clientset cs.veleroV1 = velerov1.New(c) + cs.veleroV2alpha1 = velerov2alpha1.New(c) cs.DiscoveryClient = discovery.NewDiscoveryClient(c) return &cs diff --git a/pkg/generated/clientset/versioned/fake/clientset_generated.go b/pkg/generated/clientset/versioned/fake/clientset_generated.go index 971ea5491..d514b27c1 100644 --- a/pkg/generated/clientset/versioned/fake/clientset_generated.go +++ b/pkg/generated/clientset/versioned/fake/clientset_generated.go @@ -22,6 +22,8 @@ import ( clientset "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned" velerov1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" fakevelerov1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1/fake" + velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v2alpha1" + fakevelerov2alpha1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v2alpha1/fake" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/discovery" @@ -83,3 +85,8 @@ var ( func (c *Clientset) VeleroV1() velerov1.VeleroV1Interface { return &fakevelerov1.FakeVeleroV1{Fake: &c.Fake} } + +// VeleroV2alpha1 retrieves the VeleroV2alpha1Client +func (c *Clientset) VeleroV2alpha1() velerov2alpha1.VeleroV2alpha1Interface { + return &fakevelerov2alpha1.FakeVeleroV2alpha1{Fake: &c.Fake} +} diff --git a/pkg/generated/clientset/versioned/fake/register.go b/pkg/generated/clientset/versioned/fake/register.go index 5839b1517..8e9316a47 100644 --- a/pkg/generated/clientset/versioned/fake/register.go +++ b/pkg/generated/clientset/versioned/fake/register.go @@ -20,6 +20,7 @@ package fake import ( velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -32,6 +33,7 @@ var codecs = serializer.NewCodecFactory(scheme) var localSchemeBuilder = runtime.SchemeBuilder{ velerov1.AddToScheme, + velerov2alpha1.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition diff --git a/pkg/generated/clientset/versioned/scheme/register.go b/pkg/generated/clientset/versioned/scheme/register.go index f94b9a72c..12654733e 100644 --- a/pkg/generated/clientset/versioned/scheme/register.go +++ b/pkg/generated/clientset/versioned/scheme/register.go @@ -20,6 +20,7 @@ package scheme import ( velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -32,6 +33,7 @@ var Codecs = serializer.NewCodecFactory(Scheme) var ParameterCodec = runtime.NewParameterCodec(Scheme) var localSchemeBuilder = runtime.SchemeBuilder{ velerov1.AddToScheme, + velerov2alpha1.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition diff --git a/pkg/generated/clientset/versioned/typed/velero/v2alpha1/datadownload.go b/pkg/generated/clientset/versioned/typed/velero/v2alpha1/datadownload.go new file mode 100644 index 000000000..511677675 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/velero/v2alpha1/datadownload.go @@ -0,0 +1,195 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v2alpha1 + +import ( + "context" + "time" + + v2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + scheme "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// DataDownloadsGetter has a method to return a DataDownloadInterface. +// A group's client should implement this interface. +type DataDownloadsGetter interface { + DataDownloads(namespace string) DataDownloadInterface +} + +// DataDownloadInterface has methods to work with DataDownload resources. +type DataDownloadInterface interface { + Create(ctx context.Context, dataDownload *v2alpha1.DataDownload, opts v1.CreateOptions) (*v2alpha1.DataDownload, error) + Update(ctx context.Context, dataDownload *v2alpha1.DataDownload, opts v1.UpdateOptions) (*v2alpha1.DataDownload, error) + UpdateStatus(ctx context.Context, dataDownload *v2alpha1.DataDownload, opts v1.UpdateOptions) (*v2alpha1.DataDownload, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v2alpha1.DataDownload, error) + List(ctx context.Context, opts v1.ListOptions) (*v2alpha1.DataDownloadList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v2alpha1.DataDownload, err error) + DataDownloadExpansion +} + +// dataDownloads implements DataDownloadInterface +type dataDownloads struct { + client rest.Interface + ns string +} + +// newDataDownloads returns a DataDownloads +func newDataDownloads(c *VeleroV2alpha1Client, namespace string) *dataDownloads { + return &dataDownloads{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the dataDownload, and returns the corresponding dataDownload object, and an error if there is any. +func (c *dataDownloads) Get(ctx context.Context, name string, options v1.GetOptions) (result *v2alpha1.DataDownload, err error) { + result = &v2alpha1.DataDownload{} + err = c.client.Get(). + Namespace(c.ns). + Resource("datadownloads"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of DataDownloads that match those selectors. +func (c *dataDownloads) List(ctx context.Context, opts v1.ListOptions) (result *v2alpha1.DataDownloadList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v2alpha1.DataDownloadList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("datadownloads"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested dataDownloads. +func (c *dataDownloads) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("datadownloads"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a dataDownload and creates it. Returns the server's representation of the dataDownload, and an error, if there is any. +func (c *dataDownloads) Create(ctx context.Context, dataDownload *v2alpha1.DataDownload, opts v1.CreateOptions) (result *v2alpha1.DataDownload, err error) { + result = &v2alpha1.DataDownload{} + err = c.client.Post(). + Namespace(c.ns). + Resource("datadownloads"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(dataDownload). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a dataDownload and updates it. Returns the server's representation of the dataDownload, and an error, if there is any. +func (c *dataDownloads) Update(ctx context.Context, dataDownload *v2alpha1.DataDownload, opts v1.UpdateOptions) (result *v2alpha1.DataDownload, err error) { + result = &v2alpha1.DataDownload{} + err = c.client.Put(). + Namespace(c.ns). + Resource("datadownloads"). + Name(dataDownload.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(dataDownload). + Do(ctx). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *dataDownloads) UpdateStatus(ctx context.Context, dataDownload *v2alpha1.DataDownload, opts v1.UpdateOptions) (result *v2alpha1.DataDownload, err error) { + result = &v2alpha1.DataDownload{} + err = c.client.Put(). + Namespace(c.ns). + Resource("datadownloads"). + Name(dataDownload.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(dataDownload). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the dataDownload and deletes it. Returns an error if one occurs. +func (c *dataDownloads) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("datadownloads"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *dataDownloads) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Namespace(c.ns). + Resource("datadownloads"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched dataDownload. +func (c *dataDownloads) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v2alpha1.DataDownload, err error) { + result = &v2alpha1.DataDownload{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("datadownloads"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/pkg/generated/clientset/versioned/typed/velero/v2alpha1/dataupload.go b/pkg/generated/clientset/versioned/typed/velero/v2alpha1/dataupload.go new file mode 100644 index 000000000..4da27d527 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/velero/v2alpha1/dataupload.go @@ -0,0 +1,195 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v2alpha1 + +import ( + "context" + "time" + + v2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + scheme "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// DataUploadsGetter has a method to return a DataUploadInterface. +// A group's client should implement this interface. +type DataUploadsGetter interface { + DataUploads(namespace string) DataUploadInterface +} + +// DataUploadInterface has methods to work with DataUpload resources. +type DataUploadInterface interface { + Create(ctx context.Context, dataUpload *v2alpha1.DataUpload, opts v1.CreateOptions) (*v2alpha1.DataUpload, error) + Update(ctx context.Context, dataUpload *v2alpha1.DataUpload, opts v1.UpdateOptions) (*v2alpha1.DataUpload, error) + UpdateStatus(ctx context.Context, dataUpload *v2alpha1.DataUpload, opts v1.UpdateOptions) (*v2alpha1.DataUpload, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v2alpha1.DataUpload, error) + List(ctx context.Context, opts v1.ListOptions) (*v2alpha1.DataUploadList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v2alpha1.DataUpload, err error) + DataUploadExpansion +} + +// dataUploads implements DataUploadInterface +type dataUploads struct { + client rest.Interface + ns string +} + +// newDataUploads returns a DataUploads +func newDataUploads(c *VeleroV2alpha1Client, namespace string) *dataUploads { + return &dataUploads{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the dataUpload, and returns the corresponding dataUpload object, and an error if there is any. +func (c *dataUploads) Get(ctx context.Context, name string, options v1.GetOptions) (result *v2alpha1.DataUpload, err error) { + result = &v2alpha1.DataUpload{} + err = c.client.Get(). + Namespace(c.ns). + Resource("datauploads"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of DataUploads that match those selectors. +func (c *dataUploads) List(ctx context.Context, opts v1.ListOptions) (result *v2alpha1.DataUploadList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v2alpha1.DataUploadList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("datauploads"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested dataUploads. +func (c *dataUploads) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("datauploads"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a dataUpload and creates it. Returns the server's representation of the dataUpload, and an error, if there is any. +func (c *dataUploads) Create(ctx context.Context, dataUpload *v2alpha1.DataUpload, opts v1.CreateOptions) (result *v2alpha1.DataUpload, err error) { + result = &v2alpha1.DataUpload{} + err = c.client.Post(). + Namespace(c.ns). + Resource("datauploads"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(dataUpload). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a dataUpload and updates it. Returns the server's representation of the dataUpload, and an error, if there is any. +func (c *dataUploads) Update(ctx context.Context, dataUpload *v2alpha1.DataUpload, opts v1.UpdateOptions) (result *v2alpha1.DataUpload, err error) { + result = &v2alpha1.DataUpload{} + err = c.client.Put(). + Namespace(c.ns). + Resource("datauploads"). + Name(dataUpload.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(dataUpload). + Do(ctx). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *dataUploads) UpdateStatus(ctx context.Context, dataUpload *v2alpha1.DataUpload, opts v1.UpdateOptions) (result *v2alpha1.DataUpload, err error) { + result = &v2alpha1.DataUpload{} + err = c.client.Put(). + Namespace(c.ns). + Resource("datauploads"). + Name(dataUpload.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(dataUpload). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the dataUpload and deletes it. Returns an error if one occurs. +func (c *dataUploads) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("datauploads"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *dataUploads) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Namespace(c.ns). + Resource("datauploads"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched dataUpload. +func (c *dataUploads) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v2alpha1.DataUpload, err error) { + result = &v2alpha1.DataUpload{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("datauploads"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/pkg/generated/clientset/versioned/typed/velero/v2alpha1/doc.go b/pkg/generated/clientset/versioned/typed/velero/v2alpha1/doc.go new file mode 100644 index 000000000..18b5cb4d4 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/velero/v2alpha1/doc.go @@ -0,0 +1,20 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated typed clients. +package v2alpha1 diff --git a/pkg/generated/clientset/versioned/typed/velero/v2alpha1/fake/doc.go b/pkg/generated/clientset/versioned/typed/velero/v2alpha1/fake/doc.go new file mode 100644 index 000000000..de930591e --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/velero/v2alpha1/fake/doc.go @@ -0,0 +1,20 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +// Package fake has the automatically generated clients. +package fake diff --git a/pkg/generated/clientset/versioned/typed/velero/v2alpha1/fake/fake_datadownload.go b/pkg/generated/clientset/versioned/typed/velero/v2alpha1/fake/fake_datadownload.go new file mode 100644 index 000000000..40db6018b --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/velero/v2alpha1/fake/fake_datadownload.go @@ -0,0 +1,142 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeDataDownloads implements DataDownloadInterface +type FakeDataDownloads struct { + Fake *FakeVeleroV2alpha1 + ns string +} + +var datadownloadsResource = schema.GroupVersionResource{Group: "velero.io", Version: "v2alpha1", Resource: "datadownloads"} + +var datadownloadsKind = schema.GroupVersionKind{Group: "velero.io", Version: "v2alpha1", Kind: "DataDownload"} + +// Get takes name of the dataDownload, and returns the corresponding dataDownload object, and an error if there is any. +func (c *FakeDataDownloads) Get(ctx context.Context, name string, options v1.GetOptions) (result *v2alpha1.DataDownload, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(datadownloadsResource, c.ns, name), &v2alpha1.DataDownload{}) + + if obj == nil { + return nil, err + } + return obj.(*v2alpha1.DataDownload), err +} + +// List takes label and field selectors, and returns the list of DataDownloads that match those selectors. +func (c *FakeDataDownloads) List(ctx context.Context, opts v1.ListOptions) (result *v2alpha1.DataDownloadList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(datadownloadsResource, datadownloadsKind, c.ns, opts), &v2alpha1.DataDownloadList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v2alpha1.DataDownloadList{ListMeta: obj.(*v2alpha1.DataDownloadList).ListMeta} + for _, item := range obj.(*v2alpha1.DataDownloadList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested dataDownloads. +func (c *FakeDataDownloads) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(datadownloadsResource, c.ns, opts)) + +} + +// Create takes the representation of a dataDownload and creates it. Returns the server's representation of the dataDownload, and an error, if there is any. +func (c *FakeDataDownloads) Create(ctx context.Context, dataDownload *v2alpha1.DataDownload, opts v1.CreateOptions) (result *v2alpha1.DataDownload, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(datadownloadsResource, c.ns, dataDownload), &v2alpha1.DataDownload{}) + + if obj == nil { + return nil, err + } + return obj.(*v2alpha1.DataDownload), err +} + +// Update takes the representation of a dataDownload and updates it. Returns the server's representation of the dataDownload, and an error, if there is any. +func (c *FakeDataDownloads) Update(ctx context.Context, dataDownload *v2alpha1.DataDownload, opts v1.UpdateOptions) (result *v2alpha1.DataDownload, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(datadownloadsResource, c.ns, dataDownload), &v2alpha1.DataDownload{}) + + if obj == nil { + return nil, err + } + return obj.(*v2alpha1.DataDownload), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeDataDownloads) UpdateStatus(ctx context.Context, dataDownload *v2alpha1.DataDownload, opts v1.UpdateOptions) (*v2alpha1.DataDownload, error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceAction(datadownloadsResource, "status", c.ns, dataDownload), &v2alpha1.DataDownload{}) + + if obj == nil { + return nil, err + } + return obj.(*v2alpha1.DataDownload), err +} + +// Delete takes name of the dataDownload and deletes it. Returns an error if one occurs. +func (c *FakeDataDownloads) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteAction(datadownloadsResource, c.ns, name), &v2alpha1.DataDownload{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeDataDownloads) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(datadownloadsResource, c.ns, listOpts) + + _, err := c.Fake.Invokes(action, &v2alpha1.DataDownloadList{}) + return err +} + +// Patch applies the patch and returns the patched dataDownload. +func (c *FakeDataDownloads) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v2alpha1.DataDownload, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(datadownloadsResource, c.ns, name, pt, data, subresources...), &v2alpha1.DataDownload{}) + + if obj == nil { + return nil, err + } + return obj.(*v2alpha1.DataDownload), err +} diff --git a/pkg/generated/clientset/versioned/typed/velero/v2alpha1/fake/fake_dataupload.go b/pkg/generated/clientset/versioned/typed/velero/v2alpha1/fake/fake_dataupload.go new file mode 100644 index 000000000..d40b50874 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/velero/v2alpha1/fake/fake_dataupload.go @@ -0,0 +1,142 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeDataUploads implements DataUploadInterface +type FakeDataUploads struct { + Fake *FakeVeleroV2alpha1 + ns string +} + +var datauploadsResource = schema.GroupVersionResource{Group: "velero.io", Version: "v2alpha1", Resource: "datauploads"} + +var datauploadsKind = schema.GroupVersionKind{Group: "velero.io", Version: "v2alpha1", Kind: "DataUpload"} + +// Get takes name of the dataUpload, and returns the corresponding dataUpload object, and an error if there is any. +func (c *FakeDataUploads) Get(ctx context.Context, name string, options v1.GetOptions) (result *v2alpha1.DataUpload, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(datauploadsResource, c.ns, name), &v2alpha1.DataUpload{}) + + if obj == nil { + return nil, err + } + return obj.(*v2alpha1.DataUpload), err +} + +// List takes label and field selectors, and returns the list of DataUploads that match those selectors. +func (c *FakeDataUploads) List(ctx context.Context, opts v1.ListOptions) (result *v2alpha1.DataUploadList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(datauploadsResource, datauploadsKind, c.ns, opts), &v2alpha1.DataUploadList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v2alpha1.DataUploadList{ListMeta: obj.(*v2alpha1.DataUploadList).ListMeta} + for _, item := range obj.(*v2alpha1.DataUploadList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested dataUploads. +func (c *FakeDataUploads) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(datauploadsResource, c.ns, opts)) + +} + +// Create takes the representation of a dataUpload and creates it. Returns the server's representation of the dataUpload, and an error, if there is any. +func (c *FakeDataUploads) Create(ctx context.Context, dataUpload *v2alpha1.DataUpload, opts v1.CreateOptions) (result *v2alpha1.DataUpload, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(datauploadsResource, c.ns, dataUpload), &v2alpha1.DataUpload{}) + + if obj == nil { + return nil, err + } + return obj.(*v2alpha1.DataUpload), err +} + +// Update takes the representation of a dataUpload and updates it. Returns the server's representation of the dataUpload, and an error, if there is any. +func (c *FakeDataUploads) Update(ctx context.Context, dataUpload *v2alpha1.DataUpload, opts v1.UpdateOptions) (result *v2alpha1.DataUpload, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(datauploadsResource, c.ns, dataUpload), &v2alpha1.DataUpload{}) + + if obj == nil { + return nil, err + } + return obj.(*v2alpha1.DataUpload), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeDataUploads) UpdateStatus(ctx context.Context, dataUpload *v2alpha1.DataUpload, opts v1.UpdateOptions) (*v2alpha1.DataUpload, error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceAction(datauploadsResource, "status", c.ns, dataUpload), &v2alpha1.DataUpload{}) + + if obj == nil { + return nil, err + } + return obj.(*v2alpha1.DataUpload), err +} + +// Delete takes name of the dataUpload and deletes it. Returns an error if one occurs. +func (c *FakeDataUploads) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteAction(datauploadsResource, c.ns, name), &v2alpha1.DataUpload{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeDataUploads) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(datauploadsResource, c.ns, listOpts) + + _, err := c.Fake.Invokes(action, &v2alpha1.DataUploadList{}) + return err +} + +// Patch applies the patch and returns the patched dataUpload. +func (c *FakeDataUploads) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v2alpha1.DataUpload, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(datauploadsResource, c.ns, name, pt, data, subresources...), &v2alpha1.DataUpload{}) + + if obj == nil { + return nil, err + } + return obj.(*v2alpha1.DataUpload), err +} diff --git a/pkg/generated/clientset/versioned/typed/velero/v2alpha1/fake/fake_velero_client.go b/pkg/generated/clientset/versioned/typed/velero/v2alpha1/fake/fake_velero_client.go new file mode 100644 index 000000000..25fee2e7a --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/velero/v2alpha1/fake/fake_velero_client.go @@ -0,0 +1,44 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v2alpha1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v2alpha1" + rest "k8s.io/client-go/rest" + testing "k8s.io/client-go/testing" +) + +type FakeVeleroV2alpha1 struct { + *testing.Fake +} + +func (c *FakeVeleroV2alpha1) DataDownloads(namespace string) v2alpha1.DataDownloadInterface { + return &FakeDataDownloads{c, namespace} +} + +func (c *FakeVeleroV2alpha1) DataUploads(namespace string) v2alpha1.DataUploadInterface { + return &FakeDataUploads{c, namespace} +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *FakeVeleroV2alpha1) RESTClient() rest.Interface { + var ret *rest.RESTClient + return ret +} diff --git a/pkg/generated/clientset/versioned/typed/velero/v2alpha1/generated_expansion.go b/pkg/generated/clientset/versioned/typed/velero/v2alpha1/generated_expansion.go new file mode 100644 index 000000000..1ea0b5ae2 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/velero/v2alpha1/generated_expansion.go @@ -0,0 +1,23 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v2alpha1 + +type DataDownloadExpansion interface{} + +type DataUploadExpansion interface{} diff --git a/pkg/generated/clientset/versioned/typed/velero/v2alpha1/velero_client.go b/pkg/generated/clientset/versioned/typed/velero/v2alpha1/velero_client.go new file mode 100644 index 000000000..6b2ea0980 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/velero/v2alpha1/velero_client.go @@ -0,0 +1,94 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v2alpha1 + +import ( + v2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/scheme" + rest "k8s.io/client-go/rest" +) + +type VeleroV2alpha1Interface interface { + RESTClient() rest.Interface + DataDownloadsGetter + DataUploadsGetter +} + +// VeleroV2alpha1Client is used to interact with features provided by the velero.io group. +type VeleroV2alpha1Client struct { + restClient rest.Interface +} + +func (c *VeleroV2alpha1Client) DataDownloads(namespace string) DataDownloadInterface { + return newDataDownloads(c, namespace) +} + +func (c *VeleroV2alpha1Client) DataUploads(namespace string) DataUploadInterface { + return newDataUploads(c, namespace) +} + +// NewForConfig creates a new VeleroV2alpha1Client for the given config. +func NewForConfig(c *rest.Config) (*VeleroV2alpha1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + client, err := rest.RESTClientFor(&config) + if err != nil { + return nil, err + } + return &VeleroV2alpha1Client{client}, nil +} + +// NewForConfigOrDie creates a new VeleroV2alpha1Client for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *VeleroV2alpha1Client { + client, err := NewForConfig(c) + if err != nil { + panic(err) + } + return client +} + +// New creates a new VeleroV2alpha1Client for the given RESTClient. +func New(c rest.Interface) *VeleroV2alpha1Client { + return &VeleroV2alpha1Client{c} +} + +func setConfigDefaults(config *rest.Config) error { + gv := v2alpha1.SchemeGroupVersion + config.GroupVersion = &gv + config.APIPath = "/apis" + config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() + + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } + + return nil +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *VeleroV2alpha1Client) RESTClient() rest.Interface { + if c == nil { + return nil + } + return c.restClient +} diff --git a/pkg/generated/informers/externalversions/generic.go b/pkg/generated/informers/externalversions/generic.go index 605887024..7e0533afc 100644 --- a/pkg/generated/informers/externalversions/generic.go +++ b/pkg/generated/informers/externalversions/generic.go @@ -22,6 +22,7 @@ import ( "fmt" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + v2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" schema "k8s.io/apimachinery/pkg/runtime/schema" cache "k8s.io/client-go/tools/cache" ) @@ -76,6 +77,12 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource case v1.SchemeGroupVersion.WithResource("volumesnapshotlocations"): return &genericInformer{resource: resource.GroupResource(), informer: f.Velero().V1().VolumeSnapshotLocations().Informer()}, nil + // Group=velero.io, Version=v2alpha1 + case v2alpha1.SchemeGroupVersion.WithResource("datadownloads"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Velero().V2alpha1().DataDownloads().Informer()}, nil + case v2alpha1.SchemeGroupVersion.WithResource("datauploads"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Velero().V2alpha1().DataUploads().Informer()}, nil + } return nil, fmt.Errorf("no informer found for %v", resource) diff --git a/pkg/generated/informers/externalversions/velero/interface.go b/pkg/generated/informers/externalversions/velero/interface.go index 767e18b41..87fc652e6 100644 --- a/pkg/generated/informers/externalversions/velero/interface.go +++ b/pkg/generated/informers/externalversions/velero/interface.go @@ -21,12 +21,15 @@ package velero import ( internalinterfaces "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions/internalinterfaces" v1 "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions/velero/v1" + v2alpha1 "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions/velero/v2alpha1" ) // Interface provides access to each of this group's versions. type Interface interface { // V1 provides access to shared informers for resources in V1. V1() v1.Interface + // V2alpha1 provides access to shared informers for resources in V2alpha1. + V2alpha1() v2alpha1.Interface } type group struct { @@ -44,3 +47,8 @@ func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakList func (g *group) V1() v1.Interface { return v1.New(g.factory, g.namespace, g.tweakListOptions) } + +// V2alpha1 returns a new v2alpha1.Interface. +func (g *group) V2alpha1() v2alpha1.Interface { + return v2alpha1.New(g.factory, g.namespace, g.tweakListOptions) +} diff --git a/pkg/generated/informers/externalversions/velero/v2alpha1/datadownload.go b/pkg/generated/informers/externalversions/velero/v2alpha1/datadownload.go new file mode 100644 index 000000000..3b539b332 --- /dev/null +++ b/pkg/generated/informers/externalversions/velero/v2alpha1/datadownload.go @@ -0,0 +1,90 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v2alpha1 + +import ( + "context" + time "time" + + velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + versioned "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned" + internalinterfaces "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions/internalinterfaces" + v2alpha1 "github.com/vmware-tanzu/velero/pkg/generated/listers/velero/v2alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// DataDownloadInformer provides access to a shared informer and lister for +// DataDownloads. +type DataDownloadInformer interface { + Informer() cache.SharedIndexInformer + Lister() v2alpha1.DataDownloadLister +} + +type dataDownloadInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewDataDownloadInformer constructs a new informer for DataDownload type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewDataDownloadInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredDataDownloadInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredDataDownloadInformer constructs a new informer for DataDownload type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredDataDownloadInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.VeleroV2alpha1().DataDownloads(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.VeleroV2alpha1().DataDownloads(namespace).Watch(context.TODO(), options) + }, + }, + &velerov2alpha1.DataDownload{}, + resyncPeriod, + indexers, + ) +} + +func (f *dataDownloadInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredDataDownloadInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *dataDownloadInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&velerov2alpha1.DataDownload{}, f.defaultInformer) +} + +func (f *dataDownloadInformer) Lister() v2alpha1.DataDownloadLister { + return v2alpha1.NewDataDownloadLister(f.Informer().GetIndexer()) +} diff --git a/pkg/generated/informers/externalversions/velero/v2alpha1/dataupload.go b/pkg/generated/informers/externalversions/velero/v2alpha1/dataupload.go new file mode 100644 index 000000000..f7e8f8d07 --- /dev/null +++ b/pkg/generated/informers/externalversions/velero/v2alpha1/dataupload.go @@ -0,0 +1,90 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v2alpha1 + +import ( + "context" + time "time" + + velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + versioned "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned" + internalinterfaces "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions/internalinterfaces" + v2alpha1 "github.com/vmware-tanzu/velero/pkg/generated/listers/velero/v2alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// DataUploadInformer provides access to a shared informer and lister for +// DataUploads. +type DataUploadInformer interface { + Informer() cache.SharedIndexInformer + Lister() v2alpha1.DataUploadLister +} + +type dataUploadInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewDataUploadInformer constructs a new informer for DataUpload type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewDataUploadInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredDataUploadInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredDataUploadInformer constructs a new informer for DataUpload type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredDataUploadInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.VeleroV2alpha1().DataUploads(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.VeleroV2alpha1().DataUploads(namespace).Watch(context.TODO(), options) + }, + }, + &velerov2alpha1.DataUpload{}, + resyncPeriod, + indexers, + ) +} + +func (f *dataUploadInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredDataUploadInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *dataUploadInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&velerov2alpha1.DataUpload{}, f.defaultInformer) +} + +func (f *dataUploadInformer) Lister() v2alpha1.DataUploadLister { + return v2alpha1.NewDataUploadLister(f.Informer().GetIndexer()) +} diff --git a/pkg/generated/informers/externalversions/velero/v2alpha1/interface.go b/pkg/generated/informers/externalversions/velero/v2alpha1/interface.go new file mode 100644 index 000000000..41f8edabf --- /dev/null +++ b/pkg/generated/informers/externalversions/velero/v2alpha1/interface.go @@ -0,0 +1,52 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v2alpha1 + +import ( + internalinterfaces "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions/internalinterfaces" +) + +// Interface provides access to all the informers in this group version. +type Interface interface { + // DataDownloads returns a DataDownloadInformer. + DataDownloads() DataDownloadInformer + // DataUploads returns a DataUploadInformer. + DataUploads() DataUploadInformer +} + +type version struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// DataDownloads returns a DataDownloadInformer. +func (v *version) DataDownloads() DataDownloadInformer { + return &dataDownloadInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + +// DataUploads returns a DataUploadInformer. +func (v *version) DataUploads() DataUploadInformer { + return &dataUploadInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} diff --git a/pkg/generated/listers/velero/v2alpha1/datadownload.go b/pkg/generated/listers/velero/v2alpha1/datadownload.go new file mode 100644 index 000000000..dadf14b60 --- /dev/null +++ b/pkg/generated/listers/velero/v2alpha1/datadownload.go @@ -0,0 +1,99 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v2alpha1 + +import ( + v2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// DataDownloadLister helps list DataDownloads. +// All objects returned here must be treated as read-only. +type DataDownloadLister interface { + // List lists all DataDownloads in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v2alpha1.DataDownload, err error) + // DataDownloads returns an object that can list and get DataDownloads. + DataDownloads(namespace string) DataDownloadNamespaceLister + DataDownloadListerExpansion +} + +// dataDownloadLister implements the DataDownloadLister interface. +type dataDownloadLister struct { + indexer cache.Indexer +} + +// NewDataDownloadLister returns a new DataDownloadLister. +func NewDataDownloadLister(indexer cache.Indexer) DataDownloadLister { + return &dataDownloadLister{indexer: indexer} +} + +// List lists all DataDownloads in the indexer. +func (s *dataDownloadLister) List(selector labels.Selector) (ret []*v2alpha1.DataDownload, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v2alpha1.DataDownload)) + }) + return ret, err +} + +// DataDownloads returns an object that can list and get DataDownloads. +func (s *dataDownloadLister) DataDownloads(namespace string) DataDownloadNamespaceLister { + return dataDownloadNamespaceLister{indexer: s.indexer, namespace: namespace} +} + +// DataDownloadNamespaceLister helps list and get DataDownloads. +// All objects returned here must be treated as read-only. +type DataDownloadNamespaceLister interface { + // List lists all DataDownloads in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v2alpha1.DataDownload, err error) + // Get retrieves the DataDownload from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*v2alpha1.DataDownload, error) + DataDownloadNamespaceListerExpansion +} + +// dataDownloadNamespaceLister implements the DataDownloadNamespaceLister +// interface. +type dataDownloadNamespaceLister struct { + indexer cache.Indexer + namespace string +} + +// List lists all DataDownloads in the indexer for a given namespace. +func (s dataDownloadNamespaceLister) List(selector labels.Selector) (ret []*v2alpha1.DataDownload, err error) { + err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v2alpha1.DataDownload)) + }) + return ret, err +} + +// Get retrieves the DataDownload from the indexer for a given namespace and name. +func (s dataDownloadNamespaceLister) Get(name string) (*v2alpha1.DataDownload, error) { + obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v2alpha1.Resource("datadownload"), name) + } + return obj.(*v2alpha1.DataDownload), nil +} diff --git a/pkg/generated/listers/velero/v2alpha1/dataupload.go b/pkg/generated/listers/velero/v2alpha1/dataupload.go new file mode 100644 index 000000000..0dbe6bed1 --- /dev/null +++ b/pkg/generated/listers/velero/v2alpha1/dataupload.go @@ -0,0 +1,99 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v2alpha1 + +import ( + v2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// DataUploadLister helps list DataUploads. +// All objects returned here must be treated as read-only. +type DataUploadLister interface { + // List lists all DataUploads in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v2alpha1.DataUpload, err error) + // DataUploads returns an object that can list and get DataUploads. + DataUploads(namespace string) DataUploadNamespaceLister + DataUploadListerExpansion +} + +// dataUploadLister implements the DataUploadLister interface. +type dataUploadLister struct { + indexer cache.Indexer +} + +// NewDataUploadLister returns a new DataUploadLister. +func NewDataUploadLister(indexer cache.Indexer) DataUploadLister { + return &dataUploadLister{indexer: indexer} +} + +// List lists all DataUploads in the indexer. +func (s *dataUploadLister) List(selector labels.Selector) (ret []*v2alpha1.DataUpload, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v2alpha1.DataUpload)) + }) + return ret, err +} + +// DataUploads returns an object that can list and get DataUploads. +func (s *dataUploadLister) DataUploads(namespace string) DataUploadNamespaceLister { + return dataUploadNamespaceLister{indexer: s.indexer, namespace: namespace} +} + +// DataUploadNamespaceLister helps list and get DataUploads. +// All objects returned here must be treated as read-only. +type DataUploadNamespaceLister interface { + // List lists all DataUploads in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v2alpha1.DataUpload, err error) + // Get retrieves the DataUpload from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*v2alpha1.DataUpload, error) + DataUploadNamespaceListerExpansion +} + +// dataUploadNamespaceLister implements the DataUploadNamespaceLister +// interface. +type dataUploadNamespaceLister struct { + indexer cache.Indexer + namespace string +} + +// List lists all DataUploads in the indexer for a given namespace. +func (s dataUploadNamespaceLister) List(selector labels.Selector) (ret []*v2alpha1.DataUpload, err error) { + err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v2alpha1.DataUpload)) + }) + return ret, err +} + +// Get retrieves the DataUpload from the indexer for a given namespace and name. +func (s dataUploadNamespaceLister) Get(name string) (*v2alpha1.DataUpload, error) { + obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v2alpha1.Resource("dataupload"), name) + } + return obj.(*v2alpha1.DataUpload), nil +} diff --git a/pkg/generated/listers/velero/v2alpha1/expansion_generated.go b/pkg/generated/listers/velero/v2alpha1/expansion_generated.go new file mode 100644 index 000000000..1bdb85ec0 --- /dev/null +++ b/pkg/generated/listers/velero/v2alpha1/expansion_generated.go @@ -0,0 +1,35 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v2alpha1 + +// DataDownloadListerExpansion allows custom methods to be added to +// DataDownloadLister. +type DataDownloadListerExpansion interface{} + +// DataDownloadNamespaceListerExpansion allows custom methods to be added to +// DataDownloadNamespaceLister. +type DataDownloadNamespaceListerExpansion interface{} + +// DataUploadListerExpansion allows custom methods to be added to +// DataUploadLister. +type DataUploadListerExpansion interface{} + +// DataUploadNamespaceListerExpansion allows custom methods to be added to +// DataUploadNamespaceLister. +type DataUploadNamespaceListerExpansion interface{} diff --git a/pkg/install/resources.go b/pkg/install/resources.go index 0d60e9743..d7014c22c 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" v1crds "github.com/vmware-tanzu/velero/config/crd/v1/crds" + v2alpha1crds "github.com/vmware-tanzu/velero/config/crd/v2alpha1/crds" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) @@ -252,7 +253,14 @@ func AllCRDs() *unstructured.UnstructuredList { for _, crd := range v1crds.CRDs { crd.SetLabels(Labels()) if err := appendUnstructured(resources, crd); err != nil { - fmt.Printf("error appending CRD %s: %s\n", crd.GetName(), err.Error()) + fmt.Printf("error appending v1 CRD %s: %s\n", crd.GetName(), err.Error()) + } + } + + for _, crd := range v2alpha1crds.CRDs { + crd.SetLabels(Labels()) + if err := appendUnstructured(resources, crd); err != nil { + fmt.Printf("error appending v2alpha1 CRD %s: %s\n", crd.GetName(), err.Error()) } } diff --git a/pkg/plugin/framework/common/plugin_config.go b/pkg/plugin/framework/common/plugin_config.go index f2cb133ec..58f9bafc6 100644 --- a/pkg/plugin/framework/common/plugin_config.go +++ b/pkg/plugin/framework/common/plugin_config.go @@ -27,7 +27,7 @@ import ( ) func PluginConfigLabelSelector(kind PluginKind, name string) string { - return fmt.Sprintf("velero.io/plugin-config=true,%s=%s", name, kind) + return fmt.Sprintf("velero.io/plugin-config,%s=%s", name, kind) } func GetPluginConfig(kind PluginKind, name string, client corev1client.ConfigMapInterface) (*corev1.ConfigMap, error) { diff --git a/pkg/plugin/framework/common/plugin_config_test.go b/pkg/plugin/framework/common/plugin_config_test.go index 107ff9591..3a69e1ef0 100644 --- a/pkg/plugin/framework/common/plugin_config_test.go +++ b/pkg/plugin/framework/common/plugin_config_test.go @@ -22,7 +22,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" @@ -35,7 +34,7 @@ func TestGetPluginConfig(t *testing.T) { name string objects []runtime.Object } - pluginLabelsSet, _ := labels.ConvertSelectorToLabelsMap(PluginConfigLabelSelector(PluginKindRestoreItemAction, "foo")) + pluginLabelsMap := map[string]string{"velero.io/plugin-config": "", "foo": "RestoreItemAction"} testConfigMap := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ Kind: "ConfigMap", @@ -43,7 +42,7 @@ func TestGetPluginConfig(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "foo-config", Namespace: velerov1.DefaultNamespace, - Labels: pluginLabelsSet, + Labels: pluginLabelsMap, }, } tests := []struct { @@ -75,7 +74,7 @@ func TestGetPluginConfig(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "foo-config", Namespace: velerov1.DefaultNamespace, - Labels: pluginLabelsSet, + Labels: pluginLabelsMap, }, }, &corev1.ConfigMap{ @@ -85,7 +84,7 @@ func TestGetPluginConfig(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "foo-config-duplicate", Namespace: velerov1.DefaultNamespace, - Labels: pluginLabelsSet, + Labels: pluginLabelsMap, }, }, }, diff --git a/pkg/podvolume/backupper.go b/pkg/podvolume/backupper.go index 870300739..3e99636be 100644 --- a/pkg/podvolume/backupper.go +++ b/pkg/podvolume/backupper.go @@ -131,7 +131,7 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api. if len(volumesToBackup) == 0 { return nil, nil } - + log.Infof("pod %s/%s has volumes to backup: %v", pod.Namespace, pod.Name, volumesToBackup) err := kube.IsPodRunning(pod) if err != nil { for _, volumeName := range volumesToBackup { diff --git a/pkg/podvolume/backupper_test.go b/pkg/podvolume/backupper_test.go index fb0cacd1a..f86d9ebc1 100644 --- a/pkg/podvolume/backupper_test.go +++ b/pkg/podvolume/backupper_test.go @@ -17,13 +17,19 @@ limitations under the License. package podvolume import ( + "bytes" "context" + "fmt" "testing" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/vmware-tanzu/velero/internal/resourcepolicies" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) func TestIsHostPathVolume(t *testing.T) { @@ -139,3 +145,62 @@ func (g *fakePVGetter) Get(ctx context.Context, name string, opts metav1.GetOpti return nil, errors.New("item not found") } + +func Test_backupper_BackupPodVolumes_log_test(t *testing.T) { + type args struct { + backup *velerov1api.Backup + pod *corev1api.Pod + volumesToBackup []string + resPolicies *resourcepolicies.Policies + } + tests := []struct { + name string + args args + wantLog string + }{ + { + name: "backup pod volumes should log volume names", + args: args{ + backup: &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "backup-1", + Namespace: "ns-1", + }, + }, + pod: &corev1api.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-1", + Namespace: "ns-1", + }, + Spec: corev1api.PodSpec{ + Volumes: []corev1api.Volume{ + { + Name: "vol-1", + }, + { + Name: "vol-2", + }, + }, + }, + }, + volumesToBackup: []string{"vol-1", "vol-2"}, + resPolicies: nil, + }, + wantLog: "pod ns-1/pod-1 has volumes to backup: [vol-1 vol-2]", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &backupper{ + ctx: context.Background(), + } + logOutput := bytes.Buffer{} + var log = logrus.New() + log.SetOutput(&logOutput) + b.BackupPodVolumes(tt.args.backup, tt.args.pod, tt.args.volumesToBackup, tt.args.resPolicies, log) + fmt.Println(logOutput.String()) + assert.Contains(t, logOutput.String(), tt.wantLog) + + }) + } +} diff --git a/pkg/repository/provider/unified_repo.go b/pkg/repository/provider/unified_repo.go index 7a6212f3e..592d271ac 100644 --- a/pkg/repository/provider/unified_repo.go +++ b/pkg/repository/provider/unified_repo.go @@ -18,6 +18,7 @@ package provider import ( "context" + "encoding/base64" "fmt" "net/url" "path" @@ -498,6 +499,10 @@ func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repo result[udmrepo.StoreOptionS3Endpoint] = strings.Trim(s3URL, "/") result[udmrepo.StoreOptionS3DisableTLSVerify] = config["insecureSkipTLSVerify"] result[udmrepo.StoreOptionS3DisableTLS] = strconv.FormatBool(disableTLS) + + if backupLocation.Spec.ObjectStorage != nil && backupLocation.Spec.ObjectStorage.CACert != nil { + result[udmrepo.StoreOptionS3CustomCA] = base64.StdEncoding.EncodeToString(backupLocation.Spec.ObjectStorage.CACert) + } } else if backendType == repoconfig.AzureBackend { domain, err := getAzureStorageDomain(config) if err != nil { diff --git a/pkg/repository/provider/unified_repo_test.go b/pkg/repository/provider/unified_repo_test.go index 098e3a23b..80b2611df 100644 --- a/pkg/repository/provider/unified_repo_test.go +++ b/pkg/repository/provider/unified_repo_test.go @@ -18,6 +18,7 @@ package provider import ( "context" + "encoding/base64" "errors" "testing" @@ -380,6 +381,42 @@ func TestGetStorageVariables(t *testing.T) { "skipTLSVerify": "false", }, }, + { + name: "aws, ObjectStorage section exists in BSL, s3Url exist, https, custom CA exist", + backupLocation: velerov1api.BackupStorageLocation{ + Spec: velerov1api.BackupStorageLocationSpec{ + Provider: "velero.io/aws", + Config: map[string]string{ + "bucket": "fake-bucket-config", + "prefix": "fake-prefix-config", + "region": "fake-region", + "s3Url": "https://fake-url/", + "insecureSkipTLSVerify": "false", + }, + StorageType: velerov1api.StorageType{ + ObjectStorage: &velerov1api.ObjectStorageLocation{ + Bucket: "fake-bucket-object-store", + Prefix: "fake-prefix-object-store", + CACert: []byte{0x01, 0x02, 0x03, 0x04, 0x05}, + }, + }, + }, + }, + getS3BucketRegion: func(bucket string) (string, error) { + return "region from bucket: " + bucket, nil + }, + repoBackend: "fake-repo-type", + expected: map[string]string{ + "bucket": "fake-bucket-object-store", + "prefix": "fake-prefix-object-store/fake-repo-type/", + "region": "fake-region", + "fspath": "", + "endpoint": "fake-url", + "doNotUseTLS": "false", + "skipTLSVerify": "false", + "customCA": base64.StdEncoding.EncodeToString([]byte{0x01, 0x02, 0x03, 0x04, 0x05}), + }, + }, { name: "azure, getAzureStorageDomain fail", backupLocation: velerov1api.BackupStorageLocation{ diff --git a/pkg/repository/udmrepo/kopialib/backend/s3.go b/pkg/repository/udmrepo/kopialib/backend/s3.go index b3891a36a..43a7fc821 100644 --- a/pkg/repository/udmrepo/kopialib/backend/s3.go +++ b/pkg/repository/udmrepo/kopialib/backend/s3.go @@ -44,6 +44,7 @@ func (c *S3Backend) Setup(ctx context.Context, flags map[string]string) error { c.options.DoNotUseTLS = optionalHaveBool(ctx, udmrepo.StoreOptionS3DisableTLS, flags) c.options.DoNotVerifyTLS = optionalHaveBool(ctx, udmrepo.StoreOptionS3DisableTLSVerify, flags) c.options.SessionToken = optionalHaveString(udmrepo.StoreOptionS3Token, flags) + c.options.RootCA = optionalHaveBase64(ctx, udmrepo.StoreOptionS3CustomCA, flags) c.options.Limits = setupLimits(ctx, flags) diff --git a/pkg/repository/udmrepo/kopialib/backend/utils.go b/pkg/repository/udmrepo/kopialib/backend/utils.go index 97fa2ab57..19fa7b278 100644 --- a/pkg/repository/udmrepo/kopialib/backend/utils.go +++ b/pkg/repository/udmrepo/kopialib/backend/utils.go @@ -18,6 +18,7 @@ package backend import ( "context" + "encoding/base64" "strconv" "time" @@ -84,6 +85,19 @@ func optionalHaveDuration(ctx context.Context, key string, flags map[string]stri return 0 } +func optionalHaveBase64(ctx context.Context, key string, flags map[string]string) []byte { + if value, exist := flags[key]; exist { + ret, err := base64.StdEncoding.DecodeString(value) + if err == nil { + return ret + } + + backendLog()(ctx).Errorf("Ignore %s, value [%s] is invalid, err %v", key, value, err) + } + + return nil +} + func backendLog() func(ctx context.Context) logging.Logger { return logging.Module("kopialib-bd") } diff --git a/pkg/repository/udmrepo/repo_options.go b/pkg/repository/udmrepo/repo_options.go index 4f1276514..9337018f2 100644 --- a/pkg/repository/udmrepo/repo_options.go +++ b/pkg/repository/udmrepo/repo_options.go @@ -42,6 +42,7 @@ const ( StoreOptionS3Endpoint = "endpoint" StoreOptionS3DisableTLS = "doNotUseTLS" StoreOptionS3DisableTLSVerify = "skipTLSVerify" + StoreOptionS3CustomCA = "customCA" StoreOptionAzureKey = "storageKey" StoreOptionAzureDomain = "storageDomain" diff --git a/pkg/restore/change_image_name_action_test.go b/pkg/restore/change_image_name_action_test.go index cc2c8ff66..d851e7c0d 100644 --- a/pkg/restore/change_image_name_action_test.go +++ b/pkg/restore/change_image_name_action_test.go @@ -55,7 +55,7 @@ func TestChangeImageRepositoryActionExecute(t *testing.T) { Image: "1.1.1.1:5000/abc:test", }).Result(), configMap: builder.ForConfigMap("velero", "change-image-name"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-image-name", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-image-name", "RestoreItemAction")). Data("case1", "1.1.1.1:5000 , 2.2.2.2:3000"). Result(), freshedImageName: "2.2.2.2:3000/abc:test", @@ -70,7 +70,7 @@ func TestChangeImageRepositoryActionExecute(t *testing.T) { Image: "1.1.1.1:5000/abc:test", }).Result(), configMap: builder.ForConfigMap("velero", "change-image-name"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-image-name", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-image-name", "RestoreItemAction")). Data("specific", "1.1.1.1:5000,2.2.2.2:3000"). Result(), freshedImageName: "2.2.2.2:3000/abc:test", @@ -85,7 +85,7 @@ func TestChangeImageRepositoryActionExecute(t *testing.T) { Image: "1.1.1.1:5000/abc:test", }).Result(), configMap: builder.ForConfigMap("velero", "change-image-name"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-image-name", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-image-name", "RestoreItemAction")). Data("specific", "abc:test,myproject:latest"). Result(), freshedImageName: "1.1.1.1:5000/myproject:latest", @@ -100,7 +100,7 @@ func TestChangeImageRepositoryActionExecute(t *testing.T) { Image: "1.1.1.1:5000/abc:test", }).Result(), configMap: builder.ForConfigMap("velero", "change-image-name"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-image-name", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-image-name", "RestoreItemAction")). Data("specific", "5000,3333"). Result(), freshedImageName: "1.1.1.1:5000/abc:test", @@ -115,7 +115,7 @@ func TestChangeImageRepositoryActionExecute(t *testing.T) { Image: "1.1.1.1:5000/abc:test", }).Result(), configMap: builder.ForConfigMap("velero", "change-image-name"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-image-name", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-image-name", "RestoreItemAction")). Data("specific", "test,latest"). Result(), freshedImageName: "1.1.1.1:5000/abc:test", @@ -130,7 +130,7 @@ func TestChangeImageRepositoryActionExecute(t *testing.T) { Image: "dev/image1:dev", }).Result(), configMap: builder.ForConfigMap("velero", "change-image-name"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-image-name", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-image-name", "RestoreItemAction")). Data("specific", "dev/,test/"). Result(), freshedImageName: "dev/image1:dev", diff --git a/pkg/restore/change_pvc_node_selector_test.go b/pkg/restore/change_pvc_node_selector_test.go index faca154ae..7cfbd170c 100644 --- a/pkg/restore/change_pvc_node_selector_test.go +++ b/pkg/restore/change_pvc_node_selector_test.go @@ -57,7 +57,7 @@ func TestChangePVCNodeSelectorActionExecute(t *testing.T) { builder.WithAnnotations("volume.kubernetes.io/selected-node", "source-node"), ).Result(), configMap: builder.ForConfigMap("velero", "change-pvc-node"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-pvc-node-selector", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-pvc-node-selector", "RestoreItemAction")). Data("source-node", "dest-node"). Result(), newNode: builder.ForNode("dest-node").Result(), @@ -73,7 +73,7 @@ func TestChangePVCNodeSelectorActionExecute(t *testing.T) { builder.WithAnnotations("volume.kubernetes.io/selected-node", "source-node"), ).Result(), configMap: builder.ForConfigMap("velero", "change-pvc-node"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/some-other-plugin", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/some-other-plugin", "RestoreItemAction")). Data("source-noed", "dest-node"). Result(), want: builder.ForPersistentVolumeClaim("source-ns", "pvc-1").Result(), @@ -85,7 +85,7 @@ func TestChangePVCNodeSelectorActionExecute(t *testing.T) { builder.WithAnnotations("volume.kubernetes.io/selected-node", "source-node"), ).Result(), configMap: builder.ForConfigMap("velero", "change-pvc-node"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-pvc-node-selector", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-pvc-node-selector", "RestoreItemAction")). Result(), want: builder.ForPersistentVolumeClaim("source-ns", "pvc-1").Result(), }, @@ -96,7 +96,7 @@ func TestChangePVCNodeSelectorActionExecute(t *testing.T) { builder.WithAnnotations("volume.kubernetes.io/selected-node", "source-node"), ).Result(), configMap: builder.ForConfigMap("velero", "change-pvc-node"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-pvc-node-selector", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-pvc-node-selector", "RestoreItemAction")). Result(), // MAYANK TODO node: builder.ForNode("source-node").Result(), @@ -109,7 +109,7 @@ func TestChangePVCNodeSelectorActionExecute(t *testing.T) { name: "when persistent volume claim has no node selector, the item is returned as-is", pvc: builder.ForPersistentVolumeClaim("source-ns", "pvc-1").Result(), configMap: builder.ForConfigMap("velero", "change-pvc-node"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-pvc-node-selector", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-pvc-node-selector", "RestoreItemAction")). Data("source-node", "dest-node"). Result(), want: builder.ForPersistentVolumeClaim("source-ns", "pvc-1").Result(), @@ -121,7 +121,7 @@ func TestChangePVCNodeSelectorActionExecute(t *testing.T) { builder.WithAnnotations("volume.kubernetes.io/selected-node", "source-node"), ).Result(), configMap: builder.ForConfigMap("velero", "change-pvc-node"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-pvc-node-selector", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-pvc-node-selector", "RestoreItemAction")). Data("source-node-1", "dest-node"). Result(), want: builder.ForPersistentVolumeClaim("source-ns", "pvc-1").Result(), diff --git a/pkg/restore/change_storageclass_action_test.go b/pkg/restore/change_storageclass_action_test.go index 65de052db..7f701a0d6 100644 --- a/pkg/restore/change_storageclass_action_test.go +++ b/pkg/restore/change_storageclass_action_test.go @@ -53,7 +53,7 @@ func TestChangeStorageClassActionExecute(t *testing.T) { name: "a valid mapping for a persistent volume is applied correctly", pvOrPvcOrSTS: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "storageclass-2"). Result(), storageClass: builder.ForStorageClass("storageclass-2").Result(), @@ -63,7 +63,7 @@ func TestChangeStorageClassActionExecute(t *testing.T) { name: "a valid mapping for a persistent volume claim is applied correctly", pvOrPvcOrSTS: builder.ForPersistentVolumeClaim("velero", "pvc-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "storageclass-2"). Result(), storageClass: builder.ForStorageClass("storageclass-2").Result(), @@ -73,7 +73,7 @@ func TestChangeStorageClassActionExecute(t *testing.T) { name: "when no config map exists for the plugin, the item is returned as-is", pvOrPvcOrSTS: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/some-other-plugin", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/some-other-plugin", "RestoreItemAction")). Data("storageclass-1", "storageclass-2"). Result(), want: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), @@ -82,7 +82,7 @@ func TestChangeStorageClassActionExecute(t *testing.T) { name: "when no storage class mappings exist in the plugin config map, the item is returned as-is", pvOrPvcOrSTS: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Result(), want: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), }, @@ -90,7 +90,7 @@ func TestChangeStorageClassActionExecute(t *testing.T) { name: "when persistent volume has no storage class, the item is returned as-is", pvOrPvcOrSTS: builder.ForPersistentVolume("pv-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "storageclass-2"). Result(), want: builder.ForPersistentVolume("pv-1").Result(), @@ -99,7 +99,7 @@ func TestChangeStorageClassActionExecute(t *testing.T) { name: "when persistent volume claim has no storage class, the item is returned as-is", pvOrPvcOrSTS: builder.ForPersistentVolumeClaim("velero", "pvc-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "storageclass-2"). Result(), want: builder.ForPersistentVolumeClaim("velero", "pvc-1").Result(), @@ -108,7 +108,7 @@ func TestChangeStorageClassActionExecute(t *testing.T) { name: "when persistent volume's storage class has no mapping in the config map, the item is returned as-is", pvOrPvcOrSTS: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-3", "storageclass-4"). Result(), want: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), @@ -117,7 +117,7 @@ func TestChangeStorageClassActionExecute(t *testing.T) { name: "when persistent volume claim's storage class has no mapping in the config map, the item is returned as-is", pvOrPvcOrSTS: builder.ForPersistentVolumeClaim("velero", "pvc-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-3", "storageclass-4"). Result(), want: builder.ForPersistentVolumeClaim("velero", "pvc-1").StorageClass("storageclass-1").Result(), @@ -126,7 +126,7 @@ func TestChangeStorageClassActionExecute(t *testing.T) { name: "when persistent volume's storage class is mapped to a nonexistent storage class, an error is returned", pvOrPvcOrSTS: builder.ForPersistentVolume("pv-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "nonexistent-storage-class"). Result(), wantErr: errors.New("error getting storage class nonexistent-storage-class from API: storageclasses.storage.k8s.io \"nonexistent-storage-class\" not found"), @@ -135,7 +135,7 @@ func TestChangeStorageClassActionExecute(t *testing.T) { name: "when persistent volume claim's storage class is mapped to a nonexistent storage class, an error is returned", pvOrPvcOrSTS: builder.ForPersistentVolumeClaim("velero", "pvc-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "nonexistent-storage-class"). Result(), wantErr: errors.New("error getting storage class nonexistent-storage-class from API: storageclasses.storage.k8s.io \"nonexistent-storage-class\" not found"), @@ -144,7 +144,7 @@ func TestChangeStorageClassActionExecute(t *testing.T) { name: "when statefulset's VolumeClaimTemplates has only one pvc, a valid mapping for a statefulset is applied correctly", pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "storageclass-2"). Result(), storageClass: builder.ForStorageClass("storageclass-2").Result(), @@ -154,7 +154,7 @@ func TestChangeStorageClassActionExecute(t *testing.T) { name: "when statefulset's VolumeClaimTemplates has more than one same pvc's storageClassName, a valid mapping for a statefulset is applied correctly", pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1", "storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "storageclass-2", "storageclass-3", "storageclass-4"). Result(), storageClass: builder.ForStorageClass("storageclass-2").Result(), @@ -164,7 +164,7 @@ func TestChangeStorageClassActionExecute(t *testing.T) { name: "when statefulset's VolumeClaimTemplates has more than one different pvc's storageClassName, a valid mapping for a statefulset is applied correctly", pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1", "storageclass-2", "storageclass-3").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "storageclass-a", "storageclass-2", "storageclass-b", "storageclass-3", "storageclass-c"). Result(), storageClassSlice: builder.ForStorageClassSlice("storageclass-a", "storageclass-b", "storageclass-c").SliceResult(), @@ -174,7 +174,7 @@ func TestChangeStorageClassActionExecute(t *testing.T) { name: "when no config map exists for the plugin, the statefulset item is returned as-is", pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/some-other-plugin", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/some-other-plugin", "RestoreItemAction")). Data("storageclass-1", "storageclass-2"). Result(), want: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), @@ -183,7 +183,7 @@ func TestChangeStorageClassActionExecute(t *testing.T) { name: "when no storage class mappings exist in the plugin config map, the statefulset item is returned as-is", pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Result(), want: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), }, @@ -191,7 +191,7 @@ func TestChangeStorageClassActionExecute(t *testing.T) { name: "when persistent volume claim has no storage class, the statefulset item is returned as-is", pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Result(), want: builder.ForStatefulSet("velero", "sts-1").Result(), }, @@ -199,7 +199,7 @@ func TestChangeStorageClassActionExecute(t *testing.T) { name: "when statefulset's storage class has no mapping in the config map, the item is returned as-is", pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-3", "storageclass-4"). Result(), want: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), @@ -208,7 +208,7 @@ func TestChangeStorageClassActionExecute(t *testing.T) { name: "when statefulset's storage class is mapped to a nonexistent storage class, an error is returned", pvOrPvcOrSTS: builder.ForStatefulSet("velero", "sts-1").StorageClass("storageclass-1").Result(), configMap: builder.ForConfigMap("velero", "change-storage-classs"). - ObjectMeta(builder.WithLabels("velero.io/plugin-config", "true", "velero.io/change-storage-class", "RestoreItemAction")). + ObjectMeta(builder.WithLabels("velero.io/plugin-config", "", "velero.io/change-storage-class", "RestoreItemAction")). Data("storageclass-1", "nonexistent-storage-class"). Result(), wantErr: errors.New("error getting storage class nonexistent-storage-class from API: storageclasses.storage.k8s.io \"nonexistent-storage-class\" not found"), diff --git a/pkg/test/fake_controller_runtime_client.go b/pkg/test/fake_controller_runtime_client.go index 97487d9ab..b64f57809 100644 --- a/pkg/test/fake_controller_runtime_client.go +++ b/pkg/test/fake_controller_runtime_client.go @@ -27,12 +27,15 @@ import ( k8sfake "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" ) func NewFakeControllerRuntimeClientBuilder(t *testing.T) *k8sfake.ClientBuilder { scheme := runtime.NewScheme() err := velerov1api.AddToScheme(scheme) require.NoError(t, err) + err = velerov2alpha1api.AddToScheme(scheme) + require.NoError(t, err) err = corev1api.AddToScheme(scheme) require.NoError(t, err) err = snapshotv1api.AddToScheme(scheme) @@ -44,6 +47,8 @@ func NewFakeControllerRuntimeClient(t *testing.T, initObjs ...runtime.Object) cl scheme := runtime.NewScheme() err := velerov1api.AddToScheme(scheme) require.NoError(t, err) + err = velerov2alpha1api.AddToScheme(scheme) + require.NoError(t, err) err = corev1api.AddToScheme(scheme) require.NoError(t, err) err = snapshotv1api.AddToScheme(scheme) diff --git a/pkg/uploader/kopia/shim.go b/pkg/uploader/kopia/shim.go index cb7a570c6..23fb6c4a3 100644 --- a/pkg/uploader/kopia/shim.go +++ b/pkg/uploader/kopia/shim.go @@ -174,6 +174,7 @@ func (sr *shimRepository) NewObjectWriter(ctx context.Context, option object.Wri opt.Prefix = udmrepo.ID(option.Prefix) opt.FullPath = "" opt.AccessMode = udmrepo.ObjectDataAccessModeFile + opt.AsyncWrites = option.AsyncWrites if strings.HasPrefix(option.Description, "DIR:") { opt.DataType = udmrepo.ObjectDataTypeMetadata diff --git a/site/content/docs/main/api-types/schedule.md b/site/content/docs/main/api-types/schedule.md index 58a4c454b..1bc4a81c3 100644 --- a/site/content/docs/main/api-types/schedule.md +++ b/site/content/docs/main/api-types/schedule.md @@ -94,6 +94,13 @@ spec: matchLabels: app: velero component: server + # Individual object when matched with any of the label selector specified in the set are to be included in the backup. Optional. + # orLabelSelectors as well as labelSelector cannot co-exist, only one of them can be specified in the backup request + orLabelSelectors: + - matchLabels: + app: velero + - matchLabels: + app: data-protection # Whether to snapshot volumes. Valid values are true, false, and null/unset. If unset, Velero performs snapshots as long as # a persistent volume provider is configured for Velero. snapshotVolumes: null diff --git a/site/content/posts/2023-04-26-Velero-1.11.md b/site/content/posts/2023-04-26-Velero-1.11.md new file mode 100644 index 000000000..f39a161af --- /dev/null +++ b/site/content/posts/2023-04-26-Velero-1.11.md @@ -0,0 +1,109 @@ +--- +title: "Velero 1.11: New Actions, New Horizons" +excerpt: In this release, we've grown the team and continue to welcome new members to our community.We're thrilled to have such significant contributions from the community and we're proud to deliver Velero 1.11. +author_name: Orlin Vasilev +slug: Velero-1.11 +categories: ['velero','release'] +image: /img/posts/post-1.11.jpg +# Tag should match author to drive author pages +tags: ['Velero Team', 'Orlin Vasilev', 'Velero Release'] +--- + +We haven't posted for while, but this deserves your attention! +Last week during KubeCon Europe in Amsterdam we released [Velero v1.11](https://github.com/vmware-tanzu/velero/releases/tag/v1.11.0), which brings significant improvements in its functionality, flexibility, and performance. In this blog post, we will discuss the new features and changes that come with Velero v1.11 and how they can benefit users. + +The theme of the v1.11 release is most definitely more flexibility, not only with the features added, but also in terms of the Velero contribution, development, and quality assurance processes. + +In case you missed it we have new Product Manager - [Pradeep Kumar Chaturvedi](https://github.com/pradeepkchaturvedi) and few new contributors from companies like DELL and Microsoft. + +### Full list of changes can be found [here](https://github.com/vmware-tanzu/velero/releases/tag/v1.11.0) + +## Release Highlights + +### BackupItemAction v2 +This feature implements the BackupItemAction v2. BIA v2 has two new methods: Progress() and Cancel() and modifies the Execute() return value. + +The API change is needed to facilitate long-running BackupItemAction plugin actions that may not be complete when the Execute() method returns. This will allow long-running BackupItemAction plugin actions to continue in the background while the Velero moves to the following plugin or the next item. +[https://github.com/vmware-tanzu/velero/pull/5442](https://github.com/vmware-tanzu/velero/pull/5442) + +### RestoreItemAction v2 +This feature implemented the RestoreItemAction v2. RIA v2 has three new methods: Progress(), Cancel(), and AreAdditionalItemsReady(), and it modifies RestoreItemActionExecuteOutput() structure in the RIA return value. + +The Progress() and Cancel() methods are needed to facilitate long-running RestoreItemAction plugin actions that may not be complete when the Execute() method returns. This will allow long-running RestoreItemAction plugin actions to continue in the background while the Velero moves to the following plugin or the next item. The AreAdditionalItemsReady() method is needed to allow plugins to tell Velero to wait until the returned additional items have been restored and are ready for use in the cluster before restoring the current item. + +[https://github.com/vmware-tanzu/velero/pull/5569](https://github.com/vmware-tanzu/velero/pull/5569) + +### Plugin Progress Monitoring +This is intended as a replacement for the previously-approved Upload Progress Monitoring design ([Upload Progress Monitoring](https://github.com/vmware-tanzu/velero/blob/main/design/upload-progress.md)) to expand the supported use cases beyond snapshot upload to include what was previously called Async Backup/Restore Item Actions. + +### Flexible resource policy that can filter volumes to skip in the backup +This feature provides a flexible policy to filter volumes in the backup without requiring patching any labels or annotations to the pods or volumes. This policy is configured as k8s ConfigMap and maintained by the users themselves, and it can be extended to more scenarios in the future. By now, the policy rules out volumes from backup depending on the CSI driver, NFS setting, volume size, and StorageClass setting. Please refer to [Resource policies rules](https://velero.io/docs/v1.11/resource-filtering/#resource-policies) for the policy's ConifgMap format. It is not guaranteed to work on unofficial third-party plugins as it may not follow the existing backup workflow code logic of Velero. + +### Resource Filters that can distinguish cluster scope and namespace scope resources +This feature adds four new resource filters for backup. The new filters are separated into cluster scope and namespace scope. Before this feature, Velero could not filter cluster scope resources precisely. This feature provides the ability and refactors existing resource filter parameters. + +### New parameter in installation to customize the ServiceAccount name +The `velero install` sub-command now includes a new parameter,`--service-account-name`, which allows users to specify the ServiceAccountName for the Velero and node-agent pods. This feature may be particularly useful for users who utilize IRSA (IAM Roles for Service Accounts) in Amazon EKS (Elastic Kubernetes Service)." + +### Add a parameter for setting the Velero server connection with the k8s API server's timeout +In Velero, some code pieces need to communicate with the k8s API server. Before v1.11, these code pieces used hard-code timeout settings. This feature adds a resource-timeout parameter in the velero server binary to make it configurable. + +### Add resource list in the output of the restore describe command +Before this feature, Velero restore didn't have a restored resources list as the Velero backup. It's not convenient for users to learn what is restored. This feature adds the resources list and the handling result of the resources (including created, updated, failed, and skipped). + +### Support JSON format output of backup describe command +Before the Velero v1.11 release, users could not choose Velero's backup describe command's output format. The command output format is friendly for human reading, but it's not a structured output, and it's not easy for other programs to get information from it. Velero v1.11 adds a JSON format output for the backup describe command. + +### Refactor controllers with controller-runtime +In v1.11, Backup Controller and Restore controller are refactored with controller-runtime. Till v1.11, all Velero controllers use the controller-runtime framework. + +### Runtime and dependencies +To fix CVEs and keep pace with Golang, Velero made changes as follows: +* Bump Golang runtime to v1.19.8. +* Bump several dependent libraries to new versions. +* Compile Restic (v0.15.0) with Golang v1.19.8 instead of packaging the official binary. + + +## Breaking changes +* The Velero CSI plugin now determines whether to restore Volume's data from snapshots on the restore's restorePVs setting. Before v1.11, the CSI plugin doesn't check the restorePVs parameter setting. + + +## Limitations/Known issues +* The Flexible resource policy that can filter volumes to skip in the backup is not guaranteed to work on unofficial third-party plugins because the plugins may not follow the existing backup workflow code logic of Velero. The ConfigMap used as the policy is supposed to be maintained by users. + + +### Improving Developer Experience + +As we continue to grow our community of contributors, we want to lower the barrier to entry for making contributions to the Velero project. +We’ve made huge improvements to the developer experience during this release cycle by introducing Tilt to the developer workflow. +Using Tilt enables developers to make changes to Velero and its plugins, and have those changes automatically built and deployed to your cluster. +This removes the need for any manual building or pushing of images, and provides a faster and much simpler workflow. +Our Tilt configuration also enables contributors to more easily debug the Velero process using Delve, which has integrations with many editors and IDEs. +If you would like to try it out, please see our [documentation](https://velero.io/docs/v1.11/tilt/). + +### Looking Forward + +We have more exciting additions and improvements to Velero earmarked for future releases. +For v1.12, we would like to have your input again [here](https://github.com/vmware-tanzu/velero/discussions/6217) +See our [1.12 RoadMap](https://github.com/vmware-tanzu/velero/wiki/1.12-Roadmap) for the complete list. + + +### Join the Community and Make Velero Better + +Velero is better because of our contributors and maintainers. +It is because of you that we can bring great software to the community. +Please join us during our online [community meetings every Tuesday](https://hackmd.io/Jq6F5zqZR7S80CeDWUklkA?view) and catch up with past meetings on YouTube on the [Velero Community Meetings playlist](https://www.youtube.com/watch?v=nc48ocI-6go&list=PL7bmigfV0EqQRysvqvqOtRNk4L5S7uqwM). + +You can always find the latest project information at [velero.io](https://velero.io). +Look for issues on GitHub marked ["Good first issue"](https://github.com/vmware-tanzu/velero/issues?q=is%3Aopen+is%3Aissue+label%3A%22Good+first+issue%22) or ["Help wanted"](https://github.com/vmware-tanzu/velero/issues?q=is%3Aopen+is%3Aissue+label%3A%22Help+wanted%22+) if you want to roll up your sleeves and write some code with us. + +For opportunities to help and be helped, visit our [Community Support Q&A on GitHub](https://github.com/vmware-tanzu/velero/discussions/categories/community-support-q-a). + +You can chat with us on [Kubernetes Slack in the #velero channel](https://kubernetes.slack.com/messages/C6VCGP4MT) and follow us on Twitter at [@projectvelero](https://twitter.com/projectvelero). + +Orlin Vasilev +Velero Community Lead + + +Photo by [Markus Spiske on Unsplash](https://unsplash.com/@markusspiske?utm_source=unsplash&utm_medium=referral&utm_content=creditCopyText) + diff --git a/site/static/img/posts/post-1.11.jpg b/site/static/img/posts/post-1.11.jpg new file mode 100644 index 000000000..211efadd2 Binary files /dev/null and b/site/static/img/posts/post-1.11.jpg differ diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index aaef02652..df90b7014 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -150,7 +150,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam } var snapshotCheckPoint SnapshotCheckPoint if useVolumeSnapshots { - snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, 2, deletionTest, backupName, KibishiiPodNameList) + snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, 2, deletionTest, backupName, KibishiiPVCNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslConfig, diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index 037c0a2b5..09afb23e8 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -51,7 +51,7 @@ func (b *TTL) Init() { b.testNS = "backup-ttl-test-" + UUIDgen.String() b.backupName = "backup-ttl-test-" + UUIDgen.String() b.restoreName = "restore-ttl-test-" + UUIDgen.String() - b.ttl = 20 * time.Minute + b.ttl = 10 * time.Minute } @@ -128,7 +128,7 @@ func TTLTest() { test.testNS, 2)).To(Succeed()) }) } - snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, 2, test.testNS, test.backupName, KibishiiPodNameList) + snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, 2, test.testNS, test.backupName, KibishiiPVCNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, diff --git a/test/e2e/basic/api-group/enable_api_group_versions.go b/test/e2e/basic/api-group/enable_api_group_versions.go index cc6290546..ba1d73f6e 100644 --- a/test/e2e/basic/api-group/enable_api_group_versions.go +++ b/test/e2e/basic/api-group/enable_api_group_versions.go @@ -100,7 +100,7 @@ func APIGropuVersionsTest() { DeleteBackups(context.Background(), *veleroCfg.ClientToInstallVelero) }) if veleroCfg.InstallVelero { - By("Uninstall Velero", func() { + By("Uninstall Velero in api group version case", func() { Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).NotTo(HaveOccurred()) }) } diff --git a/test/e2e/basic/namespace-mapping.go b/test/e2e/basic/namespace-mapping.go index 40e2767e5..8ea16d365 100644 --- a/test/e2e/basic/namespace-mapping.go +++ b/test/e2e/basic/namespace-mapping.go @@ -23,12 +23,16 @@ type NamespaceMapping struct { const NamespaceBaseName string = "ns-mp-" -var OneNamespaceMappingResticTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: NamespaceBaseName, NSIncluded: &[]string{NamespaceBaseName + "1"}, UseVolumeSnapshots: false}}) -var MultiNamespacesMappingResticTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: NamespaceBaseName, NSIncluded: &[]string{NamespaceBaseName + "2", NamespaceBaseName + "3"}, UseVolumeSnapshots: false}}) -var OneNamespaceMappingSnapshotTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: NamespaceBaseName, NSIncluded: &[]string{NamespaceBaseName + "4"}, UseVolumeSnapshots: true}}) -var MultiNamespacesMappingSnapshotTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: NamespaceBaseName, NSIncluded: &[]string{NamespaceBaseName + "5", NamespaceBaseName + "6"}, UseVolumeSnapshots: true}}) +var OneNamespaceMappingResticTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NamespacesTotal: 1, UseVolumeSnapshots: false}}) +var MultiNamespacesMappingResticTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NamespacesTotal: 2, UseVolumeSnapshots: false}}) +var OneNamespaceMappingSnapshotTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NamespacesTotal: 1, UseVolumeSnapshots: true}}) +var MultiNamespacesMappingSnapshotTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NamespacesTotal: 2, UseVolumeSnapshots: true}}) func (n *NamespaceMapping) Init() error { + n.TestCase.Init() + n.CaseBaseName = "ns-mp-" + n.UUIDgen + n.BackupName = "backup-" + n.CaseBaseName + n.RestoreName = "restore-" + n.CaseBaseName n.VeleroCfg = VeleroCfg n.Client = *n.VeleroCfg.ClientToInstallVelero n.VeleroCfg.UseVolumeSnapshots = n.UseVolumeSnapshots @@ -38,30 +42,25 @@ func (n *NamespaceMapping) Init() error { if n.UseVolumeSnapshots { backupType = "snapshot" } + var mappedNS string + var mappedNSList []string + n.NSIncluded = &[]string{} + for nsNum := 0; nsNum < n.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", n.CaseBaseName, nsNum) + *n.NSIncluded = append(*n.NSIncluded, createNSName) + mappedNS = mappedNS + createNSName + ":" + createNSName + "-mapped" + mappedNSList = append(mappedNSList, createNSName+"-mapped") + mappedNS = mappedNS + "," + } + mappedNS = strings.TrimRightFunc(mappedNS, func(r rune) bool { + return r == ',' + }) + n.TestMsg = &TestMSG{ Desc: fmt.Sprintf("Restore namespace %s with namespace mapping by %s test", *n.NSIncluded, backupType), FailedMSG: "Failed to restore with namespace mapping", Text: fmt.Sprintf("should restore namespace %s with namespace mapping by %s", *n.NSIncluded, backupType), } - return nil -} - -func (n *NamespaceMapping) StartRun() error { - var mappedNS string - var mappedNSList []string - - for index, ns := range *n.NSIncluded { - mappedNS = mappedNS + ns + ":" + ns + UUIDgen.String() - mappedNSList = append(mappedNSList, ns+UUIDgen.String()) - if index+1 != len(*n.NSIncluded) { - mappedNS = mappedNS + "," - } - n.BackupName = n.BackupName + ns - n.RestoreName = n.RestoreName + ns - } - n.BackupName = n.BackupName + UUIDgen.String() - n.RestoreName = n.RestoreName + UUIDgen.String() - n.MappedNamespaceList = mappedNSList fmt.Println(mappedNSList) n.BackupArgs = []string{ @@ -81,16 +80,16 @@ func (n *NamespaceMapping) StartRun() error { } return nil } + func (n *NamespaceMapping) CreateResources() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 60*time.Minute) - defer ctxCancel() + n.Ctx, n.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) for index, ns := range *n.NSIncluded { n.kibishiiData.Levels = len(*n.NSIncluded) + index By(fmt.Sprintf("Creating namespaces ...%s\n", ns), func() { - Expect(CreateNamespace(ctx, n.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns)) + Expect(CreateNamespace(n.Ctx, n.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns)) }) By("Deploy sample workload of Kibishii", func() { - Expect(KibishiiPrepareBeforeBackup(ctx, n.Client, VeleroCfg.CloudProvider, + Expect(KibishiiPrepareBeforeBackup(n.Ctx, n.Client, VeleroCfg.CloudProvider, ns, VeleroCfg.RegistryCredentialFile, VeleroCfg.Features, VeleroCfg.KibishiiDirectory, false, n.kibishiiData)).To(Succeed()) }) @@ -99,19 +98,33 @@ func (n *NamespaceMapping) CreateResources() error { } func (n *NamespaceMapping) Verify() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 60*time.Minute) - defer ctxCancel() for index, ns := range n.MappedNamespaceList { n.kibishiiData.Levels = len(*n.NSIncluded) + index By(fmt.Sprintf("Verify workload %s after restore ", ns), func() { Expect(KibishiiVerifyAfterRestore(n.Client, ns, - ctx, n.kibishiiData)).To(Succeed(), "Fail to verify workload after restore") + n.Ctx, n.kibishiiData)).To(Succeed(), "Fail to verify workload after restore") }) } for _, ns := range *n.NSIncluded { By(fmt.Sprintf("Verify namespace %s for backup is no longer exist after restore with namespace mapping", ns), func() { - Expect(NamespaceShouldNotExist(ctx, n.Client, ns)).To(Succeed()) + Expect(NamespaceShouldNotExist(n.Ctx, n.Client, ns)).To(Succeed()) }) } return nil } + +func (n *NamespaceMapping) Clean() error { + if !n.VeleroCfg.Debug { + if err := DeleteStorageClass(context.Background(), n.Client, "kibishii-storage-class"); err != nil { + return err + } + for _, ns := range n.MappedNamespaceList { + if err := DeleteNamespace(context.Background(), n.Client, ns, false); err != nil { + return err + } + } + + return n.GetTestCase().Clean() + } + return nil +} diff --git a/test/e2e/basic/nodeport.go b/test/e2e/basic/nodeport.go index 3faa518bc..440aab27d 100644 --- a/test/e2e/basic/nodeport.go +++ b/test/e2e/basic/nodeport.go @@ -3,6 +3,7 @@ package basic import ( "context" "fmt" + "strings" "time" . "github.com/onsi/ginkgo" @@ -20,40 +21,42 @@ import ( type NodePort struct { TestCase - replica int32 labels map[string]string - containers *[]v1.Container serviceName string - serviceSpec *v1.ServiceSpec + namespaceToCollision []string nodePort int32 - namespaceToCollision string - namespace string } const NodeportBaseName string = "nodeport-" -var NodePortTest func() = TestFunc(&NodePort{namespace: NodeportBaseName + "1", TestCase: TestCase{NSBaseName: NodeportBaseName}}) +var NodePortTest func() = TestFunc(&NodePort{}) func (n *NodePort) Init() error { + n.TestCase.Init() + n.CaseBaseName = NodeportBaseName + n.UUIDgen + n.BackupName = "backup-" + n.CaseBaseName + n.RestoreName = "restore-" + n.CaseBaseName + n.serviceName = "nginx-service-" + n.CaseBaseName n.VeleroCfg = VeleroCfg n.Client = *n.VeleroCfg.ClientToInstallVelero - n.NSBaseName = NodeportBaseName + n.NamespacesTotal = 1 n.TestMsg = &TestMSG{ - Desc: fmt.Sprintf("Nodeport preservation"), + Desc: "Nodeport preservation", FailedMSG: "Failed to restore with nodeport preservation", - Text: fmt.Sprintf("Nodeport can be preserved or omit during restore"), + Text: "Nodeport can be preserved or omit during restore", + } + + n.labels = map[string]string{"app": "nginx"} + n.NSIncluded = &[]string{} + for nsNum := 0; nsNum < n.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", n.CaseBaseName, nsNum) + n.namespaceToCollision = append(n.namespaceToCollision, createNSName+"tmp") + *n.NSIncluded = append(*n.NSIncluded, createNSName) } - n.BackupName = "backup-nodeport-" + UUIDgen.String() - n.RestoreName = "restore-" + UUIDgen.String() - n.serviceName = "nginx-service-" + UUIDgen.String() - n.labels = map[string]string{"app": "nginx"} - return nil -} -func (n *NodePort) StartRun() error { n.BackupArgs = []string{ "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", n.BackupName, - "--include-namespaces", n.namespace, "--wait", + "--include-namespaces", strings.Join(*n.NSIncluded, ","), "--wait", } n.RestoreArgs = []string{ "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", @@ -63,47 +66,45 @@ func (n *NodePort) StartRun() error { return nil } func (n *NodePort) CreateResources() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 60*time.Minute) - defer ctxCancel() + n.Ctx, n.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) - By(fmt.Sprintf("Creating service %s in namespaces %s ......\n", n.serviceName, n.namespace), func() { - Expect(CreateNamespace(ctx, n.Client, n.namespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", n.namespace)) - Expect(createServiceWithNodeport(ctx, n.Client, n.namespace, n.serviceName, n.labels, 0)).To(Succeed(), fmt.Sprintf("Failed to create service %s", n.serviceName)) - service, err := GetService(ctx, n.Client, n.namespace, n.serviceName) - Expect(err).To(Succeed()) - Expect(len(service.Spec.Ports)).To(Equal(1)) - n.nodePort = service.Spec.Ports[0].NodePort - _, err = GetAllService(ctx) - Expect(err).To(Succeed(), "fail to get service") - }) + for _, ns := range *n.NSIncluded { + By(fmt.Sprintf("Creating service %s in namespaces %s ......\n", n.serviceName, ns), func() { + Expect(CreateNamespace(n.Ctx, n.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns)) + Expect(createServiceWithNodeport(n.Ctx, n.Client, ns, n.serviceName, n.labels, 0)).To(Succeed(), fmt.Sprintf("Failed to create service %s", n.serviceName)) + service, err := GetService(n.Ctx, n.Client, ns, n.serviceName) + Expect(err).To(Succeed()) + Expect(len(service.Spec.Ports)).To(Equal(1)) + n.nodePort = service.Spec.Ports[0].NodePort + _, err = GetAllService(n.Ctx) + Expect(err).To(Succeed(), "fail to get service") + }) + } return nil } func (n *NodePort) Destroy() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 60*time.Minute) - defer ctxCancel() - By(fmt.Sprintf("Start to destroy namespace %s......", n.NSBaseName), func() { - Expect(CleanupNamespacesWithPoll(ctx, n.Client, NodeportBaseName)).To(Succeed(), - fmt.Sprintf("Failed to delete namespace %s", n.NSBaseName)) - Expect(WaitForServiceDelete(n.Client, n.namespace, n.serviceName, false)).To(Succeed(), "fail to delete service") - _, err := GetAllService(ctx) - Expect(err).To(Succeed(), "fail to get service") - }) + for i, ns := range *n.NSIncluded { + By(fmt.Sprintf("Start to destroy namespace %s......", n.CaseBaseName), func() { + Expect(CleanupNamespacesWithPoll(n.Ctx, n.Client, NodeportBaseName)).To(Succeed(), + fmt.Sprintf("Failed to delete namespace %s", n.CaseBaseName)) + Expect(WaitForServiceDelete(n.Client, ns, n.serviceName, false)).To(Succeed(), "fail to delete service") + _, err := GetAllService(n.Ctx) + Expect(err).To(Succeed(), "fail to get service") + }) - n.namespaceToCollision = NodeportBaseName + "tmp" - By(fmt.Sprintf("Creating a new service which has the same nodeport as backed up service has in a new namespaces for nodeport collision ...%s\n", n.namespaceToCollision), func() { - Expect(CreateNamespace(ctx, n.Client, n.namespaceToCollision)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", n.namespaceToCollision)) - Expect(createServiceWithNodeport(ctx, n.Client, n.namespaceToCollision, n.serviceName, n.labels, n.nodePort)).To(Succeed(), fmt.Sprintf("Failed to create service %s", n.serviceName)) - _, err := GetAllService(ctx) - Expect(err).To(Succeed(), "fail to get service") - }) + By(fmt.Sprintf("Creating a new service which has the same nodeport as backed up service has in a new namespaces for nodeport collision ...%s\n", n.namespaceToCollision[i]), func() { + Expect(CreateNamespace(n.Ctx, n.Client, n.namespaceToCollision[i])).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", n.namespaceToCollision[i])) + Expect(createServiceWithNodeport(n.Ctx, n.Client, n.namespaceToCollision[i], n.serviceName, n.labels, n.nodePort)).To(Succeed(), fmt.Sprintf("Failed to create service %s", n.serviceName)) + _, err := GetAllService(n.Ctx) + Expect(err).To(Succeed(), "fail to get service") + }) + } return nil } func (n *NodePort) Restore() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 60*time.Minute) - defer ctxCancel() index := 4 restoreName1 := n.RestoreName + "-1" restoreName2 := restoreName1 + "-1" @@ -112,7 +113,7 @@ func (n *NodePort) Restore() error { args = append(args[:index], append([]string{n.RestoreName}, args[index:]...)...) args = append(args, "--preserve-nodeports=true") By(fmt.Sprintf("Start to restore %s with nodeports preservation when port %d is already occupied by other service", n.RestoreName, n.nodePort), func() { - Expect(VeleroRestoreExec(ctx, n.VeleroCfg.VeleroCLI, + Expect(VeleroRestoreExec(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.RestoreName, args, velerov1api.RestorePhasePartiallyFailed)).To( Succeed(), @@ -127,44 +128,45 @@ func (n *NodePort) Restore() error { args = append(args[:index], append([]string{restoreName1}, args[index:]...)...) args = append(args, "--preserve-nodeports=false") By(fmt.Sprintf("Start to restore %s without nodeports preservation ......", restoreName1), func() { - Expect(VeleroRestoreExec(ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, + Expect(VeleroRestoreExec(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, restoreName1, args, velerov1api.RestorePhaseCompleted)).To(Succeed(), func() string { RunDebug(context.Background(), n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, "", restoreName1) return "Fail to restore workload" }) }) - By(fmt.Sprintf("Delete service %s by deleting namespace %s", n.serviceName, n.namespace), func() { - service, err := GetService(ctx, n.Client, n.namespace, n.serviceName) - Expect(err).To(Succeed()) - Expect(len(service.Spec.Ports)).To(Equal(1)) - fmt.Println(service.Spec.Ports) - Expect(DeleteNamespace(ctx, n.Client, n.namespace, true)).To(Succeed()) - }) - - By(fmt.Sprintf("Start to delete service %s in namespace %s ......", n.serviceName, n.namespaceToCollision), func() { - Expect(WaitForServiceDelete(n.Client, n.namespaceToCollision, n.serviceName, true)).To(Succeed(), "fail to delete service") - _, err := GetAllService(ctx) - Expect(err).To(Succeed(), "fail to get service") - }) - - args = n.RestoreArgs - args = append(args[:index], append([]string{restoreName2}, args[index:]...)...) - args = append(args, "--preserve-nodeports=true") - By(fmt.Sprintf("Start to restore %s with nodeports preservation ......", restoreName2), func() { - Expect(VeleroRestoreExec(ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, - restoreName2, args, velerov1api.RestorePhaseCompleted)).To(Succeed(), func() string { - RunDebug(context.Background(), n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, "", restoreName2) - return "Fail to restore workload" + for i, ns := range *n.NSIncluded { + By(fmt.Sprintf("Delete service %s by deleting namespace %s", n.serviceName, ns), func() { + service, err := GetService(n.Ctx, n.Client, ns, n.serviceName) + Expect(err).To(Succeed()) + Expect(len(service.Spec.Ports)).To(Equal(1)) + fmt.Println(service.Spec.Ports) + Expect(DeleteNamespace(n.Ctx, n.Client, ns, true)).To(Succeed()) }) - }) - By(fmt.Sprintf("Verify service %s was restore successfully with the origin nodeport.", n.namespace), func() { - service, err := GetService(ctx, n.Client, n.namespace, n.serviceName) - Expect(err).To(Succeed()) - Expect(len(service.Spec.Ports)).To(Equal(1)) - Expect(service.Spec.Ports[0].NodePort).To(Equal(n.nodePort)) - }) + By(fmt.Sprintf("Start to delete service %s in namespace %s ......", n.serviceName, n.namespaceToCollision[i]), func() { + Expect(WaitForServiceDelete(n.Client, n.namespaceToCollision[i], n.serviceName, true)).To(Succeed(), "fail to delete service") + _, err := GetAllService(n.Ctx) + Expect(err).To(Succeed(), "fail to get service") + }) + args = n.RestoreArgs + args = append(args[:index], append([]string{restoreName2}, args[index:]...)...) + args = append(args, "--preserve-nodeports=true") + By(fmt.Sprintf("Start to restore %s with nodeports preservation ......", restoreName2), func() { + Expect(VeleroRestoreExec(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, + restoreName2, args, velerov1api.RestorePhaseCompleted)).To(Succeed(), func() string { + RunDebug(context.Background(), n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, "", restoreName2) + return "Fail to restore workload" + }) + }) + + By(fmt.Sprintf("Verify service %s was restore successfully with the origin nodeport.", ns), func() { + service, err := GetService(n.Ctx, n.Client, ns, n.serviceName) + Expect(err).To(Succeed()) + Expect(len(service.Spec.Ports)).To(Equal(1)) + Expect(service.Spec.Ports[0].NodePort).To(Equal(n.nodePort)) + }) + } return nil } diff --git a/test/e2e/basic/pvc-selected-node-changing.go b/test/e2e/basic/pvc-selected-node-changing.go index 9e085addf..847188e34 100644 --- a/test/e2e/basic/pvc-selected-node-changing.go +++ b/test/e2e/basic/pvc-selected-node-changing.go @@ -31,24 +31,22 @@ type PVCSelectedNodeChanging struct { ann string } -const PSNCBaseName string = "psnc-" - -var PVCSelectedNodeChangingTest func() = TestFunc(&PVCSelectedNodeChanging{ - namespace: PSNCBaseName + "1", TestCase: TestCase{NSBaseName: PSNCBaseName}}) +var PVCSelectedNodeChangingTest func() = TestFunc(&PVCSelectedNodeChanging{}) func (p *PVCSelectedNodeChanging) Init() error { + p.TestCase.Init() + p.CaseBaseName = "psnc-" + p.UUIDgen + p.namespace = p.CaseBaseName + p.mappedNS = p.namespace + "-mapped" p.VeleroCfg = VeleroCfg p.Client = *p.VeleroCfg.ClientToInstallVelero - p.NSBaseName = PSNCBaseName - p.namespace = p.NSBaseName + UUIDgen.String() - p.mappedNS = p.namespace + "-mapped" p.TestMsg = &TestMSG{ Desc: "Changing PVC node selector", FailedMSG: "Failed to changing PVC node selector", Text: "Change node selectors of persistent volume claims during restores", } - p.BackupName = "backup-sc-" + UUIDgen.String() - p.RestoreName = "restore-" + UUIDgen.String() + p.BackupName = "backup-" + p.CaseBaseName + p.RestoreName = "restore-" + p.CaseBaseName p.labels = map[string]string{"velero.io/plugin-config": "", "velero.io/change-pvc-node-selector": "RestoreItemAction"} p.configmaptName = "change-pvc-node-selector-config" @@ -56,12 +54,6 @@ func (p *PVCSelectedNodeChanging) Init() error { p.podName = "pod-1" p.pvcName = "pvc-1" p.ann = "volume.kubernetes.io/selected-node" - return nil -} - -func (p *PVCSelectedNodeChanging) StartRun() error { - p.BackupName = p.BackupName + "backup-" + UUIDgen.String() - p.RestoreName = p.RestoreName + "restore-" + UUIDgen.String() p.BackupArgs = []string{ "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", p.BackupName, "--include-namespaces", p.namespace, @@ -73,14 +65,16 @@ func (p *PVCSelectedNodeChanging) StartRun() error { } return nil } + func (p *PVCSelectedNodeChanging) CreateResources() error { + p.Ctx, p.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) By(fmt.Sprintf("Create namespace %s", p.namespace), func() { - Expect(CreateNamespace(context.Background(), p.Client, p.namespace)).To(Succeed(), + Expect(CreateNamespace(p.Ctx, p.Client, p.namespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", p.namespace)) }) By(fmt.Sprintf("Create pod %s in namespace %s", p.podName, p.namespace), func() { - nodeNameList, err := GetWorkerNodes(context.Background()) + nodeNameList, err := GetWorkerNodes(p.Ctx) Expect(err).To(Succeed()) for _, nodeName := range nodeNameList { p.oldNodeName = nodeName @@ -88,14 +82,14 @@ func (p *PVCSelectedNodeChanging) CreateResources() error { pvcAnn := map[string]string{p.ann: nodeName} _, err := CreatePod(p.Client, p.namespace, p.podName, "default", p.pvcName, []string{p.volume}, pvcAnn, nil) Expect(err).To(Succeed()) - err = WaitForPods(context.Background(), p.Client, p.namespace, []string{p.podName}) + err = WaitForPods(p.Ctx, p.Client, p.namespace, []string{p.podName}) Expect(err).To(Succeed()) break } }) By("Prepare ConfigMap data", func() { - nodeNameList, err := GetWorkerNodes(context.Background()) + nodeNameList, err := GetWorkerNodes(p.Ctx) Expect(err).To(Succeed()) Expect(len(nodeNameList) > 2).To(Equal(true)) for _, nodeName := range nodeNameList { @@ -116,18 +110,16 @@ func (p *PVCSelectedNodeChanging) CreateResources() error { } func (p *PVCSelectedNodeChanging) Destroy() error { - By(fmt.Sprintf("Start to destroy namespace %s......", p.NSBaseName), func() { - Expect(CleanupNamespacesWithPoll(context.Background(), p.Client, p.NSBaseName)).To(Succeed(), - fmt.Sprintf("Failed to delete namespace %s", p.NSBaseName)) + By(fmt.Sprintf("Start to destroy namespace %s......", p.CaseBaseName), func() { + Expect(CleanupNamespacesWithPoll(p.Ctx, p.Client, p.CaseBaseName)).To(Succeed(), + fmt.Sprintf("Failed to delete namespace %s", p.CaseBaseName)) }) return nil } func (p *PVCSelectedNodeChanging) Restore() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 60*time.Minute) - defer ctxCancel() By(fmt.Sprintf("Start to restore %s .....", p.RestoreName), func() { - Expect(VeleroRestoreExec(ctx, p.VeleroCfg.VeleroCLI, + Expect(VeleroRestoreExec(p.Ctx, p.VeleroCfg.VeleroCLI, p.VeleroCfg.VeleroNamespace, p.RestoreName, p.RestoreArgs, velerov1api.RestorePhaseCompleted)).To( Succeed(), @@ -136,19 +128,29 @@ func (p *PVCSelectedNodeChanging) Restore() error { p.VeleroCfg.VeleroNamespace, "", p.RestoreName) return "Fail to restore workload" }) - err := WaitForPods(ctx, p.Client, p.mappedNS, []string{p.podName}) + err := WaitForPods(p.Ctx, p.Client, p.mappedNS, []string{p.podName}) Expect(err).To(Succeed()) }) return nil } func (p *PVCSelectedNodeChanging) Verify() error { By(fmt.Sprintf("PVC selected node should be %s", p.newNodeName), func() { - pvcNameList, err := GetPvcByPodName(context.Background(), p.mappedNS, p.pvcName) + pvcNameList, err := GetPvcByPodName(p.Ctx, p.mappedNS, p.pvcName) Expect(err).To(Succeed()) Expect(len(pvcNameList)).Should(Equal(1)) - pvc, err := GetPVC(context.Background(), p.Client, p.mappedNS, pvcNameList[0]) + pvc, err := GetPVC(p.Ctx, p.Client, p.mappedNS, pvcNameList[0]) Expect(err).To(Succeed()) Expect(pvc.Annotations[p.ann]).To(Equal(p.newNodeName)) }) return nil } + +func (p *PVCSelectedNodeChanging) Clean() error { + if !p.VeleroCfg.Debug { + p.TestCase.Clean() + By(fmt.Sprintf("Clean namespace with prefix %s after test", p.mappedNS), func() { + CleanupNamespaces(p.Ctx, p.Client, p.mappedNS) + }) + } + return nil +} diff --git a/test/e2e/basic/resources-check/namespaces.go b/test/e2e/basic/resources-check/namespaces.go index 3acf299a1..5b9d1460f 100644 --- a/test/e2e/basic/resources-check/namespaces.go +++ b/test/e2e/basic/resources-check/namespaces.go @@ -19,11 +19,9 @@ package basic import ( "context" "fmt" - "math/rand" "strings" "time" - "github.com/google/uuid" "github.com/pkg/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -40,11 +38,11 @@ type MultiNSBackup struct { } func (m *MultiNSBackup) Init() error { - rand.Seed(time.Now().UnixNano()) - UUIDgen, _ = uuid.NewRandom() - m.BackupName = "backup-" + UUIDgen.String() - m.RestoreName = "restore-" + UUIDgen.String() - m.NSBaseName = "nstest-" + UUIDgen.String() + m.TestCase.Init() + m.CaseBaseName = "nstest-" + m.UUIDgen + m.BackupName = "backup-" + m.CaseBaseName + m.RestoreName = "restore-" + m.CaseBaseName + m.VeleroCfg = VeleroCfg m.Client = *m.VeleroCfg.ClientToInstallVelero m.NSExcluded = &[]string{} @@ -64,10 +62,7 @@ func (m *MultiNSBackup) Init() error { FailedMSG: "Failed to successfully backup and restore multiple namespaces", } } - return nil -} -func (m *MultiNSBackup) StartRun() error { // Currently it's hard to build a large list of namespaces to include and wildcards do not work so instead // we will exclude all of the namespaces that existed prior to the test from the backup namespaces, err := m.Client.ClientGo.CoreV1().Namespaces().List(context.Background(), v1.ListOptions{}) @@ -93,15 +88,14 @@ func (m *MultiNSBackup) StartRun() error { } func (m *MultiNSBackup) CreateResources() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 60*time.Minute) - defer ctxCancel() + m.Ctx, m.CtxCancel = context.WithTimeout(context.Background(), m.TimeoutDuration) fmt.Printf("Creating namespaces ...\n") labels := map[string]string{ "ns-test": "true", } for nsNum := 0; nsNum < m.NamespacesTotal; nsNum++ { - createNSName := fmt.Sprintf("%s-%00000d", m.NSBaseName, nsNum) - if err := CreateNamespaceWithLabel(ctx, m.Client, createNSName, labels); err != nil { + createNSName := fmt.Sprintf("%s-%00000d", m.CaseBaseName, nsNum) + if err := CreateNamespaceWithLabel(m.Ctx, m.Client, createNSName, labels); err != nil { return errors.Wrapf(err, "Failed to create namespace %s", createNSName) } } @@ -109,12 +103,10 @@ func (m *MultiNSBackup) CreateResources() error { } func (m *MultiNSBackup) Verify() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), m.TimeoutDuration) - defer ctxCancel() // Verify that we got back all of the namespaces we created for nsNum := 0; nsNum < m.NamespacesTotal; nsNum++ { - checkNSName := fmt.Sprintf("%s-%00000d", m.NSBaseName, nsNum) - checkNS, err := GetNamespace(ctx, m.Client, checkNSName) + checkNSName := fmt.Sprintf("%s-%00000d", m.CaseBaseName, nsNum) + checkNS, err := GetNamespace(m.Ctx, m.Client, checkNSName) if err != nil { return errors.Wrapf(err, "Could not retrieve test namespace %s", checkNSName) } else if checkNS.Name != checkNSName { @@ -125,11 +117,9 @@ func (m *MultiNSBackup) Verify() error { } func (m *MultiNSBackup) Destroy() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 60*time.Minute) - defer ctxCancel() - err := CleanupNamespaces(ctx, m.Client, m.NSBaseName) + err := CleanupNamespaces(m.Ctx, m.Client, m.CaseBaseName) if err != nil { return errors.Wrap(err, "Could cleanup retrieve namespaces") } - return WaitAllSelectedNSDeleted(ctx, m.Client, "ns-test=true") + return WaitAllSelectedNSDeleted(m.Ctx, m.Client, "ns-test=true") } diff --git a/test/e2e/basic/resources-check/namespaces_annotation.go b/test/e2e/basic/resources-check/namespaces_annotation.go index 3d41e5a59..fada2b28c 100644 --- a/test/e2e/basic/resources-check/namespaces_annotation.go +++ b/test/e2e/basic/resources-check/namespaces_annotation.go @@ -19,11 +19,9 @@ package basic import ( "context" "fmt" - "math/rand" "strings" "time" - "github.com/google/uuid" "github.com/pkg/errors" . "github.com/vmware-tanzu/velero/test/e2e" @@ -36,17 +34,17 @@ type NSAnnotationCase struct { } func (n *NSAnnotationCase) Init() error { - rand.Seed(time.Now().UnixNano()) - UUIDgen, _ = uuid.NewRandom() - n.BackupName = "backup-namespace-annotations" + UUIDgen.String() - n.RestoreName = "restore-namespace-annotations" + UUIDgen.String() - n.NSBaseName = "namespace-annotations-" + UUIDgen.String() + n.TestCase.Init() + n.CaseBaseName = "namespace-annotations-" + n.UUIDgen + n.BackupName = "backup-" + n.CaseBaseName + n.RestoreName = "restore-" + n.CaseBaseName + n.NamespacesTotal = 1 n.NSIncluded = &[]string{} n.VeleroCfg = VeleroCfg n.Client = *n.VeleroCfg.ClientToInstallVelero for nsNum := 0; nsNum < n.NamespacesTotal; nsNum++ { - createNSName := fmt.Sprintf("%s-%00000d", n.NSBaseName, nsNum) + createNSName := fmt.Sprintf("%s-%00000d", n.CaseBaseName, nsNum) *n.NSIncluded = append(*n.NSIncluded, createNSName) } n.TestMsg = &TestMSG{ @@ -68,12 +66,11 @@ func (n *NSAnnotationCase) Init() error { } func (n *NSAnnotationCase) CreateResources() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 60*time.Minute) - defer ctxCancel() + n.Ctx, n.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) for nsNum := 0; nsNum < n.NamespacesTotal; nsNum++ { - createNSName := fmt.Sprintf("%s-%00000d", n.NSBaseName, nsNum) - createAnnotationName := fmt.Sprintf("annotation-%s-%00000d", n.NSBaseName, nsNum) - if err := CreateNamespaceWithAnnotation(ctx, n.Client, createNSName, map[string]string{"testAnnotation": createAnnotationName}); err != nil { + createNSName := fmt.Sprintf("%s-%00000d", n.CaseBaseName, nsNum) + createAnnotationName := fmt.Sprintf("annotation-%s-%00000d", n.CaseBaseName, nsNum) + if err := CreateNamespaceWithAnnotation(n.Ctx, n.Client, createNSName, map[string]string{"testAnnotation": createAnnotationName}); err != nil { return errors.Wrapf(err, "Failed to create namespace %s", createNSName) } } @@ -81,12 +78,10 @@ func (n *NSAnnotationCase) CreateResources() error { } func (n *NSAnnotationCase) Verify() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 60*time.Minute) - defer ctxCancel() for nsNum := 0; nsNum < n.NamespacesTotal; nsNum++ { - checkNSName := fmt.Sprintf("%s-%00000d", n.NSBaseName, nsNum) - checkAnnoName := fmt.Sprintf("annotation-%s-%00000d", n.NSBaseName, nsNum) - checkNS, err := GetNamespace(ctx, n.Client, checkNSName) + checkNSName := fmt.Sprintf("%s-%00000d", n.CaseBaseName, nsNum) + checkAnnoName := fmt.Sprintf("annotation-%s-%00000d", n.CaseBaseName, nsNum) + checkNS, err := GetNamespace(n.Ctx, n.Client, checkNSName) if err != nil { return errors.Wrapf(err, "Could not retrieve test namespace %s", checkNSName) diff --git a/test/e2e/basic/resources-check/rbac.go b/test/e2e/basic/resources-check/rbac.go index b4eaf7d37..deb3203a5 100644 --- a/test/e2e/basic/resources-check/rbac.go +++ b/test/e2e/basic/resources-check/rbac.go @@ -35,11 +35,9 @@ package basic import ( "context" "fmt" - "math/rand" "strings" "time" - "github.com/google/uuid" "github.com/pkg/errors" . "github.com/vmware-tanzu/velero/test/e2e" @@ -52,15 +50,14 @@ type RBACCase struct { } func (r *RBACCase) Init() error { - rand.Seed(time.Now().UnixNano()) - UUIDgen, _ = uuid.NewRandom() - r.BackupName = "backup-rbac" + UUIDgen.String() - r.RestoreName = "restore-rbac" + UUIDgen.String() - r.NSBaseName = "rabc-" + UUIDgen.String() + r.TestCase.Init() + r.CaseBaseName = "rabc-" + r.UUIDgen + r.BackupName = "backup-" + r.CaseBaseName + r.RestoreName = "restore-" + r.CaseBaseName r.NamespacesTotal = 1 r.NSIncluded = &[]string{} for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ { - createNSName := fmt.Sprintf("%s-%00000d", r.NSBaseName, nsNum) + createNSName := fmt.Sprintf("%s-%00000d", r.CaseBaseName, nsNum) *r.NSIncluded = append(*r.NSIncluded, createNSName) } r.TestMsg = &TestMSG{ @@ -84,22 +81,21 @@ func (r *RBACCase) Init() error { } func (r *RBACCase) CreateResources() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() + r.Ctx, r.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ { - createNSName := fmt.Sprintf("%s-%00000d", r.NSBaseName, nsNum) + createNSName := fmt.Sprintf("%s-%00000d", r.CaseBaseName, nsNum) fmt.Printf("Creating namespaces ...%s\n", createNSName) - if err := CreateNamespace(ctx, r.Client, createNSName); err != nil { + if err := CreateNamespace(r.Ctx, r.Client, createNSName); err != nil { return errors.Wrapf(err, "Failed to create namespace %s", createNSName) } - serviceAccountName := fmt.Sprintf("service-account-%s-%00000d", r.NSBaseName, nsNum) + serviceAccountName := fmt.Sprintf("service-account-%s-%00000d", r.CaseBaseName, nsNum) fmt.Printf("Creating service account ...%s\n", createNSName) - if err := CreateServiceAccount(ctx, r.Client, createNSName, serviceAccountName); err != nil { + if err := CreateServiceAccount(r.Ctx, r.Client, createNSName, serviceAccountName); err != nil { return errors.Wrapf(err, "Failed to create service account %s", serviceAccountName) } - clusterRoleName := fmt.Sprintf("clusterrole-%s-%00000d", r.NSBaseName, nsNum) - clusterRoleBindingName := fmt.Sprintf("clusterrolebinding-%s-%00000d", r.NSBaseName, nsNum) - if err := CreateRBACWithBindingSA(ctx, r.Client, createNSName, serviceAccountName, clusterRoleName, clusterRoleBindingName); err != nil { + clusterRoleName := fmt.Sprintf("clusterrole-%s-%00000d", r.CaseBaseName, nsNum) + clusterRoleBindingName := fmt.Sprintf("clusterrolebinding-%s-%00000d", r.CaseBaseName, nsNum) + if err := CreateRBACWithBindingSA(r.Ctx, r.Client, createNSName, serviceAccountName, clusterRoleName, clusterRoleBindingName); err != nil { return errors.Wrapf(err, "Failed to create cluster role %s with role binding %s", clusterRoleName, clusterRoleBindingName) } } @@ -107,14 +103,12 @@ func (r *RBACCase) CreateResources() error { } func (r *RBACCase) Verify() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ { - checkNSName := fmt.Sprintf("%s-%00000d", r.NSBaseName, nsNum) - checkServiceAccountName := fmt.Sprintf("service-account-%s-%00000d", r.NSBaseName, nsNum) - checkClusterRoleName := fmt.Sprintf("clusterrole-%s-%00000d", r.NSBaseName, nsNum) - checkClusterRoleBindingName := fmt.Sprintf("clusterrolebinding-%s-%00000d", r.NSBaseName, nsNum) - checkNS, err := GetNamespace(ctx, r.Client, checkNSName) + checkNSName := fmt.Sprintf("%s-%00000d", r.CaseBaseName, nsNum) + checkServiceAccountName := fmt.Sprintf("service-account-%s-%00000d", r.CaseBaseName, nsNum) + checkClusterRoleName := fmt.Sprintf("clusterrole-%s-%00000d", r.CaseBaseName, nsNum) + checkClusterRoleBindingName := fmt.Sprintf("clusterrolebinding-%s-%00000d", r.CaseBaseName, nsNum) + checkNS, err := GetNamespace(r.Ctx, r.Client, checkNSName) if err != nil { return errors.Wrapf(err, "Could not retrieve test namespace %s", checkNSName) } @@ -123,7 +117,7 @@ func (r *RBACCase) Verify() error { } //getting service account from the restore - checkSA, err := GetServiceAccount(ctx, r.Client, checkNSName, checkServiceAccountName) + checkSA, err := GetServiceAccount(r.Ctx, r.Client, checkNSName, checkServiceAccountName) if err != nil { return errors.Wrapf(err, "Could not retrieve test service account %s", checkSA) @@ -134,7 +128,7 @@ func (r *RBACCase) Verify() error { } //getting cluster role from the restore - checkClusterRole, err := GetClusterRole(ctx, r.Client, checkClusterRoleName) + checkClusterRole, err := GetClusterRole(r.Ctx, r.Client, checkClusterRoleName) if err != nil { return errors.Wrapf(err, "Could not retrieve test cluster role %s", checkClusterRole) @@ -145,7 +139,7 @@ func (r *RBACCase) Verify() error { } //getting cluster role binding from the restore - checkClusterRoleBinding, err := GetClusterRoleBinding(ctx, r.Client, checkClusterRoleBindingName) + checkClusterRoleBinding, err := GetClusterRoleBinding(r.Ctx, r.Client, checkClusterRoleBindingName) if err != nil { return errors.Wrapf(err, "Could not retrieve test cluster role binding %s", checkClusterRoleBinding) @@ -166,21 +160,19 @@ func (r *RBACCase) Verify() error { } func (r *RBACCase) Destroy() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() //cleanup clusterrole - err := CleanupClusterRole(ctx, r.Client, r.NSBaseName) + err := CleanupClusterRole(r.Ctx, r.Client, r.CaseBaseName) if err != nil { return errors.Wrap(err, "Could not cleanup clusterroles") } //cleanup cluster rolebinding - err = CleanupClusterRoleBinding(ctx, r.Client, r.NSBaseName) + err = CleanupClusterRoleBinding(r.Ctx, r.Client, r.CaseBaseName) if err != nil { return errors.Wrap(err, "Could not cleanup clusterrolebindings") } - err = CleanupNamespacesWithPoll(ctx, r.Client, r.NSBaseName) + err = CleanupNamespacesWithPoll(r.Ctx, r.Client, r.CaseBaseName) if err != nil { return errors.Wrap(err, "Could cleanup retrieve namespaces") } @@ -189,5 +181,8 @@ func (r *RBACCase) Destroy() error { } func (r *RBACCase) Clean() error { - return r.Destroy() + if !r.VeleroCfg.Debug { + return r.Destroy() + } + return nil } diff --git a/test/e2e/basic/resources-check/resources_check.go b/test/e2e/basic/resources-check/resources_check.go index 4fc32d568..9db5198bc 100644 --- a/test/e2e/basic/resources-check/resources_check.go +++ b/test/e2e/basic/resources-check/resources_check.go @@ -33,15 +33,14 @@ limitations under the License. package basic import ( - . "github.com/vmware-tanzu/velero/test/e2e" . "github.com/vmware-tanzu/velero/test/e2e/test" ) func GetResourcesCheckTestCases() []VeleroBackupRestoreTest { return []VeleroBackupRestoreTest{ - &NSAnnotationCase{TestCase{VeleroCfg: VeleroCfg}}, - &MultiNSBackup{IsScalTest: false, TestCase: TestCase{VeleroCfg: VeleroCfg}}, - &RBACCase{TestCase{VeleroCfg: VeleroCfg}}, + &NSAnnotationCase{}, + &MultiNSBackup{IsScalTest: false}, + &RBACCase{}, } } diff --git a/test/e2e/basic/storage-class-changing.go b/test/e2e/basic/storage-class-changing.go index d05ffcd80..a653ec549 100644 --- a/test/e2e/basic/storage-class-changing.go +++ b/test/e2e/basic/storage-class-changing.go @@ -31,15 +31,17 @@ type StorageClasssChanging struct { const SCCBaseName string = "scc-" -var StorageClasssChangingTest func() = TestFunc(&StorageClasssChanging{ - namespace: SCCBaseName + "1", TestCase: TestCase{NSBaseName: SCCBaseName}}) +var StorageClasssChangingTest func() = TestFunc(&StorageClasssChanging{}) func (s *StorageClasssChanging) Init() error { + s.TestCase.Init() + s.CaseBaseName = SCCBaseName + s.UUIDgen + s.namespace = s.CaseBaseName + s.BackupName = "backup-" + s.CaseBaseName + s.RestoreName = "restore-" + s.CaseBaseName + s.mappedNS = s.namespace + "-mapped" s.VeleroCfg = VeleroCfg s.Client = *s.VeleroCfg.ClientToInstallVelero - s.NSBaseName = SCCBaseName - s.namespace = s.NSBaseName + UUIDgen.String() - s.mappedNS = s.namespace + "-mapped" s.TestMsg = &TestMSG{ Desc: "Changing PV/PVC Storage Classes", FailedMSG: "Failed to changing PV/PVC Storage Classes", @@ -56,10 +58,7 @@ func (s *StorageClasssChanging) Init() error { s.configmaptName = "change-storage-class-config" s.volume = "volume-1" s.podName = "pod-1" - return nil -} -func (s *StorageClasssChanging) StartRun() error { s.BackupName = s.BackupName + "backup-" + UUIDgen.String() s.RestoreName = s.RestoreName + "restore-" + UUIDgen.String() s.BackupArgs = []string{ @@ -74,14 +73,13 @@ func (s *StorageClasssChanging) StartRun() error { return nil } func (s *StorageClasssChanging) CreateResources() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() + s.Ctx, s.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) By(fmt.Sprintf("Create a storage class %s", s.desStorageClass), func() { - Expect(InstallStorageClass(context.Background(), fmt.Sprintf("testdata/storage-class/%s.yaml", + Expect(InstallStorageClass(s.Ctx, fmt.Sprintf("testdata/storage-class/%s.yaml", s.VeleroCfg.CloudProvider))).To(Succeed()) }) By(fmt.Sprintf("Create namespace %s", s.namespace), func() { - Expect(CreateNamespace(ctx, s.Client, s.namespace)).To(Succeed(), + Expect(CreateNamespace(s.Ctx, s.Client, s.namespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", s.namespace)) }) @@ -97,30 +95,26 @@ func (s *StorageClasssChanging) CreateResources() error { } func (s *StorageClasssChanging) Destroy() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() By(fmt.Sprintf("Expect storage class of PV %s to be %s ", s.volume, s.srcStorageClass), func() { pvName, err := GetPVByPodName(s.Client, s.namespace, s.volume) Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV name by pod name %s", s.podName)) - pv, err := GetPersistentVolume(ctx, s.Client, s.namespace, pvName) + pv, err := GetPersistentVolume(s.Ctx, s.Client, s.namespace, pvName) Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV by pod name %s", s.podName)) fmt.Println(pv) Expect(pv.Spec.StorageClassName).To(Equal(s.srcStorageClass), fmt.Sprintf("PV storage %s is not as expected %s", pv.Spec.StorageClassName, s.srcStorageClass)) }) - By(fmt.Sprintf("Start to destroy namespace %s......", s.NSBaseName), func() { - Expect(CleanupNamespacesWithPoll(ctx, s.Client, s.NSBaseName)).To(Succeed(), - fmt.Sprintf("Failed to delete namespace %s", s.NSBaseName)) + By(fmt.Sprintf("Start to destroy namespace %s......", s.CaseBaseName), func() { + Expect(CleanupNamespacesWithPoll(s.Ctx, s.Client, s.CaseBaseName)).To(Succeed(), + fmt.Sprintf("Failed to delete namespace %s", s.CaseBaseName)) }) return nil } func (s *StorageClasssChanging) Restore() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() By(fmt.Sprintf("Start to restore %s .....", s.RestoreName), func() { - Expect(VeleroRestoreExec(ctx, s.VeleroCfg.VeleroCLI, + Expect(VeleroRestoreExec(s.Ctx, s.VeleroCfg.VeleroCLI, s.VeleroCfg.VeleroNamespace, s.RestoreName, s.RestoreArgs, velerov1api.RestorePhaseCompleted)).To( Succeed(), @@ -133,13 +127,11 @@ func (s *StorageClasssChanging) Restore() error { return nil } func (s *StorageClasssChanging) Verify() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() By(fmt.Sprintf("Expect storage class of PV %s to be %s ", s.volume, s.desStorageClass), func() { - time.Sleep(1 * time.Minute) + Expect(WaitForPods(s.Ctx, s.Client, s.mappedNS, []string{s.podName})).To(Succeed(), fmt.Sprintf("Failed to wait pod ready %s", s.podName)) pvName, err := GetPVByPodName(s.Client, s.mappedNS, s.volume) Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV name by pod name %s", s.podName)) - pv, err := GetPersistentVolume(ctx, s.Client, s.mappedNS, pvName) + pv, err := GetPersistentVolume(s.Ctx, s.Client, s.mappedNS, pvName) Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV by pod name %s", s.podName)) fmt.Println(pv) Expect(pv.Spec.StorageClassName).To(Equal(s.desStorageClass), @@ -147,3 +139,12 @@ func (s *StorageClasssChanging) Verify() error { }) return nil } + +func (s *StorageClasssChanging) Clean() error { + if !s.VeleroCfg.Debug { + DeleteConfigmap(s.Client.ClientGo, s.VeleroCfg.VeleroNamespace, s.configmaptName) + DeleteStorageClass(s.Ctx, s.Client, s.desStorageClass) + s.TestCase.Clean() + } + return nil +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 2dfd3820b..f4e1a4ddf 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -84,7 +84,7 @@ func init() { } -var _ = Describe("[APIGroup][Common] Velero tests with various CRD API group versions", APIGropuVersionsTest) +var _ = Describe("[APIGroup][APIVersion] Velero tests with various CRD API group versions", APIGropuVersionsTest) var _ = Describe("[APIGroup][APIExtensions] CRD of apiextentions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", APIExtensionsVersionsTest) // Test backup and restore of Kibishi using restic diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 1511cc307..c9b7571a6 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -208,7 +208,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) snapshotCheckPoint.NamespaceBackedUp = migrationNamespace By("Snapshot should be created in cloud object store", func() { snapshotCheckPoint, err := GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, 2, - migrationNamespace, backupName, KibishiiPodNameList) + migrationNamespace, backupName, KibishiiPVCNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, @@ -226,7 +226,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if (veleroCfg.CloudProvider == "aws" || veleroCfg.CloudProvider == "vsphere") && useVolumeSnapshots { + if veleroCfg.CloudProvider == "aws" && useVolumeSnapshots { fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") time.Sleep(5 * time.Minute) } diff --git a/test/e2e/pkg/client/factory.go b/test/e2e/pkg/client/factory.go index 3a0933af7..75bffae82 100644 --- a/test/e2e/pkg/client/factory.go +++ b/test/e2e/pkg/client/factory.go @@ -32,6 +32,7 @@ import ( "k8s.io/client-go/rest" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" clientset "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned" ) @@ -154,6 +155,7 @@ func (f *factory) KubebuilderClient() (kbclient.Client, error) { scheme := runtime.NewScheme() velerov1api.AddToScheme(scheme) + velerov2alpha1api.AddToScheme(scheme) k8scheme.AddToScheme(scheme) apiextv1beta1.AddToScheme(scheme) apiextv1.AddToScheme(scheme) diff --git a/test/e2e/pv-backup/pv-backup-filter.go b/test/e2e/pv-backup/pv-backup-filter.go index cb319a189..2ed6f2cf5 100644 --- a/test/e2e/pv-backup/pv-backup-filter.go +++ b/test/e2e/pv-backup/pv-backup-filter.go @@ -31,30 +31,22 @@ var OptInPVBackupTest func() = TestFunc(&PVBackupFiltering{annotation: OPT_IN_AN var OptOutPVBackupTest func() = TestFunc(&PVBackupFiltering{annotation: OPT_OUT_ANN, id: "opt-out"}) func (p *PVBackupFiltering) Init() error { + p.TestCase.Init() + p.CaseBaseName = "pv-filter-" + p.UUIDgen + p.BackupName = "backup-" + p.CaseBaseName + p.id + p.RestoreName = "restore-" + p.CaseBaseName + p.id p.VeleroCfg = VeleroCfg p.Client = *p.VeleroCfg.ClientToInstallVelero p.VeleroCfg.UseVolumeSnapshots = false p.VeleroCfg.UseNodeAgent = true - p.NSBaseName = "ns" - p.NSIncluded = &[]string{fmt.Sprintf("%s-%s-%d", p.NSBaseName, p.id, 1), fmt.Sprintf("%s-%s-%d", p.NSBaseName, p.id, 2)} + p.NSIncluded = &[]string{fmt.Sprintf("%s-%s-%d", p.CaseBaseName, p.id, 1), fmt.Sprintf("%s-%s-%d", p.CaseBaseName, p.id, 2)} p.TestMsg = &TestMSG{ Desc: "Backup PVs filtering by opt-in/opt-out annotation", FailedMSG: "Failed to PVs filtering by opt-in/opt-out annotation", Text: fmt.Sprintf("Should backup PVs in namespace %s according to annotation %s", *p.NSIncluded, p.annotation), } - return nil -} -func (p *PVBackupFiltering) StartRun() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() - err := InstallStorageClass(ctx, fmt.Sprintf("testdata/storage-class/%s.yaml", VeleroCfg.CloudProvider)) - if err != nil { - return err - } - p.BackupName = p.BackupName + "backup-" + p.id + "-" + UUIDgen.String() - p.RestoreName = p.RestoreName + "restore-" + p.id + "-" + UUIDgen.String() p.BackupArgs = []string{ "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", p.BackupName, "--include-namespaces", strings.Join(*p.NSIncluded, ","), @@ -72,12 +64,17 @@ func (p *PVBackupFiltering) StartRun() error { } return nil } + func (p *PVBackupFiltering) CreateResources() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() + p.Ctx, p.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) + err := InstallStorageClass(p.Ctx, fmt.Sprintf("testdata/storage-class/%s.yaml", VeleroCfg.CloudProvider)) + if err != nil { + return errors.Wrapf(err, "failed to install storage class for pv backup filtering test") + } + for _, ns := range *p.NSIncluded { By(fmt.Sprintf("Create namespaces %s for workload\n", ns), func() { - Expect(CreateNamespace(ctx, p.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns)) + Expect(CreateNamespace(p.Ctx, p.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns)) }) var pods []string By(fmt.Sprintf("Deploy a few pods with several PVs in namespace %s", ns), func() { @@ -105,7 +102,7 @@ func (p *PVBackupFiltering) CreateResources() error { p.annotation: volumesToAnnotation, } By(fmt.Sprintf("Add annotation to pod %s of namespace %s", pod.Name, ns), func() { - _, err := AddAnnotationToPod(ctx, p.Client, ns, pod.Name, ann) + _, err := AddAnnotationToPod(p.Ctx, p.Client, ns, pod.Name, ann) Expect(err).To(Succeed()) }) }) @@ -116,17 +113,17 @@ func (p *PVBackupFiltering) CreateResources() error { By(fmt.Sprintf("Waiting for all pods to start %s\n", p.podsList), func() { for index, ns := range *p.NSIncluded { By(fmt.Sprintf("Waiting for all pods to start %d in namespace %s", index, ns), func() { - WaitForPods(ctx, p.Client, ns, p.podsList[index]) + Expect(WaitForPods(p.Ctx, p.Client, ns, p.podsList[index])).To(Succeed()) }) } }) By(fmt.Sprintf("Populate all pods %s with file %s", p.podsList, FILE_NAME), func() { for index, ns := range *p.NSIncluded { By(fmt.Sprintf("Creating file in all pods to start %d in namespace %s", index, ns), func() { - WaitForPods(ctx, p.Client, ns, p.podsList[index]) + Expect(WaitForPods(p.Ctx, p.Client, ns, p.podsList[index])).To(Succeed()) for i, pod := range p.podsList[index] { for j := range p.volumesList[i] { - Expect(CreateFileToPod(ctx, ns, pod, pod, p.volumesList[i][j], + Expect(CreateFileToPod(p.Ctx, ns, pod, pod, p.volumesList[i][j], FILE_NAME, fileContent(ns, pod, p.volumesList[i][j]))).To(Succeed()) } } @@ -137,12 +134,10 @@ func (p *PVBackupFiltering) CreateResources() error { } func (p *PVBackupFiltering) Verify() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60) - defer ctxCancel() By(fmt.Sprintf("Waiting for all pods to start %s", p.podsList), func() { for index, ns := range *p.NSIncluded { By(fmt.Sprintf("Waiting for all pods to start %d in namespace %s", index, ns), func() { - WaitForPods(ctx, p.Client, ns, p.podsList[index]) + WaitForPods(p.Ctx, p.Client, ns, p.podsList[index]) }) } }) @@ -155,21 +150,21 @@ func (p *PVBackupFiltering) Verify() error { if j%2 == 0 { if p.annotation == OPT_IN_ANN { By(fmt.Sprintf("File should exists in PV %s of pod %s under namespace %s\n", p.volumesList[i][j], p.podsList[k][i], ns), func() { - Expect(fileExist(ctx, ns, p.podsList[k][i], p.volumesList[i][j])).To(Succeed(), "File not exist as expect") + Expect(fileExist(p.Ctx, ns, p.podsList[k][i], p.volumesList[i][j])).To(Succeed(), "File not exist as expect") }) } else { By(fmt.Sprintf("File should not exist in PV %s of pod %s under namespace %s\n", p.volumesList[i][j], p.podsList[k][i], ns), func() { - Expect(fileNotExist(ctx, ns, p.podsList[k][i], p.volumesList[i][j])).To(Succeed(), "File exists, not as expect") + Expect(fileNotExist(p.Ctx, ns, p.podsList[k][i], p.volumesList[i][j])).To(Succeed(), "File exists, not as expect") }) } } else { if p.annotation == OPT_OUT_ANN { By(fmt.Sprintf("File should exists in PV %s of pod %s under namespace %s\n", p.volumesList[i][j], p.podsList[k][i], ns), func() { - Expect(fileExist(ctx, ns, p.podsList[k][i], p.volumesList[i][j])).To(Succeed(), "File not exist as expect") + Expect(fileExist(p.Ctx, ns, p.podsList[k][i], p.volumesList[i][j])).To(Succeed(), "File not exist as expect") }) } else { By(fmt.Sprintf("File should not exist in PV %s of pod %s under namespace %s\n", p.volumesList[i][j], p.podsList[k][i], ns), func() { - Expect(fileNotExist(ctx, ns, p.podsList[k][i], p.volumesList[i][j])).To(Succeed(), "File exists, not as expect") + Expect(fileNotExist(p.Ctx, ns, p.podsList[k][i], p.volumesList[i][j])).To(Succeed(), "File exists, not as expect") }) } } diff --git a/test/e2e/resource-filtering/base.go b/test/e2e/resource-filtering/base.go index 1d8ce9026..256bf5167 100644 --- a/test/e2e/resource-filtering/base.go +++ b/test/e2e/resource-filtering/base.go @@ -19,10 +19,8 @@ package filtering import ( "context" "fmt" - "math/rand" "time" - "github.com/google/uuid" "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -43,8 +41,8 @@ var testInBackup = FilteringCase{IsTestInBackup: true} var testInRestore = FilteringCase{IsTestInBackup: false} func (f *FilteringCase) Init() error { - rand.Seed(time.Now().UnixNano()) - UUIDgen, _ = uuid.NewRandom() + f.TestCase.Init() + f.replica = int32(2) f.labels = map[string]string{"resourcefiltering": "true"} f.labelSelector = "resourcefiltering" @@ -66,26 +64,16 @@ func (f *FilteringCase) Init() error { } func (f *FilteringCase) CreateResources() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() + f.Ctx, f.CtxCancel = context.WithTimeout(context.Background(), 30*time.Minute) for nsNum := 0; nsNum < f.NamespacesTotal; nsNum++ { - namespace := fmt.Sprintf("%s-%00000d", f.NSBaseName, nsNum) + namespace := fmt.Sprintf("%s-%00000d", f.CaseBaseName, nsNum) fmt.Printf("Creating resources in namespace ...%s\n", namespace) - if err := CreateNamespace(ctx, f.Client, namespace); err != nil { + if err := CreateNamespace(f.Ctx, f.Client, namespace); err != nil { return errors.Wrapf(err, "Failed to create namespace %s", namespace) } - serviceAccountName := "default" - // wait until the service account is created before patch the image pull secret - if err := WaitUntilServiceAccountCreated(ctx, f.Client, namespace, serviceAccountName, 10*time.Minute); err != nil { - return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, namespace) - } - // add the image pull secret to avoid the image pull limit issue of Docker Hub - if err := PatchServiceAccountWithImagePullSecret(ctx, f.Client, namespace, serviceAccountName, VeleroCfg.RegistryCredentialFile); err != nil { - return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, namespace) - } //Create deployment fmt.Printf("Creating deployment in namespaces ...%s\n", namespace) - deployment := NewDeployment(f.NSBaseName, namespace, f.replica, f.labels, nil).Result() + deployment := NewDeployment(f.CaseBaseName, namespace, f.replica, f.labels, nil).Result() deployment, err := CreateDeployment(f.Client.ClientGo, namespace, deployment) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) @@ -95,7 +83,7 @@ func (f *FilteringCase) CreateResources() error { return errors.Wrap(err, fmt.Sprintf("failed to ensure job completion in namespace: %q", namespace)) } //Create Secret - secretName := f.NSBaseName + secretName := f.CaseBaseName fmt.Printf("Creating secret %s in namespaces ...%s\n", secretName, namespace) _, err = CreateSecret(f.Client.ClientGo, namespace, secretName, f.labels) if err != nil { @@ -106,7 +94,7 @@ func (f *FilteringCase) CreateResources() error { return errors.Wrap(err, fmt.Sprintf("failed to ensure secret completion in namespace: %q", namespace)) } //Create Configmap - configmaptName := f.NSBaseName + configmaptName := f.CaseBaseName fmt.Printf("Creating configmap %s in namespaces ...%s\n", configmaptName, namespace) _, err = CreateConfigMap(f.Client.ClientGo, namespace, configmaptName, f.labels, nil) if err != nil { @@ -121,13 +109,11 @@ func (f *FilteringCase) CreateResources() error { } func (f *FilteringCase) Verify() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() for nsNum := 0; nsNum < f.NamespacesTotal; nsNum++ { - namespace := fmt.Sprintf("%s-%00000d", f.NSBaseName, nsNum) + namespace := fmt.Sprintf("%s-%00000d", f.CaseBaseName, nsNum) fmt.Printf("Checking resources in namespaces ...%s\n", namespace) //Check namespace - checkNS, err := GetNamespace(ctx, f.Client, namespace) + checkNS, err := GetNamespace(f.Ctx, f.Client, namespace) if err != nil { return errors.Wrapf(err, "Could not retrieve test namespace %s", namespace) } @@ -135,7 +121,7 @@ func (f *FilteringCase) Verify() error { return errors.Errorf("Retrieved namespace for %s has name %s instead", namespace, checkNS.Name) } //Check deployment - _, err = GetDeployment(f.Client.ClientGo, namespace, f.NSBaseName) + _, err = GetDeployment(f.Client.ClientGo, namespace, f.CaseBaseName) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) } diff --git a/test/e2e/resource-filtering/exclude_label.go b/test/e2e/resource-filtering/exclude_label.go index b2202cdec..24db8288e 100644 --- a/test/e2e/resource-filtering/exclude_label.go +++ b/test/e2e/resource-filtering/exclude_label.go @@ -45,16 +45,17 @@ var ExcludeFromBackupTest func() = TestFunc(&ExcludeFromBackup{testInBackup}) func (e *ExcludeFromBackup) Init() error { e.FilteringCase.Init() - e.BackupName = "backup-exclude-from-backup-" + UUIDgen.String() - e.RestoreName = "restore-" + UUIDgen.String() - e.NSBaseName = "exclude-from-backup-" + UUIDgen.String() + e.CaseBaseName = "exclude-from-backup-" + e.UUIDgen + e.BackupName = "backup-" + e.CaseBaseName + e.RestoreName = "restore-" + e.CaseBaseName + e.TestMsg = &TestMSG{ Desc: "Backup with the label velero.io/exclude-from-backup=true are not included test", Text: "Should not backup resources with the label velero.io/exclude-from-backup=true", FailedMSG: "Failed to backup resources with the label velero.io/exclude-from-backup=true", } for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { - createNSName := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) + createNSName := fmt.Sprintf("%s-%00000d", e.CaseBaseName, nsNum) *e.NSIncluded = append(*e.NSIncluded, createNSName) } e.labels = map[string]string{ @@ -64,7 +65,7 @@ func (e *ExcludeFromBackup) Init() error { e.BackupArgs = []string{ "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", e.BackupName, - "--include-namespaces", e.NSBaseName, + "--include-namespaces", e.CaseBaseName, "--default-volumes-to-fs-backup", "--wait", } @@ -76,9 +77,8 @@ func (e *ExcludeFromBackup) Init() error { } func (e *ExcludeFromBackup) CreateResources() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() - namespace := e.NSBaseName + e.Ctx, e.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) + namespace := e.CaseBaseName // These 2 labels for resources to be included label1 := map[string]string{ "meaningless-label-resource-to-include": "true", @@ -87,21 +87,12 @@ func (e *ExcludeFromBackup) CreateResources() error { "velero.io/exclude-from-backup": "false", } fmt.Printf("Creating resources in namespace ...%s\n", namespace) - if err := CreateNamespace(ctx, e.Client, namespace); err != nil { + if err := CreateNamespace(e.Ctx, e.Client, namespace); err != nil { return errors.Wrapf(err, "Failed to create namespace %s", namespace) } - serviceAccountName := "default" - // wait until the service account is created before patch the image pull secret - if err := WaitUntilServiceAccountCreated(ctx, e.Client, namespace, serviceAccountName, 10*time.Minute); err != nil { - return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, namespace) - } - // add the image pull secret to avoid the image pull limit issue of Docker Hub - if err := PatchServiceAccountWithImagePullSecret(ctx, e.Client, namespace, serviceAccountName, VeleroCfg.RegistryCredentialFile); err != nil { - return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, namespace) - } //Create deployment: to be included fmt.Printf("Creating deployment in namespaces ...%s\n", namespace) - deployment := NewDeployment(e.NSBaseName, namespace, e.replica, label2, nil).Result() + deployment := NewDeployment(e.CaseBaseName, namespace, e.replica, label2, nil).Result() deployment, err := CreateDeployment(e.Client.ClientGo, namespace, deployment) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) @@ -111,7 +102,7 @@ func (e *ExcludeFromBackup) CreateResources() error { return errors.Wrap(err, fmt.Sprintf("failed to ensure job completion in namespace: %q", namespace)) } //Create Secret - secretName := e.NSBaseName + secretName := e.CaseBaseName fmt.Printf("Creating secret %s in namespaces ...%s\n", secretName, namespace) _, err = CreateSecret(e.Client.ClientGo, namespace, secretName, e.labels) if err != nil { @@ -122,11 +113,11 @@ func (e *ExcludeFromBackup) CreateResources() error { return errors.Wrap(err, fmt.Sprintf("failed to ensure secret completion in namespace: %q", namespace)) } By(fmt.Sprintf("Checking secret %s should exists in namespaces ...%s\n", secretName, namespace), func() { - _, err = GetSecret(e.Client.ClientGo, namespace, e.NSBaseName) + _, err = GetSecret(e.Client.ClientGo, namespace, e.CaseBaseName) Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) }) //Create Configmap: to be included - configmaptName := e.NSBaseName + configmaptName := e.CaseBaseName fmt.Printf("Creating configmap %s in namespaces ...%s\n", configmaptName, namespace) _, err = CreateConfigMap(e.Client.ClientGo, namespace, configmaptName, label1, nil) if err != nil { @@ -140,26 +131,24 @@ func (e *ExcludeFromBackup) CreateResources() error { } func (e *ExcludeFromBackup) Verify() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() - namespace := e.NSBaseName + namespace := e.CaseBaseName By(fmt.Sprintf("Checking resources in namespaces ...%s\n", namespace), func() { //Check namespace - checkNS, err := GetNamespace(ctx, e.Client, namespace) + checkNS, err := GetNamespace(e.Ctx, e.Client, namespace) Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("Could not retrieve test namespace %s", namespace)) Expect(checkNS.Name == namespace).To(Equal(true), fmt.Sprintf("Retrieved namespace for %s has name %s instead", namespace, checkNS.Name)) //Check deployment: should be included - _, err = GetDeployment(e.Client.ClientGo, namespace, e.NSBaseName) + _, err = GetDeployment(e.Client.ClientGo, namespace, e.CaseBaseName) Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) //Check secrets: secrets should not be included - _, err = GetSecret(e.Client.ClientGo, namespace, e.NSBaseName) + _, err = GetSecret(e.Client.ClientGo, namespace, e.CaseBaseName) Expect(err).Should(HaveOccurred(), fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) Expect(apierrors.IsNotFound(err)).To(Equal(true)) //Check configmap: should be included - _, err = GetConfigmap(e.Client.ClientGo, namespace, e.NSBaseName) + _, err = GetConfigmap(e.Client.ClientGo, namespace, e.CaseBaseName) Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("failed to list configmap in namespace: %q", namespace)) }) return nil diff --git a/test/e2e/resource-filtering/exclude_namespaces.go b/test/e2e/resource-filtering/exclude_namespaces.go index 05485e8e2..a1cd29b69 100644 --- a/test/e2e/resource-filtering/exclude_namespaces.go +++ b/test/e2e/resource-filtering/exclude_namespaces.go @@ -50,19 +50,20 @@ var RestoreWithExcludeNamespaces func() = TestFunc(&ExcludeNamespaces{FilteringC func (e *ExcludeNamespaces) Init() error { e.FilteringCase.Init() + e.CaseBaseName = "exclude-namespaces-" + e.UUIDgen e.namespacesExcluded = e.NamespacesTotal / 2 - e.NSBaseName = "exclude-namespaces-" + UUIDgen.String() + if e.IsTestInBackup { - e.BackupName = "backup-exclude-namespaces-" + UUIDgen.String() - e.RestoreName = "restore-" + UUIDgen.String() + e.BackupName = "backup-" + e.CaseBaseName + e.RestoreName = "restore-" + e.UUIDgen e.TestMsg = &TestMSG{ Desc: "Backup resources with exclude namespace test", FailedMSG: "Failed to backup and restore with namespace include", Text: fmt.Sprintf("should not backup %d namespaces of %d", e.namespacesExcluded, e.NamespacesTotal), } } else { - e.BackupName = "backup-" + UUIDgen.String() - e.RestoreName = "restore-exclude-namespaces-" + UUIDgen.String() + e.BackupName = "backup-" + e.UUIDgen + e.RestoreName = "restore-" + e.CaseBaseName e.TestMsg = &TestMSG{ Desc: "Restore resources with exclude namespace test", FailedMSG: "Failed to restore with namespace exclude", @@ -71,7 +72,7 @@ func (e *ExcludeNamespaces) Init() error { } e.nsExcluded = &[]string{} for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { - createNSName := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) + createNSName := fmt.Sprintf("%s-%00000d", e.CaseBaseName, nsNum) if nsNum < e.namespacesExcluded { *e.nsExcluded = append(*e.nsExcluded, createNSName) } else { @@ -109,12 +110,11 @@ func (e *ExcludeNamespaces) Init() error { } func (e *ExcludeNamespaces) CreateResources() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() + e.Ctx, e.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { - createNSName := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) + createNSName := fmt.Sprintf("%s-%00000d", e.CaseBaseName, nsNum) fmt.Printf("Creating namespaces ...%s\n", createNSName) - if err := CreateNamespace(ctx, e.Client, createNSName); err != nil { + if err := CreateNamespace(e.Ctx, e.Client, createNSName); err != nil { return errors.Wrapf(err, "Failed to create namespace %s", createNSName) } } @@ -122,12 +122,10 @@ func (e *ExcludeNamespaces) CreateResources() error { } func (e *ExcludeNamespaces) Verify() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() // Verify that we got back all of the namespaces we created for nsNum := 0; nsNum < e.namespacesExcluded; nsNum++ { - excludeNSName := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) - _, err := GetNamespace(ctx, e.Client, excludeNSName) + excludeNSName := fmt.Sprintf("%s-%00000d", e.CaseBaseName, nsNum) + _, err := GetNamespace(e.Ctx, e.Client, excludeNSName) if err == nil { return errors.Wrapf(err, "Resource filtering with exclude namespace but exclude namespace %s exist", excludeNSName) } @@ -138,8 +136,8 @@ func (e *ExcludeNamespaces) Verify() error { } for nsNum := e.namespacesExcluded; nsNum < e.NamespacesTotal; nsNum++ { - checkNSName := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) - checkNS, err := GetNamespace(ctx, e.Client, checkNSName) + checkNSName := fmt.Sprintf("%s-%00000d", e.CaseBaseName, nsNum) + checkNS, err := GetNamespace(e.Ctx, e.Client, checkNSName) if err != nil { return errors.Wrapf(err, "Could not retrieve test namespace %s", checkNSName) } diff --git a/test/e2e/resource-filtering/exclude_resources.go b/test/e2e/resource-filtering/exclude_resources.go index 844a4243e..d40574298 100644 --- a/test/e2e/resource-filtering/exclude_resources.go +++ b/test/e2e/resource-filtering/exclude_resources.go @@ -17,7 +17,6 @@ limitations under the License. package filtering import ( - "context" "fmt" "strings" @@ -49,9 +48,9 @@ var RestoreWithExcludeResources func() = TestFunc(&ExcludeResources{testInRestor func (e *ExcludeResources) Init() error { e.FilteringCase.Init() - e.NSBaseName = "exclude-resources-" + UUIDgen.String() + e.CaseBaseName = "exclude-resources-" + e.UUIDgen for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { - createNSName := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) + createNSName := fmt.Sprintf("%s-%00000d", e.CaseBaseName, nsNum) *e.NSIncluded = append(*e.NSIncluded, createNSName) } if e.IsTestInBackup { // testing case backup with exclude-resources option @@ -60,7 +59,7 @@ func (e *ExcludeResources) Init() error { Text: "Should not backup resources which is excluded others should be backup", FailedMSG: "Failed to backup with resource exclude", } - e.BackupName = "backup-exclude-resources-" + UUIDgen.String() + e.BackupName = "backup-" + e.CaseBaseName e.RestoreName = "restore-" + UUIDgen.String() e.BackupArgs = []string{ "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", e.BackupName, @@ -74,15 +73,13 @@ func (e *ExcludeResources) Init() error { "--from-backup", e.BackupName, "--wait", } } else { // testing case restore with exclude-resources option - e.BackupName = "backup-" + UUIDgen.String() - e.RestoreName = "restore-exclude-resources-" + UUIDgen.String() + e.BackupName = "backup-" + e.UUIDgen + e.RestoreName = "restore-" + e.CaseBaseName e.TestMsg = &TestMSG{ Desc: "Restore resources with resources included test", Text: "Should not restore resources which is excluded others should be backup", FailedMSG: "Failed to restore with resource exclude", } - e.BackupName = "backup-exclude-resources-" + UUIDgen.String() - e.RestoreName = "restore-exclude-resources-" + UUIDgen.String() e.BackupArgs = []string{ "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", e.BackupName, "--include-namespaces", strings.Join(*e.NSIncluded, ","), @@ -99,15 +96,15 @@ func (e *ExcludeResources) Init() error { func (e *ExcludeResources) Verify() error { for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { - namespace := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) + namespace := fmt.Sprintf("%s-%00000d", e.CaseBaseName, nsNum) fmt.Printf("Checking resources in namespaces ...%s\n", namespace) //Check deployment - _, err := GetDeployment(e.Client.ClientGo, namespace, e.NSBaseName) + _, err := GetDeployment(e.Client.ClientGo, namespace, e.CaseBaseName) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) } //Check secrets - secretsList, err := e.Client.ClientGo.CoreV1().Secrets(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: e.labelSelector}) + secretsList, err := e.Client.ClientGo.CoreV1().Secrets(namespace).List(e.Ctx, metav1.ListOptions{LabelSelector: e.labelSelector}) if err != nil { if apierrors.IsNotFound(err) { //resource should be excluded return nil @@ -118,7 +115,7 @@ func (e *ExcludeResources) Verify() error { } //Check configmap - configmapList, err := e.Client.ClientGo.CoreV1().ConfigMaps(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: e.labelSelector}) + configmapList, err := e.Client.ClientGo.CoreV1().ConfigMaps(namespace).List(e.Ctx, metav1.ListOptions{LabelSelector: e.labelSelector}) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to list configmap in namespace: %q", namespace)) } else if len(configmapList.Items) == 0 { diff --git a/test/e2e/resource-filtering/include_namespaces.go b/test/e2e/resource-filtering/include_namespaces.go index 003c99d99..789d9d787 100644 --- a/test/e2e/resource-filtering/include_namespaces.go +++ b/test/e2e/resource-filtering/include_namespaces.go @@ -51,11 +51,11 @@ var RestoreWithIncludeNamespaces func() = TestFunc(&IncludeNamespaces{FilteringC func (i *IncludeNamespaces) Init() error { i.FilteringCase.Init() + i.CaseBaseName = "include-namespaces-" + i.UUIDgen i.namespacesIncluded = i.NamespacesTotal / 2 i.allTestNamespaces = &[]string{} - i.NSBaseName = "include-namespaces-" + UUIDgen.String() for nsNum := 0; nsNum < i.NamespacesTotal; nsNum++ { - createNSName := fmt.Sprintf("%s-%00000d", i.NSBaseName, nsNum) + createNSName := fmt.Sprintf("%s-%00000d", i.CaseBaseName, nsNum) if nsNum < i.namespacesIncluded { *i.NSIncluded = append(*i.NSIncluded, createNSName) } @@ -63,8 +63,8 @@ func (i *IncludeNamespaces) Init() error { } if i.IsTestInBackup { - i.BackupName = "backup-include-namespaces-" + UUIDgen.String() - i.RestoreName = "restore-" + UUIDgen.String() + i.BackupName = "backup-" + i.CaseBaseName + i.RestoreName = "restore-" + i.UUIDgen i.TestMsg = &TestMSG{ Desc: "Backup resources with include namespace test", FailedMSG: "Failed to backup with namespace include", @@ -82,8 +82,8 @@ func (i *IncludeNamespaces) Init() error { } } else { - i.BackupName = "backup-" + UUIDgen.String() - i.RestoreName = "restore-include-namespaces-" + UUIDgen.String() + i.BackupName = "backup-" + i.UUIDgen + i.RestoreName = "restore-" + i.CaseBaseName i.TestMsg = &TestMSG{ Desc: "Restore resources with include namespace test", FailedMSG: "Failed to restore with namespace include", @@ -105,12 +105,11 @@ func (i *IncludeNamespaces) Init() error { } func (i *IncludeNamespaces) CreateResources() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() + i.Ctx, i.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) for nsNum := 0; nsNum < i.NamespacesTotal; nsNum++ { - createNSName := fmt.Sprintf("%s-%00000d", i.NSBaseName, nsNum) + createNSName := fmt.Sprintf("%s-%00000d", i.CaseBaseName, nsNum) fmt.Printf("Creating namespaces ...%s\n", createNSName) - if err := CreateNamespace(ctx, i.Client, createNSName); err != nil { + if err := CreateNamespace(i.Ctx, i.Client, createNSName); err != nil { return errors.Wrapf(err, "Failed to create namespace %s", createNSName) } } @@ -118,12 +117,10 @@ func (i *IncludeNamespaces) CreateResources() error { } func (i *IncludeNamespaces) Verify() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() // Verify that we got back all of the namespaces we created for nsNum := 0; nsNum < i.namespacesIncluded; nsNum++ { - checkNSName := fmt.Sprintf("%s-%00000d", i.NSBaseName, nsNum) - checkNS, err := GetNamespace(ctx, i.Client, checkNSName) + checkNSName := fmt.Sprintf("%s-%00000d", i.CaseBaseName, nsNum) + checkNS, err := GetNamespace(i.Ctx, i.Client, checkNSName) if err != nil { return errors.Wrapf(err, "Could not retrieve test namespace %s", checkNSName) } @@ -133,8 +130,8 @@ func (i *IncludeNamespaces) Verify() error { } for nsNum := i.namespacesIncluded; nsNum < i.NamespacesTotal; nsNum++ { - excludeNSName := fmt.Sprintf("%s-%00000d", i.NSBaseName, nsNum) - _, err := GetNamespace(ctx, i.Client, excludeNSName) + excludeNSName := fmt.Sprintf("%s-%00000d", i.CaseBaseName, nsNum) + _, err := GetNamespace(i.Ctx, i.Client, excludeNSName) if err == nil { return errors.Wrapf(err, "Resource filtering with include namespace but exclude namespace %s exist", excludeNSName) } diff --git a/test/e2e/resource-filtering/include_resources.go b/test/e2e/resource-filtering/include_resources.go index 268ee6fb8..28b166f0f 100644 --- a/test/e2e/resource-filtering/include_resources.go +++ b/test/e2e/resource-filtering/include_resources.go @@ -17,7 +17,6 @@ limitations under the License. package filtering import ( - "context" "fmt" "strings" @@ -47,9 +46,9 @@ var RestoreWithIncludeResources func() = TestFunc(&IncludeResources{testInRestor func (i *IncludeResources) Init() error { i.FilteringCase.Init() - i.NSBaseName = "include-resources-" + UUIDgen.String() + i.CaseBaseName = "include-resources-" + i.UUIDgen for nsNum := 0; nsNum < i.NamespacesTotal; nsNum++ { - createNSName := fmt.Sprintf("%s-%00000d", i.NSBaseName, nsNum) + createNSName := fmt.Sprintf("%s-%00000d", i.CaseBaseName, nsNum) *i.NSIncluded = append(*i.NSIncluded, createNSName) } if i.IsTestInBackup { // testing case backup with include-resources option @@ -58,8 +57,8 @@ func (i *IncludeResources) Init() error { Text: "Should backup resources which is included others should not be backup", FailedMSG: "Failed to backup with resource include", } - i.BackupName = "backup-include-resources-" + UUIDgen.String() - i.RestoreName = "restore-" + UUIDgen.String() + i.BackupName = "backup-" + i.CaseBaseName + i.RestoreName = "restore-" + i.UUIDgen i.BackupArgs = []string{ "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", i.BackupName, "--include-resources", "deployments,configmaps", @@ -76,8 +75,8 @@ func (i *IncludeResources) Init() error { Text: "Should restore resources which is included others should not be backup", FailedMSG: "Failed to restore with resource include", } - i.BackupName = "backup-" + UUIDgen.String() - i.RestoreName = "restore-include-resources-" + UUIDgen.String() + i.BackupName = "backup-" + i.UUIDgen + i.RestoreName = "restore-" + i.CaseBaseName i.BackupArgs = []string{ "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", i.BackupName, "--include-namespaces", strings.Join(*i.NSIncluded, ","), @@ -94,15 +93,15 @@ func (i *IncludeResources) Init() error { func (i *IncludeResources) Verify() error { for nsNum := 0; nsNum < i.NamespacesTotal; nsNum++ { - namespace := fmt.Sprintf("%s-%00000d", i.NSBaseName, nsNum) + namespace := fmt.Sprintf("%s-%00000d", i.CaseBaseName, nsNum) fmt.Printf("Checking resources in namespaces ...%s\n", namespace) //Check deployment - _, err := GetDeployment(i.Client.ClientGo, namespace, i.NSBaseName) + _, err := GetDeployment(i.Client.ClientGo, namespace, i.CaseBaseName) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) } //Check secrets - secretsList, err := i.Client.ClientGo.CoreV1().Secrets(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: i.labelSelector}) + secretsList, err := i.Client.ClientGo.CoreV1().Secrets(namespace).List(i.Ctx, metav1.ListOptions{LabelSelector: i.labelSelector}) if err != nil { if apierrors.IsNotFound(err) { //resource should be excluded return nil @@ -113,7 +112,7 @@ func (i *IncludeResources) Verify() error { } //Check configmap - configmapList, err := i.Client.ClientGo.CoreV1().ConfigMaps(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: i.labelSelector}) + configmapList, err := i.Client.ClientGo.CoreV1().ConfigMaps(namespace).List(i.Ctx, metav1.ListOptions{LabelSelector: i.labelSelector}) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to list configmap in namespace: %q", namespace)) } else if len(configmapList.Items) == 0 { diff --git a/test/e2e/resource-filtering/label_selector.go b/test/e2e/resource-filtering/label_selector.go index 90a4bdb8f..81dbffc8e 100644 --- a/test/e2e/resource-filtering/label_selector.go +++ b/test/e2e/resource-filtering/label_selector.go @@ -44,11 +44,12 @@ var BackupWithLabelSelector func() = TestFunc(&LabelSelector{testInBackup}) func (l *LabelSelector) Init() error { l.FilteringCase.Init() - l.BackupName = "backup-label-selector-" + UUIDgen.String() - l.RestoreName = "restore-" + UUIDgen.String() - l.NSBaseName = "backup-label-selector-" + UUIDgen.String() + l.CaseBaseName = "backup-label-selector-" + l.UUIDgen + l.BackupName = "backup-" + l.CaseBaseName + l.RestoreName = "restore-" + l.CaseBaseName + for nsNum := 0; nsNum < l.NamespacesTotal; nsNum++ { - createNSName := fmt.Sprintf("%s-%00000d", l.NSBaseName, nsNum) + createNSName := fmt.Sprintf("%s-%00000d", l.CaseBaseName, nsNum) *l.NSIncluded = append(*l.NSIncluded, createNSName) } l.TestMsg = &TestMSG{ @@ -75,10 +76,9 @@ func (l *LabelSelector) Init() error { } func (l *LabelSelector) CreateResources() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() + l.Ctx, l.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) for nsNum := 0; nsNum < l.NamespacesTotal; nsNum++ { - namespace := fmt.Sprintf("%s-%00000d", l.NSBaseName, nsNum) + namespace := fmt.Sprintf("%s-%00000d", l.CaseBaseName, nsNum) fmt.Printf("Creating resources in namespace ...%s\n", namespace) labels := l.labels if nsNum%2 == 0 { @@ -86,23 +86,13 @@ func (l *LabelSelector) CreateResources() error { "resourcefiltering": "false", } } - if err := CreateNamespaceWithLabel(ctx, l.Client, namespace, labels); err != nil { + if err := CreateNamespaceWithLabel(l.Ctx, l.Client, namespace, labels); err != nil { return errors.Wrapf(err, "Failed to create namespace %s", namespace) } - - serviceAccountName := "default" - // wait until the service account is created before patch the image pull secret - if err := WaitUntilServiceAccountCreated(ctx, l.Client, namespace, serviceAccountName, 10*time.Minute); err != nil { - return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, namespace) - } - // add the image pull secret to avoid the image pull limit issue of Docker Hub - if err := PatchServiceAccountWithImagePullSecret(ctx, l.Client, namespace, serviceAccountName, VeleroCfg.RegistryCredentialFile); err != nil { - return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, namespace) - } //Create deployment fmt.Printf("Creating deployment in namespaces ...%s\n", namespace) - deployment := NewDeployment(l.NSBaseName, namespace, l.replica, labels, nil).Result() + deployment := NewDeployment(l.CaseBaseName, namespace, l.replica, labels, nil).Result() deployment, err := CreateDeployment(l.Client.ClientGo, namespace, deployment) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) @@ -112,7 +102,7 @@ func (l *LabelSelector) CreateResources() error { return errors.Wrap(err, fmt.Sprintf("failed to ensure job completion in namespace: %q", namespace)) } //Create Secret - secretName := l.NSBaseName + secretName := l.CaseBaseName fmt.Printf("Creating secret %s in namespaces ...%s\n", secretName, namespace) _, err = CreateSecret(l.Client.ClientGo, namespace, secretName, l.labels) if err != nil { @@ -128,10 +118,10 @@ func (l *LabelSelector) CreateResources() error { func (l *LabelSelector) Verify() error { for nsNum := 0; nsNum < l.NamespacesTotal; nsNum++ { - namespace := fmt.Sprintf("%s-%00000d", l.NSBaseName, nsNum) + namespace := fmt.Sprintf("%s-%00000d", l.CaseBaseName, nsNum) fmt.Printf("Checking resources in namespaces ...%s\n", namespace) //Check deployment - _, err := GetDeployment(l.Client.ClientGo, namespace, l.NSBaseName) + _, err := GetDeployment(l.Client.ClientGo, namespace, l.CaseBaseName) if nsNum%2 == 1 { //include if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) @@ -148,7 +138,7 @@ func (l *LabelSelector) Verify() error { } //Check secrets - secretsList, err := l.Client.ClientGo.CoreV1().Secrets(namespace).List(context.TODO(), metav1.ListOptions{ + secretsList, err := l.Client.ClientGo.CoreV1().Secrets(namespace).List(l.Ctx, metav1.ListOptions{ LabelSelector: l.labelSelector, }) diff --git a/test/e2e/resourcepolicies/resource_policies.go b/test/e2e/resourcepolicies/resource_policies.go index 3d84de30d..c8cf5db34 100644 --- a/test/e2e/resourcepolicies/resource_policies.go +++ b/test/e2e/resourcepolicies/resource_policies.go @@ -20,7 +20,6 @@ import ( "context" "fmt" "io/ioutil" - "math/rand" "os" "strings" "time" @@ -28,7 +27,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/google/uuid" "github.com/pkg/errors" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -61,19 +59,19 @@ type ResourcePoliciesCase struct { var ResourcePoliciesTest func() = TestFunc(&ResourcePoliciesCase{}) func (r *ResourcePoliciesCase) Init() error { - rand.Seed(time.Now().UnixNano()) - UUIDgen, _ = uuid.NewRandom() + r.TestCase.Init() + r.CaseBaseName = "resource-policies-" + r.UUIDgen + r.cmName = "cm-" + r.CaseBaseName r.yamlConfig = yamlData r.VeleroCfg = VeleroCfg r.Client = *r.VeleroCfg.ClientToInstallVelero r.VeleroCfg.UseVolumeSnapshots = false r.VeleroCfg.UseNodeAgent = true r.NamespacesTotal = 3 - r.NSBaseName = "resource-policies-" + UUIDgen.String() - r.cmName = "cm-resource-policies-sc" + r.NSIncluded = &[]string{} for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ { - createNSName := fmt.Sprintf("%s-%00000d", r.NSBaseName, nsNum) + createNSName := fmt.Sprintf("%s-%00000d", r.CaseBaseName, nsNum) *r.NSIncluded = append(*r.NSIncluded, createNSName) } @@ -102,8 +100,7 @@ func (r *ResourcePoliciesCase) Init() error { } func (r *ResourcePoliciesCase) CreateResources() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() + r.Ctx, r.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) By(("Installing storage class..."), func() { Expect(r.installTestStorageClasses(fmt.Sprintf("testdata/storage-class/%s.yaml", VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install storage class") }) @@ -117,12 +114,12 @@ func (r *ResourcePoliciesCase) CreateResources() error { }) for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ { - namespace := fmt.Sprintf("%s-%00000d", r.NSBaseName, nsNum) + namespace := fmt.Sprintf("%s-%00000d", r.CaseBaseName, nsNum) By(fmt.Sprintf("Create namespaces %s for workload\n", namespace), func() { - Expect(CreateNamespace(ctx, r.Client, namespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", namespace)) + Expect(CreateNamespace(r.Ctx, r.Client, namespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", namespace)) }) - volName := fmt.Sprintf("vol-%s-%00000d", r.NSBaseName, nsNum) + volName := fmt.Sprintf("vol-%s-%00000d", r.CaseBaseName, nsNum) volList := PrepareVolumeList([]string{volName}) // Create PVC @@ -145,23 +142,21 @@ func (r *ResourcePoliciesCase) CreateResources() error { } func (r *ResourcePoliciesCase) Verify() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() for i, ns := range *r.NSIncluded { By(fmt.Sprintf("Verify pod data in namespace %s", ns), func() { - By(fmt.Sprintf("Waiting for deployment %s in namespace %s ready", r.NSBaseName, ns), func() { - Expect(WaitForReadyDeployment(r.Client.ClientGo, ns, r.NSBaseName)).To(Succeed(), fmt.Sprintf("Failed to waiting for deployment %s in namespace %s ready", r.NSBaseName, ns)) + By(fmt.Sprintf("Waiting for deployment %s in namespace %s ready", r.CaseBaseName, ns), func() { + Expect(WaitForReadyDeployment(r.Client.ClientGo, ns, r.CaseBaseName)).To(Succeed(), fmt.Sprintf("Failed to waiting for deployment %s in namespace %s ready", r.CaseBaseName, ns)) }) - podList, err := ListPods(ctx, r.Client, ns) + podList, err := ListPods(r.Ctx, r.Client, ns) Expect(err).To(Succeed(), fmt.Sprintf("failed to list pods in namespace: %q with error %v", ns, err)) - volName := fmt.Sprintf("vol-%s-%00000d", r.NSBaseName, i) + volName := fmt.Sprintf("vol-%s-%00000d", r.CaseBaseName, i) for _, pod := range podList.Items { for _, vol := range pod.Spec.Volumes { if vol.Name != volName { continue } - content, err := ReadFileFromPodVolume(ctx, ns, pod.Name, "container-busybox", vol.Name, FileName) + content, err := ReadFileFromPodVolume(r.Ctx, ns, pod.Name, "container-busybox", vol.Name, FileName) if i%2 == 0 { Expect(err).To(HaveOccurred(), "Expected file not found") // File should not exist } else { @@ -183,15 +178,18 @@ func (r *ResourcePoliciesCase) Verify() error { } func (r *ResourcePoliciesCase) Clean() error { - if err := r.deleteTestStorageClassList([]string{"e2e-storage-class", "e2e-storage-class-2"}); err != nil { - return err - } + if !r.VeleroCfg.Debug { + if err := r.deleteTestStorageClassList([]string{"e2e-storage-class", "e2e-storage-class-2"}); err != nil { + return err + } - if err := DeleteConfigmap(r.Client.ClientGo, r.VeleroCfg.VeleroNamespace, r.cmName); err != nil { - return err - } + if err := DeleteConfigmap(r.Client.ClientGo, r.VeleroCfg.VeleroNamespace, r.cmName); err != nil { + return err + } - return r.GetTestCase().Clean() + return r.GetTestCase().Clean() + } + return nil } func (r *ResourcePoliciesCase) createPVC(index int, namespace string, volList []*v1.Volume) error { @@ -217,7 +215,7 @@ func (r *ResourcePoliciesCase) createPVC(index int, namespace string, volList [] } func (r *ResourcePoliciesCase) createDeploymentWithVolume(namespace string, volList []*v1.Volume) error { - deployment := NewDeployment(r.NSBaseName, namespace, 1, map[string]string{"resource-policies": "resource-policies"}, nil).WithVolume(volList).Result() + deployment := NewDeployment(r.CaseBaseName, namespace, 1, map[string]string{"resource-policies": "resource-policies"}, nil).WithVolume(volList).Result() deployment, err := CreateDeployment(r.Client.ClientGo, namespace, deployment) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to create deloyment %s the namespace %q", deployment.Name, namespace)) @@ -230,9 +228,7 @@ func (r *ResourcePoliciesCase) createDeploymentWithVolume(namespace string, volL } func (r *ResourcePoliciesCase) writeDataIntoPods(namespace, volName string) error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() - podList, err := ListPods(ctx, r.Client, namespace) + podList, err := ListPods(r.Ctx, r.Client, namespace) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to list pods in namespace: %q with error %v", namespace, err)) } @@ -241,7 +237,7 @@ func (r *ResourcePoliciesCase) writeDataIntoPods(namespace, volName string) erro if vol.Name != volName { continue } - err := CreateFileToPod(ctx, namespace, pod.Name, "container-busybox", vol.Name, FileName, fmt.Sprintf("ns-%s pod-%s volume-%s", namespace, pod.Name, vol.Name)) + err := CreateFileToPod(r.Ctx, namespace, pod.Name, "container-busybox", vol.Name, FileName, fmt.Sprintf("ns-%s pod-%s volume-%s", namespace, pod.Name, vol.Name)) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to create file into pod %s in namespace: %q", pod.Name, namespace)) } @@ -251,10 +247,8 @@ func (r *ResourcePoliciesCase) writeDataIntoPods(namespace, volName string) erro } func (r *ResourcePoliciesCase) deleteTestStorageClassList(scList []string) error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() for _, v := range scList { - if err := DeleteStorageClass(ctx, r.Client, v); err != nil { + if err := DeleteStorageClass(r.Ctx, r.Client, v); err != nil { return err } } @@ -262,9 +256,7 @@ func (r *ResourcePoliciesCase) deleteTestStorageClassList(scList []string) error } func (r *ResourcePoliciesCase) installTestStorageClasses(path string) error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() - err := InstallStorageClass(ctx, path) + err := InstallStorageClass(r.Ctx, path) if err != nil { return err } @@ -285,5 +277,5 @@ func (r *ResourcePoliciesCase) installTestStorageClasses(path string) error { if _, err := tmpFile.WriteString(newContent); err != nil { return errors.Wrapf(err, "failed to write content into temp file %s when install storage class", tmpFile.Name()) } - return InstallStorageClass(ctx, tmpFile.Name()) + return InstallStorageClass(r.Ctx, tmpFile.Name()) } diff --git a/test/e2e/scale/multiple_namespaces.go b/test/e2e/scale/multiple_namespaces.go index d17f37f6b..d63880c6b 100644 --- a/test/e2e/scale/multiple_namespaces.go +++ b/test/e2e/scale/multiple_namespaces.go @@ -17,10 +17,8 @@ limitations under the License. package scale import ( - "time" - basic "github.com/vmware-tanzu/velero/test/e2e/basic/resources-check" . "github.com/vmware-tanzu/velero/test/e2e/test" ) -var MultiNSBackupRestore func() = TestFunc(&basic.MultiNSBackup{IsScalTest: true, TestCase: TestCase{Timeout: time.Hour}}) +var MultiNSBackupRestore func() = TestFunc(&basic.MultiNSBackup{IsScalTest: true}) diff --git a/test/e2e/schedule/ordered_resources.go b/test/e2e/schedule/ordered_resources.go index 8be393c17..3bc4c7bed 100644 --- a/test/e2e/schedule/ordered_resources.go +++ b/test/e2e/schedule/ordered_resources.go @@ -19,13 +19,10 @@ limitations under the License. //the ordered resources test related to https://github.com/vmware-tanzu/velero/issues/4561 import ( "context" - "flag" "fmt" - "math/rand" "strings" "time" - "github.com/google/uuid" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/pkg/errors" @@ -39,6 +36,8 @@ import ( . "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) +var ScheduleOrderedResources func() = TestFunc(&OrderedResources{}) + type OrderedResources struct { Namespace string ScheduleName string @@ -47,87 +46,23 @@ type OrderedResources struct { TestCase } -func ScheduleOrderedResources() { - veleroCfg := VeleroCfg - BeforeEach(func() { - flag.Parse() - if veleroCfg.InstallVelero { - veleroCfg.UseVolumeSnapshots = false - Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) - } - }) - - AfterEach(func() { - if veleroCfg.InstallVelero && !veleroCfg.Debug { - Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed()) - } - }) - - It("Create a schedule to backup resources in a specific order should be successful", func() { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() - test := &OrderedResources{} - test.VeleroCfg = VeleroCfg - err := test.Init() - Expect(err).To(Succeed(), err) - defer func() { - Expect(DeleteNamespace(ctx, test.Client, test.Namespace, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.Namespace)) - err = VeleroScheduleDelete(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, test.ScheduleName) - Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete schedule with err %v", err)) - err = test.DeleteBackups() - Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete backups with err %v", err)) - }() - - By(fmt.Sprintf("Prepare workload as target to backup in base namespace %s", test.Namespace), func() { - err = test.CreateResources() - Expect(err).To(Succeed(), fmt.Sprintf("Failed to create resources to backup with err %v", err)) - }) - - By(fmt.Sprintf("Create schedule the workload in %s namespace", test.Namespace), func() { - err = VeleroScheduleCreate(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, test.ScheduleName, test.ScheduleArgs) - Expect(err).To(Succeed(), fmt.Sprintf("Failed to create schedule %s with err %v", test.ScheduleName, err)) - }) - - By(fmt.Sprintf("Checking resource order in %s schedule cr", test.ScheduleName), func() { - err = CheckScheduleWithResourceOrder(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, test.ScheduleName, test.OrderMap) - Expect(err).To(Succeed(), fmt.Sprintf("Failed to check schedule %s with err %v", test.ScheduleName, err)) - }) - - By("Checking resource order in backup cr", func() { - backupList := new(velerov1api.BackupList) - err = waitutil.PollImmediate(10*time.Second, time.Minute*5, func() (bool, error) { - if err = test.Client.Kubebuilder.List(ctx, backupList, &kbclient.ListOptions{Namespace: veleroCfg.VeleroNamespace}); err != nil { - return false, fmt.Errorf("failed to list backup object in %s namespace with err %v", veleroCfg.VeleroNamespace, err) - } - - for _, backup := range backupList.Items { - if err = CheckBackupWithResourceOrder(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, backup.Name, test.OrderMap); err == nil { - return true, nil - } - } - fmt.Printf("still finding backup created by schedule %s ...\n", test.ScheduleName) - return false, nil - }) - Expect(err).To(Succeed(), fmt.Sprintf("Failed to check schedule %s created backup with err %v", test.ScheduleName, err)) - }) - - }) -} - func (o *OrderedResources) Init() error { - rand.Seed(time.Now().UnixNano()) - UUIDgen, _ = uuid.NewRandom() + o.TestCase.Init() + o.CaseBaseName = "ordered-resources-" + o.UUIDgen + o.ScheduleName = "schedule-" + o.CaseBaseName + o.Namespace = o.CaseBaseName + "-" + o.UUIDgen o.VeleroCfg = VeleroCfg o.Client = *o.VeleroCfg.ClientToInstallVelero - o.ScheduleName = "schedule-ordered-resources-" + UUIDgen.String() - o.NSBaseName = "schedule-ordered-resources" - o.Namespace = o.NSBaseName + "-" + UUIDgen.String() o.OrderMap = map[string]string{ - "deployments": fmt.Sprintf("deploy-%s", o.NSBaseName), - "secrets": fmt.Sprintf("secret-%s", o.NSBaseName), - "configmaps": fmt.Sprintf("configmap-%s", o.NSBaseName), + "deployments": fmt.Sprintf("deploy-%s", o.CaseBaseName), + "secrets": fmt.Sprintf("secret-%s", o.CaseBaseName), + "configmaps": fmt.Sprintf("configmap-%s", o.CaseBaseName), + } + o.TestMsg = &TestMSG{ + Desc: "Create a schedule to backup resources in a specific order should be successful", + FailedMSG: "Failed to verify schedule backup resources in a specific order", + Text: "Create a schedule to backup resources in a specific order should be successful", } - o.ScheduleArgs = []string{"--schedule", "@every 1m", "--include-namespaces", o.Namespace, "--default-volumes-to-fs-backup", "--ordered-resources"} var orderStr string @@ -140,27 +75,16 @@ func (o *OrderedResources) Init() error { } func (o *OrderedResources) CreateResources() error { - veleroCfg := o.VeleroCfg - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() + o.Ctx, o.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) label := map[string]string{ "orderedresources": "true", } fmt.Printf("Creating resources in %s namespace ...\n", o.Namespace) - if err := CreateNamespace(ctx, o.Client, o.Namespace); err != nil { + if err := CreateNamespace(o.Ctx, o.Client, o.Namespace); err != nil { return errors.Wrapf(err, "failed to create namespace %s", o.Namespace) } - serviceAccountName := "default" - // wait until the service account is created before patch the image pull secret - if err := WaitUntilServiceAccountCreated(ctx, o.Client, o.Namespace, serviceAccountName, 10*time.Minute); err != nil { - return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, o.Namespace) - } - // add the image pull secret to avoid the image pull limit issue of Docker Hub - if err := PatchServiceAccountWithImagePullSecret(ctx, o.Client, o.Namespace, serviceAccountName, veleroCfg.RegistryCredentialFile); err != nil { - return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, o.Namespace) - } //Create deployment - deploymentName := fmt.Sprintf("deploy-%s", o.NSBaseName) + deploymentName := fmt.Sprintf("deploy-%s", o.CaseBaseName) fmt.Printf("Creating deployment %s in %s namespaces ...\n", deploymentName, o.Namespace) deployment := NewDeployment(deploymentName, o.Namespace, 1, label, nil).Result() deployment, err := CreateDeployment(o.Client.ClientGo, o.Namespace, deployment) @@ -172,7 +96,7 @@ func (o *OrderedResources) CreateResources() error { return errors.Wrap(err, fmt.Sprintf("failed to ensure job completion in namespace: %q", o.Namespace)) } //Create Secret - secretName := fmt.Sprintf("secret-%s", o.NSBaseName) + secretName := fmt.Sprintf("secret-%s", o.CaseBaseName) fmt.Printf("Creating secret %s in %s namespaces ...\n", secretName, o.Namespace) _, err = CreateSecret(o.Client.ClientGo, o.Namespace, secretName, label) if err != nil { @@ -183,7 +107,7 @@ func (o *OrderedResources) CreateResources() error { return errors.Wrap(err, fmt.Sprintf("failed to ensure secret completion in namespace: %q", o.Namespace)) } //Create Configmap - configmapName := fmt.Sprintf("configmap-%s", o.NSBaseName) + configmapName := fmt.Sprintf("configmap-%s", o.CaseBaseName) fmt.Printf("Creating configmap %s in %s namespaces ...\n", configmapName, o.Namespace) _, err = CreateConfigMap(o.Client.ClientGo, o.Namespace, configmapName, label, nil) if err != nil { @@ -196,16 +120,55 @@ func (o *OrderedResources) CreateResources() error { return nil } +func (o *OrderedResources) Backup() error { + By(fmt.Sprintf("Create schedule the workload in %s namespace", o.Namespace), func() { + err := VeleroScheduleCreate(o.Ctx, o.VeleroCfg.VeleroCLI, o.VeleroCfg.VeleroNamespace, o.ScheduleName, o.ScheduleArgs) + Expect(err).To(Succeed(), fmt.Sprintf("Failed to create schedule %s with err %v", o.ScheduleName, err)) + }) + return nil +} + +func (o *OrderedResources) Verify() error { + By(fmt.Sprintf("Checking resource order in %s schedule cr", o.ScheduleName), func() { + err := CheckScheduleWithResourceOrder(o.Ctx, o.VeleroCfg.VeleroCLI, o.VeleroCfg.VeleroNamespace, o.ScheduleName, o.OrderMap) + Expect(err).To(Succeed(), fmt.Sprintf("Failed to check schedule %s with err %v", o.ScheduleName, err)) + }) + + By("Checking resource order in backup cr", func() { + backupList := new(velerov1api.BackupList) + err := waitutil.PollImmediate(10*time.Second, time.Minute*5, func() (bool, error) { + if err := o.Client.Kubebuilder.List(o.Ctx, backupList, &kbclient.ListOptions{Namespace: o.VeleroCfg.VeleroNamespace}); err != nil { + return false, fmt.Errorf("failed to list backup object in %s namespace with err %v", o.VeleroCfg.VeleroNamespace, err) + } + + for _, backup := range backupList.Items { + if err := CheckBackupWithResourceOrder(o.Ctx, o.VeleroCfg.VeleroCLI, o.VeleroCfg.VeleroNamespace, backup.Name, o.OrderMap); err == nil { + return true, nil + } + } + fmt.Printf("still finding backup created by schedule %s ...\n", o.ScheduleName) + return false, nil + }) + Expect(err).To(Succeed(), fmt.Sprintf("Failed to check schedule %s created backup with err %v", o.ScheduleName, err)) + }) + return nil +} + +func (o *OrderedResources) Clean() error { + if !o.VeleroCfg.Debug { + Expect(VeleroScheduleDelete(o.Ctx, o.VeleroCfg.VeleroCLI, o.VeleroCfg.VeleroNamespace, o.ScheduleName)).To(Succeed()) + Expect(o.TestCase.Clean()).To(Succeed()) + } + return nil +} + func (o *OrderedResources) DeleteBackups() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() - veleroCfg := o.VeleroCfg backupList := new(velerov1api.BackupList) - if err := o.Client.Kubebuilder.List(ctx, backupList, &kbclient.ListOptions{Namespace: veleroCfg.VeleroNamespace}); err != nil { - return fmt.Errorf("failed to list backup object in %s namespace with err %v", veleroCfg.VeleroNamespace, err) + if err := o.Client.Kubebuilder.List(o.Ctx, backupList, &kbclient.ListOptions{Namespace: o.VeleroCfg.VeleroNamespace}); err != nil { + return fmt.Errorf("failed to list backup object in %s namespace with err %v", o.VeleroCfg.VeleroNamespace, err) } for _, backup := range backupList.Items { - if err := VeleroBackupDelete(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, backup.Name); err != nil { + if err := VeleroBackupDelete(o.Ctx, o.VeleroCfg.VeleroCLI, o.VeleroCfg.VeleroNamespace, backup.Name); err != nil { return err } } diff --git a/test/e2e/schedule/schedule-backup-creation.go b/test/e2e/schedule/schedule-backup-creation.go index 3e26149b7..02f5b1540 100644 --- a/test/e2e/schedule/schedule-backup-creation.go +++ b/test/e2e/schedule/schedule-backup-creation.go @@ -31,9 +31,13 @@ type ScheduleBackupCreation struct { podSleepDuration time.Duration } -var ScheduleBackupCreationTest func() = TestFunc(&ScheduleBackupCreation{namespace: "sch1", TestCase: TestCase{NSBaseName: "schedule-backup-creation-test"}}) +var ScheduleBackupCreationTest func() = TestFunc(&ScheduleBackupCreation{}) func (n *ScheduleBackupCreation) Init() error { + n.TestCase.Init() + n.CaseBaseName = "schedule-backup-creation-test" + n.UUIDgen + n.ScheduleName = n.ScheduleName + "schedule-" + UUIDgen.String() + n.namespace = n.CaseBaseName n.VeleroCfg = VeleroCfg n.Client = *n.VeleroCfg.ClientToInstallVelero n.Period = 3 // Unit is minute @@ -53,14 +57,6 @@ func (n *ScheduleBackupCreation) Init() error { n.volume = "volume-1" n.podName = "pod-1" n.pvcName = "pvc-1" - return nil -} - -func (n *ScheduleBackupCreation) StartRun() error { - n.namespace = fmt.Sprintf("%s-%s", n.NSBaseName, "ns") - n.ScheduleName = n.ScheduleName + "schedule-" + UUIDgen.String() - n.RestoreName = n.RestoreName + "restore-ns-mapping-" + UUIDgen.String() - n.ScheduleArgs = []string{ "--include-namespaces", n.namespace, "--schedule=*/" + fmt.Sprintf("%v", n.Period) + " * * * *", @@ -68,24 +64,24 @@ func (n *ScheduleBackupCreation) StartRun() error { Expect(n.Period < 30).To(Equal(true)) return nil } + func (p *ScheduleBackupCreation) CreateResources() error { + p.Ctx, p.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) By(fmt.Sprintf("Create namespace %s", p.namespace), func() { - Expect(CreateNamespace(context.Background(), p.Client, p.namespace)).To(Succeed(), + Expect(CreateNamespace(p.Ctx, p.Client, p.namespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", p.namespace)) }) By(fmt.Sprintf("Create pod %s in namespace %s", p.podName, p.namespace), func() { _, err := CreatePod(p.Client, p.namespace, p.podName, "default", p.pvcName, []string{p.volume}, nil, p.podAnn) Expect(err).To(Succeed()) - err = WaitForPods(context.Background(), p.Client, p.namespace, []string{p.podName}) + err = WaitForPods(p.Ctx, p.Client, p.namespace, []string{p.podName}) Expect(err).To(Succeed()) }) return nil } func (n *ScheduleBackupCreation) Backup() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() // Wait until the beginning of the given period to create schedule, it will give us // a predictable period to wait for the first scheduled backup, and verify no immediate // scheduled backup was created between schedule creation and first scheduled backup. @@ -95,9 +91,9 @@ func (n *ScheduleBackupCreation) Backup() error { now := time.Now().Minute() triggerNow := now % n.Period if triggerNow == 0 { - Expect(VeleroScheduleCreate(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, n.ScheduleName, n.ScheduleArgs)).To(Succeed(), func() string { + Expect(VeleroScheduleCreate(n.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, n.ScheduleName, n.ScheduleArgs)).To(Succeed(), func() string { RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", "") - return "Fail to restore workload" + return "Fail to create schedule" }) break } @@ -114,7 +110,7 @@ func (n *ScheduleBackupCreation) Backup() error { mi, _ := time.ParseDuration("60s") time.Sleep(n.podSleepDuration + mi) bMap := make(map[string]string) - backupsInfo, err := GetScheduledBackupsCreationTime(context.Background(), VeleroCfg.VeleroCLI, "default", n.ScheduleName) + backupsInfo, err := GetScheduledBackupsCreationTime(n.Ctx, VeleroCfg.VeleroCLI, "default", n.ScheduleName) Expect(err).To(Succeed()) Expect(len(backupsInfo) == i).To(Equal(true)) for index, bi := range backupsInfo { @@ -133,6 +129,10 @@ func (n *ScheduleBackupCreation) Backup() error { return nil } -func (n *ScheduleBackupCreation) Restore() error { +func (n *ScheduleBackupCreation) Clean() error { + if !n.VeleroCfg.Debug { + Expect(VeleroScheduleDelete(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed()) + Expect(n.TestCase.Clean()).To(Succeed()) + } return nil } diff --git a/test/e2e/schedule/schedule.go b/test/e2e/schedule/schedule.go index 92100880e..eb1b0d9e4 100644 --- a/test/e2e/schedule/schedule.go +++ b/test/e2e/schedule/schedule.go @@ -25,9 +25,14 @@ type ScheduleBackup struct { verifyTimes int } -var ScheduleBackupTest func() = TestFunc(&ScheduleBackup{TestCase: TestCase{NSBaseName: "schedule-test"}}) +var ScheduleBackupTest func() = TestFunc(&ScheduleBackup{}) func (n *ScheduleBackup) Init() error { + n.TestCase.Init() + n.CaseBaseName = "schedule-backup-" + n.UUIDgen + n.NSIncluded = &[]string{n.CaseBaseName} + n.ScheduleName = "schedule-" + n.CaseBaseName + n.VeleroCfg = VeleroCfg n.Client = *n.VeleroCfg.ClientToInstallVelero n.Period = 3 // Unit is minute @@ -37,14 +42,6 @@ func (n *ScheduleBackup) Init() error { FailedMSG: "Failed to schedule a backup", Text: "should backup periodly according to the schedule", } - return nil -} - -func (n *ScheduleBackup) StartRun() error { - n.NSIncluded = &[]string{fmt.Sprintf("%s-%s", n.NSBaseName, "ns")} - n.ScheduleName = n.ScheduleName + "schedule-" + UUIDgen.String() - n.RestoreName = n.RestoreName + "restore-ns-mapping-" + UUIDgen.String() - n.ScheduleArgs = []string{ "--include-namespaces", strings.Join(*n.NSIncluded, ","), "--schedule=*/" + fmt.Sprintf("%v", n.Period) + " * * * *", @@ -52,14 +49,14 @@ func (n *ScheduleBackup) StartRun() error { Expect(n.Period < 30).To(Equal(true)) return nil } + func (n *ScheduleBackup) CreateResources() error { - ctx, ctxCanel := context.WithTimeout(context.Background(), 60*time.Minute) - defer ctxCanel() + n.Ctx, n.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) for _, ns := range *n.NSIncluded { By(fmt.Sprintf("Creating namespaces %s ......\n", ns), func() { - Expect(CreateNamespace(ctx, n.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns)) + Expect(CreateNamespace(n.Ctx, n.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns)) }) - configmaptName := n.NSBaseName + configmaptName := n.CaseBaseName fmt.Printf("Creating configmap %s in namespaces ...%s\n", configmaptName, ns) _, err := CreateConfigMap(n.Client.ClientGo, ns, configmaptName, nil, nil) Expect(err).To(Succeed(), fmt.Sprintf("failed to create configmap in the namespace %q", ns)) @@ -70,8 +67,6 @@ func (n *ScheduleBackup) CreateResources() error { } func (n *ScheduleBackup) Backup() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer ctxCancel() // Wait until the beginning of the given period to create schedule, it will give us // a predictable period to wait for the first scheduled backup, and verify no immediate // scheduled backup was created between schedule creation and first scheduled backup. @@ -81,21 +76,17 @@ func (n *ScheduleBackup) Backup() error { now := time.Now().Minute() triggerNow := now % n.Period if triggerNow == 0 { - Expect(VeleroScheduleCreate(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, n.ScheduleName, n.ScheduleArgs)).To(Succeed(), func() string { + Expect(VeleroScheduleCreate(n.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, n.ScheduleName, n.ScheduleArgs)).To(Succeed(), func() string { RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", "") - return "Fail to restore workload" + return "Fail to create schedule" }) break } } }) - return nil -} -func (n *ScheduleBackup) Destroy() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 60*time.Minute) - defer ctxCancel() + By(fmt.Sprintf("Schedule %s is created without any delay\n", n.ScheduleName), func() { - creationTimestamp, err := GetSchedule(context.Background(), VeleroCfg.VeleroNamespace, n.ScheduleName) + creationTimestamp, err := GetSchedule(n.Ctx, VeleroCfg.VeleroNamespace, n.ScheduleName) Expect(err).To(Succeed()) creationTime, err := time.Parse(time.RFC3339, strings.Replace(creationTimestamp, "'", "", -1)) @@ -113,7 +104,7 @@ func (n *ScheduleBackup) Destroy() error { fmt.Printf("Get backup for #%d time at %v\n", i, now) //Ignore the last minute in the period avoiding met the 1st backup by schedule if i != n.Period-1 { - backupsInfo, err := GetScheduledBackupsCreationTime(context.Background(), VeleroCfg.VeleroCLI, "default", n.ScheduleName) + backupsInfo, err := GetScheduledBackupsCreationTime(n.Ctx, VeleroCfg.VeleroCLI, "default", n.ScheduleName) Expect(err).To(Succeed()) Expect(len(backupsInfo) == 0).To(Equal(true)) } @@ -121,7 +112,7 @@ func (n *ScheduleBackup) Destroy() error { }) By("Delay one more minute to make sure the new backup was created in the given period", func() { - time.Sleep(1 * time.Minute) + time.Sleep(time.Minute) }) By(fmt.Sprintf("Get backups every %d minute, and backups count should increase 1 more step in the same pace\n", n.Period), func() { @@ -129,7 +120,7 @@ func (n *ScheduleBackup) Destroy() error { fmt.Printf("Start to sleep %d minute #%d time...\n", n.Period, i+1) time.Sleep(time.Duration(n.Period) * time.Minute) bMap := make(map[string]string) - backupsInfo, err := GetScheduledBackupsCreationTime(context.Background(), VeleroCfg.VeleroCLI, "default", n.ScheduleName) + backupsInfo, err := GetScheduledBackupsCreationTime(n.Ctx, VeleroCfg.VeleroCLI, "default", n.ScheduleName) Expect(err).To(Succeed()) Expect(len(backupsInfo) == i+2).To(Equal(true)) for index, bi := range backupsInfo { @@ -149,24 +140,18 @@ func (n *ScheduleBackup) Destroy() error { n.BackupName = strings.Replace(n.randBackupName, " ", "", -1) By("Delete all namespaces", func() { - Expect(CleanupNamespacesWithPoll(ctx, n.Client, n.NSBaseName)).To(Succeed(), "Could cleanup retrieve namespaces") + Expect(CleanupNamespacesWithPoll(n.Ctx, n.Client, n.CaseBaseName)).To(Succeed(), "Could cleanup retrieve namespaces") }) - n.RestoreArgs = []string{ - "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", n.RestoreName, - "--from-backup", n.BackupName, - "--wait", - } - - backupsInfo, err := GetScheduledBackupsCreationTime(context.Background(), VeleroCfg.VeleroCLI, "default", n.ScheduleName) + backupsInfo, err := GetScheduledBackupsCreationTime(n.Ctx, VeleroCfg.VeleroCLI, "default", n.ScheduleName) Expect(err).To(Succeed(), fmt.Sprintf("Fail to get backups from schedule %s", n.ScheduleName)) fmt.Println(backupsInfo) backupCount := len(backupsInfo) By(fmt.Sprintf("Pause schedule %s ......\n", n.ScheduleName), func() { - Expect(VeleroSchedulePause(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed(), func() string { + Expect(VeleroSchedulePause(n.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed(), func() string { RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", "") - return "Fail to restore workload" + return "Fail to pause schedule" }) }) @@ -176,7 +161,7 @@ func (n *ScheduleBackup) Destroy() error { time.Sleep(sleepDuration) }) - backupsInfo, err = GetScheduledBackupsCreationTime(context.Background(), VeleroCfg.VeleroCLI, "default", n.ScheduleName) + backupsInfo, err = GetScheduledBackupsCreationTime(n.Ctx, VeleroCfg.VeleroCLI, "default", n.ScheduleName) Expect(err).To(Succeed(), fmt.Sprintf("Fail to get backups from schedule %s", n.ScheduleName)) backupCountPostPause := len(backupsInfo) @@ -187,7 +172,7 @@ func (n *ScheduleBackup) Destroy() error { }) By(fmt.Sprintf("Unpause schedule %s ......\n", n.ScheduleName), func() { - Expect(VeleroScheduleUnpause(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed(), func() string { + Expect(VeleroScheduleUnpause(n.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed(), func() string { RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", "") return "Fail to unpause schedule" }) @@ -197,7 +182,7 @@ func (n *ScheduleBackup) Destroy() error { time.Sleep(sleepDuration) }) - backupsInfo, err = GetScheduledBackupsCreationTime(context.Background(), VeleroCfg.VeleroCLI, "default", n.ScheduleName) + backupsInfo, err = GetScheduledBackupsCreationTime(n.Ctx, VeleroCfg.VeleroCLI, "default", n.ScheduleName) Expect(err).To(Succeed(), fmt.Sprintf("Fail to get backups from schedule %s", n.ScheduleName)) fmt.Println(backupsInfo) backupCountPostUnpause := len(backupsInfo) @@ -211,7 +196,7 @@ func (n *ScheduleBackup) Destroy() error { func (n *ScheduleBackup) Verify() error { By("Namespaces were restored", func() { for _, ns := range *n.NSIncluded { - configmap, err := GetConfigmap(n.Client.ClientGo, ns, n.NSBaseName) + configmap, err := GetConfigmap(n.Client.ClientGo, ns, n.CaseBaseName) fmt.Printf("Restored configmap is %v\n", configmap) Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("failed to list configmap in namespace: %q\n", ns)) } @@ -219,3 +204,15 @@ func (n *ScheduleBackup) Verify() error { }) return nil } + +func (n *ScheduleBackup) Destroy() error { + return nil +} + +func (n *ScheduleBackup) Clean() error { + if !n.VeleroCfg.Debug { + Expect(VeleroScheduleDelete(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed()) + Expect(n.TestCase.Clean()).To(Succeed()) + } + return nil +} diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index 3d1c06017..fd51ad420 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -20,6 +20,7 @@ import ( "context" "flag" "fmt" + "math/rand" "strings" "time" @@ -42,7 +43,6 @@ depends on your test patterns. */ type VeleroBackupRestoreTest interface { Init() error - StartRun() error CreateResources() error Backup() error Destroy() error @@ -62,7 +62,7 @@ type TestMSG struct { type TestCase struct { BackupName string RestoreName string - NSBaseName string + CaseBaseName string BackupArgs []string RestoreArgs []string NamespacesTotal int @@ -72,23 +72,20 @@ type TestCase struct { UseVolumeSnapshots bool VeleroCfg VeleroConfig RestorePhaseExpect velerov1api.RestorePhase - Timeout time.Duration + Ctx context.Context + CtxCancel context.CancelFunc + UUIDgen string } func TestFunc(test VeleroBackupRestoreTest) func() { return func() { Expect(test.Init()).To(Succeed(), "Failed to instantiate test cases") veleroCfg := test.GetTestCase().VeleroCfg - // If TestCase.Timeout is not set, then make 10 minutes as default value for backup - // or restore CLI - if test.GetTestCase().Timeout == 0 { - test.GetTestCase().Timeout = 10 * time.Minute - } BeforeEach(func() { flag.Parse() veleroCfg := test.GetTestCase().VeleroCfg // TODO: Skip nodeport test until issue https://github.com/kubernetes/kubernetes/issues/114384 fixed - if veleroCfg.CloudProvider == "azure" && strings.Contains(test.GetTestCase().NSBaseName, "nodeport") { + if veleroCfg.CloudProvider == "azure" && strings.Contains(test.GetTestCase().CaseBaseName, "nodeport") { Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS") } if veleroCfg.InstallVelero { @@ -116,11 +113,9 @@ func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() { var veleroCfg VeleroConfig for k := range tests { Expect(tests[k].Init()).To(Succeed(), fmt.Sprintf("Failed to instantiate test %s case", tests[k].GetTestMsg().Desc)) - if tests[k].GetTestCase().Timeout == 0 { - tests[k].GetTestCase().Timeout = 10 * time.Minute - } veleroCfg = tests[k].GetTestCase().VeleroCfg useVolumeSnapshots = tests[k].GetTestCase().UseVolumeSnapshots + defer tests[k].GetTestCase().CtxCancel() } BeforeEach(func() { @@ -155,22 +150,23 @@ func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() { } func (t *TestCase) Init() error { + t.Ctx, t.CtxCancel = context.WithTimeout(context.Background(), 1*time.Hour) + t.UUIDgen = t.GenerateUUID() return nil } +func (t *TestCase) GenerateUUID() string { + rand.Seed(time.Now().UnixNano()) + return fmt.Sprintf("%08d", rand.Intn(100000000)) +} + func (t *TestCase) CreateResources() error { return nil } -func (t *TestCase) StartRun() error { - return nil -} - func (t *TestCase) Backup() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), t.Timeout) - defer ctxCancel() veleroCfg := t.GetTestCase().VeleroCfg - if err := VeleroBackupExec(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, t.BackupArgs); err != nil { + if err := VeleroBackupExec(t.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, t.BackupArgs); err != nil { RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, "") return errors.Wrapf(err, "Failed to backup resources") } @@ -178,22 +174,22 @@ func (t *TestCase) Backup() error { } func (t *TestCase) Destroy() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 60*time.Minute) - defer ctxCancel() - By(fmt.Sprintf("Start to destroy namespace %s......", t.NSBaseName), func() { - Expect(CleanupNamespacesWithPoll(ctx, t.Client, t.NSBaseName)).To(Succeed(), "Could cleanup retrieve namespaces") + By(fmt.Sprintf("Start to destroy namespace %s......", t.CaseBaseName), func() { + Expect(CleanupNamespacesWithPoll(t.Ctx, t.Client, t.CaseBaseName)).To(Succeed(), "Could cleanup retrieve namespaces") }) return nil } func (t *TestCase) Restore() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), t.Timeout) - defer ctxCancel() + if len(t.RestoreArgs) == 0 { + return nil + } + veleroCfg := t.GetTestCase().VeleroCfg // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if t.UseVolumeSnapshots { + if t.UseVolumeSnapshots && veleroCfg.CloudProvider != "vsphere" { fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") time.Sleep(5 * time.Minute) } @@ -202,7 +198,7 @@ func (t *TestCase) Restore() error { if t.RestorePhaseExpect == "" { t.RestorePhaseExpect = velerov1api.RestorePhaseCompleted } - Expect(VeleroRestoreExec(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.RestoreName, t.RestoreArgs, t.RestorePhaseExpect)).To(Succeed(), func() string { + Expect(VeleroRestoreExec(t.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.RestoreName, t.RestoreArgs, t.RestorePhaseExpect)).To(Succeed(), func() string { RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", t.RestoreName) return "Fail to restore workload" }) @@ -215,15 +211,13 @@ func (t *TestCase) Verify() error { } func (t *TestCase) Clean() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), 60*time.Minute) - defer ctxCancel() veleroCfg := t.GetTestCase().VeleroCfg if !veleroCfg.Debug { - By(fmt.Sprintf("Clean namespace with prefix %s after test", t.NSBaseName), func() { - CleanupNamespaces(ctx, t.Client, t.NSBaseName) + By(fmt.Sprintf("Clean namespace with prefix %s after test", t.CaseBaseName), func() { + CleanupNamespaces(t.Ctx, t.Client, t.CaseBaseName) }) By("Clean backups after test", func() { - DeleteBackups(ctx, t.Client) + DeleteBackups(t.Ctx, t.Client) }) } return nil @@ -236,35 +230,38 @@ func (t *TestCase) GetTestMsg() *TestMSG { func (t *TestCase) GetTestCase() *TestCase { return t } + func RunTestCase(test VeleroBackupRestoreTest) error { - fmt.Printf("Running test case %s\n", test.GetTestMsg().Desc) + fmt.Printf("Running test case %s %s\n", test.GetTestMsg().Desc, time.Now().Format("2006-01-02 15:04:05")) if test == nil { return errors.New("No case should be tested") } defer test.Clean() - err := test.StartRun() - if err != nil { - return err - } - err = test.CreateResources() + fmt.Printf("CreateResources %s\n", time.Now().Format("2006-01-02 15:04:05")) + err := test.CreateResources() if err != nil { return err } + fmt.Printf("Backup %s\n", time.Now().Format("2006-01-02 15:04:05")) err = test.Backup() if err != nil { return err } + fmt.Printf("Destroy %s\n", time.Now().Format("2006-01-02 15:04:05")) err = test.Destroy() if err != nil { return err } + fmt.Printf("Restore %s\n", time.Now().Format("2006-01-02 15:04:05")) err = test.Restore() if err != nil { return err } + fmt.Printf("Verify %s\n", time.Now().Format("2006-01-02 15:04:05")) err = test.Verify() if err != nil { return err } + fmt.Printf("Finish run test %s\n", time.Now().Format("2006-01-02 15:04:05")) return nil } diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index c9f0c00ab..d2fe8830d 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -181,7 +181,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC snapshotCheckPoint.NamespaceBackedUp = upgradeNamespace By("Snapshot should be created in cloud object store", func() { snapshotCheckPoint, err := GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 2, - upgradeNamespace, backupName, KibishiiPodNameList) + upgradeNamespace, backupName, KibishiiPVCNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, diff --git a/test/e2e/util/k8s/deployment.go b/test/e2e/util/k8s/deployment.go index bc4c30a3d..bdd26dde8 100644 --- a/test/e2e/util/k8s/deployment.go +++ b/test/e2e/util/k8s/deployment.go @@ -26,6 +26,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + + "github.com/vmware-tanzu/velero/pkg/util/boolptr" ) const ( @@ -52,6 +54,19 @@ func NewDeployment(name, ns string, replicas int32, labels map[string]string, co Name: "container-busybox", Image: "gcr.io/velero-gcp/busybox:latest", Command: []string{"sleep", "1000000"}, + // Make pod obeys the restricted pod security standards. + SecurityContext: &v1.SecurityContext{ + AllowPrivilegeEscalation: boolptr.False(), + Capabilities: &v1.Capabilities{ + Drop: []v1.Capability{"ALL"}, + }, + RunAsNonRoot: boolptr.True(), + RunAsUser: func(i int64) *int64 { return &i }(65534), + RunAsGroup: func(i int64) *int64 { return &i }(65534), + SeccompProfile: &v1.SeccompProfile{ + Type: v1.SeccompProfileTypeRuntimeDefault, + }, + }, }, } } diff --git a/test/e2e/util/k8s/namespace.go b/test/e2e/util/k8s/namespace.go index e19582dd7..c74f8e293 100644 --- a/test/e2e/util/k8s/namespace.go +++ b/test/e2e/util/k8s/namespace.go @@ -69,7 +69,13 @@ func GetNamespace(ctx context.Context, client TestClient, namespace string) (*co func DeleteNamespace(ctx context.Context, client TestClient, namespace string, wait bool) error { tenMinuteTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*10) defer ctxCancel() - if err := client.ClientGo.CoreV1().Namespaces().Delete(context.TODO(), namespace, metav1.DeleteOptions{}); err != nil { + + var zero int64 = 0 + policy := metav1.DeletePropagationForeground + if err := client.ClientGo.CoreV1().Namespaces().Delete(context.TODO(), namespace, metav1.DeleteOptions{ + GracePeriodSeconds: &zero, + PropagationPolicy: &policy, + }); err != nil { return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) } if !wait { @@ -89,14 +95,14 @@ func DeleteNamespace(ctx context.Context, client TestClient, namespace string, w }) } -func CleanupNamespacesWithPoll(ctx context.Context, client TestClient, nsBaseName string) error { +func CleanupNamespacesWithPoll(ctx context.Context, client TestClient, CaseBaseName string) error { namespaces, err := client.ClientGo.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) if err != nil { return errors.Wrap(err, "Could not retrieve namespaces") } for _, checkNamespace := range namespaces.Items { - if strings.HasPrefix(checkNamespace.Name, nsBaseName) { + if strings.HasPrefix(checkNamespace.Name, CaseBaseName) { err := DeleteNamespace(ctx, client, checkNamespace.Name, true) if err != nil { return errors.Wrapf(err, "Could not delete namespace %s", checkNamespace.Name) @@ -107,13 +113,13 @@ func CleanupNamespacesWithPoll(ctx context.Context, client TestClient, nsBaseNam return nil } -func CleanupNamespaces(ctx context.Context, client TestClient, nsBaseName string) error { +func CleanupNamespaces(ctx context.Context, client TestClient, CaseBaseName string) error { namespaces, err := client.ClientGo.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) if err != nil { return errors.Wrap(err, "Could not retrieve namespaces") } for _, checkNamespace := range namespaces.Items { - if strings.HasPrefix(checkNamespace.Name, nsBaseName) { + if strings.HasPrefix(checkNamespace.Name, CaseBaseName) { err = client.ClientGo.CoreV1().Namespaces().Delete(ctx, checkNamespace.Name, metav1.DeleteOptions{}) if err != nil { return errors.Wrapf(err, "Could not delete namespace %s", checkNamespace.Name) diff --git a/test/e2e/util/k8s/pod.go b/test/e2e/util/k8s/pod.go index d730d6d5f..7a9f60f46 100644 --- a/test/e2e/util/k8s/pod.go +++ b/test/e2e/util/k8s/pod.go @@ -22,8 +22,10 @@ import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" - + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/vmware-tanzu/velero/pkg/util/boolptr" ) func CreatePod(client TestClient, ns, name, sc, pvcName string, volumeNameList []string, pvcAnn, ann map[string]string) (*corev1.Pod, error) { @@ -74,6 +76,19 @@ func CreatePod(client TestClient, ns, name, sc, pvcName string, volumeNameList [ Image: "gcr.io/velero-gcp/busybox", Command: []string{"sleep", "3600"}, VolumeMounts: vmList, + // Make pod obeys the restricted pod security standards. + SecurityContext: &v1.SecurityContext{ + AllowPrivilegeEscalation: boolptr.False(), + Capabilities: &v1.Capabilities{ + Drop: []v1.Capability{"ALL"}, + }, + RunAsNonRoot: boolptr.True(), + RunAsUser: func(i int64) *int64 { return &i }(65534), + RunAsGroup: func(i int64) *int64 { return &i }(65534), + SeccompProfile: &v1.SeccompProfile{ + Type: v1.SeccompProfileTypeRuntimeDefault, + }, + }, }, }, Volumes: volumes, diff --git a/test/e2e/util/k8s/rbac.go b/test/e2e/util/k8s/rbac.go index 7486e8761..ac7cfb56d 100644 --- a/test/e2e/util/k8s/rbac.go +++ b/test/e2e/util/k8s/rbac.go @@ -75,7 +75,7 @@ func GetClusterRoleBinding(ctx context.Context, client TestClient, rolebinding s return client.ClientGo.RbacV1().ClusterRoleBindings().Get(ctx, rolebinding, metav1.GetOptions{}) } -func CleanupClusterRole(ctx context.Context, client TestClient, nsBaseName string) error { +func CleanupClusterRole(ctx context.Context, client TestClient, CaseBaseName string) error { clusterroles, err := client.ClientGo.RbacV1().ClusterRoles().List(ctx, metav1.ListOptions{}) if err != nil { @@ -83,7 +83,7 @@ func CleanupClusterRole(ctx context.Context, client TestClient, nsBaseName strin } for _, checkClusterRole := range clusterroles.Items { - if strings.HasPrefix(checkClusterRole.Name, "clusterrole-"+nsBaseName) { + if strings.HasPrefix(checkClusterRole.Name, "clusterrole-"+CaseBaseName) { fmt.Printf("Cleaning up clusterrole %s\n", checkClusterRole.Name) err = client.ClientGo.RbacV1().ClusterRoles().Delete(ctx, checkClusterRole.Name, metav1.DeleteOptions{}) if err != nil { @@ -94,7 +94,7 @@ func CleanupClusterRole(ctx context.Context, client TestClient, nsBaseName strin return nil } -func CleanupClusterRoleBinding(ctx context.Context, client TestClient, nsBaseName string) error { +func CleanupClusterRoleBinding(ctx context.Context, client TestClient, CaseBaseName string) error { clusterrolebindings, err := client.ClientGo.RbacV1().ClusterRoleBindings().List(ctx, metav1.ListOptions{}) if err != nil { @@ -102,7 +102,7 @@ func CleanupClusterRoleBinding(ctx context.Context, client TestClient, nsBaseNam } for _, checkClusterRoleBinding := range clusterrolebindings.Items { - if strings.HasPrefix(checkClusterRoleBinding.Name, "clusterrolebinding-"+nsBaseName) { + if strings.HasPrefix(checkClusterRoleBinding.Name, "clusterrolebinding-"+CaseBaseName) { fmt.Printf("Cleaning up clusterrolebinding %s\n", checkClusterRoleBinding.Name) err = client.ClientGo.RbacV1().ClusterRoleBindings().Delete(ctx, checkClusterRoleBinding.Name, metav1.DeleteOptions{}) if err != nil { diff --git a/test/e2e/util/kibishii/kibishii_utils.go b/test/e2e/util/kibishii/kibishii_utils.go index f14e702ec..66ad2116d 100644 --- a/test/e2e/util/kibishii/kibishii_utils.go +++ b/test/e2e/util/kibishii/kibishii_utils.go @@ -49,12 +49,12 @@ type KibishiiData struct { } var DefaultKibishiiData = &KibishiiData{2, 10, 10, 1024, 1024, 0, 2} -var KibishiiPodNameList = []string{"kibishii-deployment-0", "kibishii-deployment-1"} +var KibishiiPVCNameList = []string{"kibishii-data-kibishii-deployment-0", "kibishii-data-kibishii-deployment-1"} // RunKibishiiTests runs kibishii tests on the provider. func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string, useVolumeSnapshots, defaultVolumesToFsBackup bool) error { - pvCount := len(KibishiiPodNameList) + pvCount := len(KibishiiPVCNameList) client := *veleroCfg.ClientToInstallVelero oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60) defer ctxCancel() @@ -80,12 +80,13 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc } } }() - + fmt.Printf("KibishiiPrepareBeforeBackup %s\n", time.Now().Format("2006-01-02 15:04:05")) if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, kibishiiNamespace, registryCredentialFile, veleroFeatures, kibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData); err != nil { return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", kibishiiNamespace) } + fmt.Printf("KibishiiPrepareBeforeBackup done %s\n", time.Now().Format("2006-01-02 15:04:05")) var BackupCfg BackupConfig BackupCfg.BackupName = backupName @@ -95,20 +96,14 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc BackupCfg.DefaultVolumesToFsBackup = defaultVolumesToFsBackup BackupCfg.Selector = "" BackupCfg.ProvideSnapshotsVolumeParam = veleroCfg.ProvideSnapshotsVolumeParam + + fmt.Printf("VeleroBackupNamespace %s\n", time.Now().Format("2006-01-02 15:04:05")) if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, BackupCfg); err != nil { RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "") return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace) } - var snapshotCheckPoint SnapshotCheckPoint - var err error - pvbs, err := GetPVB(oneHourTimeout, veleroCfg.VeleroNamespace, kibishiiNamespace) - if err != nil { - return errors.Wrapf(err, "failed to get PVB for namespace %s", kibishiiNamespace) - } + fmt.Printf("VeleroBackupNamespace done %s\n", time.Now().Format("2006-01-02 15:04:05")) if useVolumeSnapshots { - if len(pvbs) != 0 { - return errors.New(fmt.Sprintln("PVB count should be 0 in namespace %s", kibishiiNamespace)) - } if providerName == "vsphere" { // Wait for uploads started by the Velero Plug-in for vSphere to complete // TODO - remove after upload progress monitoring is implemented @@ -117,7 +112,7 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc return errors.Wrapf(err, "Error waiting for uploads to complete") } } - snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, 2, kibishiiNamespace, backupName, KibishiiPodNameList) + snapshotCheckPoint, err := GetSnapshotCheckPoint(client, veleroCfg, 2, kibishiiNamespace, backupName, KibishiiPVCNameList) if err != nil { return errors.Wrap(err, "Fail to get snapshot checkpoint") } @@ -128,6 +123,10 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } } else { + pvbs, err := GetPVB(oneHourTimeout, veleroCfg.VeleroNamespace, kibishiiNamespace) + if err != nil { + return errors.Wrapf(err, "failed to get PVB for namespace %s", kibishiiNamespace) + } if len(pvbs) != pvCount { return errors.New(fmt.Sprintf("PVB count %d should be %d in namespace %s", len(pvbs), pvCount, kibishiiNamespace)) } @@ -147,11 +146,14 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc time.Sleep(5 * time.Minute) if strings.EqualFold(veleroFeatures, "EnableCSI") { _, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 0, - kibishiiNamespace, backupName, KibishiiPodNameList) + kibishiiNamespace, backupName, KibishiiPVCNameList) + if err != nil { + return errors.Wrap(err, "failed to get snapshot checkPoint") + } } else { err = SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, - backupName, snapshotCheckPoint) + backupName, SnapshotCheckPoint{}) if err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } @@ -159,7 +161,7 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc } } - fmt.Printf("Simulating a disaster by removing namespace %s\n", kibishiiNamespace) + fmt.Printf("Simulating a disaster by removing namespace %s %s\n", kibishiiNamespace, time.Now().Format("2006-01-02 15:04:05")) if err := DeleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil { return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace) } @@ -172,11 +174,12 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc time.Sleep(5 * time.Minute) } + fmt.Printf("VeleroRestore %s\n", time.Now().Format("2006-01-02 15:04:05")) if err := VeleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName, ""); err != nil { RunDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName) return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName) } - if !useVolumeSnapshots { + if !useVolumeSnapshots && providerName != "vsphere" { pvrs, err := GetPVR(oneHourTimeout, veleroCfg.VeleroNamespace, kibishiiNamespace) if err != nil { return errors.Wrapf(err, "failed to get PVR for namespace %s", kibishiiNamespace) @@ -184,11 +187,11 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc return errors.New(fmt.Sprintf("PVR count %d is not as expected %d", len(pvrs), pvCount)) } } - + fmt.Printf("KibishiiVerifyAfterRestore %s\n", time.Now().Format("2006-01-02 15:04:05")) if err := KibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout, DefaultKibishiiData); err != nil { return errors.Wrapf(err, "Error verifying kibishii after restore") } - fmt.Printf("kibishii test completed successfully\n") + fmt.Printf("kibishii test completed successfully %s\n", time.Now().Format("2006-01-02 15:04:05")) return nil } @@ -287,35 +290,25 @@ func waitForKibishiiPods(ctx context.Context, client TestClient, kibishiiNamespa func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClient, providerName, kibishiiNamespace, registryCredentialFile, veleroFeatures, kibishiiDirectory string, useVolumeSnapshots bool, kibishiiData *KibishiiData) error { - serviceAccountName := "default" - - // wait until the service account is created before patch the image pull secret - if err := WaitUntilServiceAccountCreated(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, 10*time.Minute); err != nil { - return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, kibishiiNamespace) - } - - // add the image pull secret to avoid the image pull limit issue of Docker Hub - if err := PatchServiceAccountWithImagePullSecret(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, registryCredentialFile); err != nil { - return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, kibishiiNamespace) - } - + fmt.Printf("installKibishii %s\n", time.Now().Format("2006-01-02 15:04:05")) if err := installKibishii(oneHourTimeout, kibishiiNamespace, providerName, veleroFeatures, kibishiiDirectory, useVolumeSnapshots); err != nil { return errors.Wrap(err, "Failed to install Kibishii workload") } - // wait for kibishii pod startup // TODO - Fix kibishii so we can check that it is ready to go - fmt.Printf("Waiting for kibishii pods to be ready\n") + fmt.Printf("Waiting for kibishii pods to be ready %s\n", time.Now().Format("2006-01-02 15:04:05")) if err := waitForKibishiiPods(oneHourTimeout, client, kibishiiNamespace); err != nil { return errors.Wrapf(err, "Failed to wait for ready status of kibishii pods in %s", kibishiiNamespace) } if kibishiiData == nil { kibishiiData = DefaultKibishiiData } + fmt.Printf("generateData %s\n", time.Now().Format("2006-01-02 15:04:05")) if err := generateData(oneHourTimeout, kibishiiNamespace, kibishiiData); err != nil { return errors.Wrap(err, "Failed to generate data") } + fmt.Printf("generateData done %s\n", time.Now().Format("2006-01-02 15:04:05")) return nil } @@ -330,7 +323,7 @@ func KibishiiVerifyAfterRestore(client TestClient, kibishiiNamespace string, one if err := waitForKibishiiPods(oneHourTimeout, client, kibishiiNamespace); err != nil { return errors.Wrapf(err, "Failed to wait for ready status of kibishii pods in %s", kibishiiNamespace) } - time.Sleep(60 * time.Second) + // TODO - check that namespace exists fmt.Printf("running kibishii verify\n") if err := verifyData(oneHourTimeout, kibishiiNamespace, kibishiiData); err != nil { diff --git a/test/e2e/util/velero/install.go b/test/e2e/util/velero/install.go index 34a58e802..112e041d5 100644 --- a/test/e2e/util/velero/install.go +++ b/test/e2e/util/velero/install.go @@ -55,6 +55,7 @@ type installOptions struct { } func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig) error { + fmt.Printf("Velero install %s\n", time.Now().Format("2006-01-02 15:04:05")) if veleroCfg.CloudProvider != "kind" { fmt.Printf("For cloud platforms, object store plugin provider will be set as cloud provider") veleroCfg.ObjectStoreProvider = veleroCfg.CloudProvider @@ -109,7 +110,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig) error { RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", "") return errors.WithMessagef(err, "Failed to install Velero in the cluster") } - + fmt.Printf("Finish velero install %s\n", time.Now().Format("2006-01-02 15:04:05")) return nil } diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index 68df0bcf9..500f6cd17 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -627,7 +627,7 @@ func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, actualCount := 0 for _, curLine := range lines { - fmt.Println(curLine) + fmt.Printf("%s %s\n", curLine, time.Now().Format("2006-01-02 15:04:05")) comps := strings.Split(curLine, "=") // SnapshotPhase represents the lifecycle phase of a Snapshot. // New - No work yet, next phase is InProgress @@ -668,7 +668,7 @@ func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, return err } -func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace string, podNameList []string) ([]string, error) { +func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace string, pvcNameList []string) ([]string, error) { checkSnapshotCmd := exec.CommandContext(ctx, "kubectl", "get", "-n", namespace, "snapshots.backupdriver.cnsdp.vmware.com", "-o=jsonpath='{range .items[*]}{.spec.resourceHandle.name}{\"=\"}{.status.snapshotID}{\"\\n\"}{end}'") fmt.Printf("checkSnapshotCmd cmd =%v\n", checkSnapshotCmd) @@ -688,8 +688,8 @@ func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace continue } var Exist bool - for _, podName := range podNameList { - if podName != "" && strings.Contains(curLine, podName) { + for _, pvcName := range pvcNameList { + if pvcName != "" && strings.Contains(curLine, pvcName) { Exist = true break } @@ -1070,12 +1070,12 @@ func GetResticRepositories(ctx context.Context, veleroNamespace, targetNamespace return common.GetListByCmdPipes(ctx, cmds) } -func GetSnapshotCheckPoint(client TestClient, VeleroCfg VeleroConfig, expectCount int, namespaceBackedUp, backupName string, kibishiiPodNameList []string) (SnapshotCheckPoint, error) { +func GetSnapshotCheckPoint(client TestClient, VeleroCfg VeleroConfig, expectCount int, namespaceBackedUp, backupName string, KibishiiPVCNameList []string) (SnapshotCheckPoint, error) { var snapshotCheckPoint SnapshotCheckPoint var err error snapshotCheckPoint.ExpectCount = expectCount snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp - snapshotCheckPoint.PodName = kibishiiPodNameList + snapshotCheckPoint.PodName = KibishiiPVCNameList if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") { snapshotCheckPoint.EnableCSI = true if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, backupName, expectCount); err != nil { diff --git a/tilt-resources/examples/deployment.yaml b/tilt-resources/examples/deployment.yaml index b5bf4fdf6..bda59ae28 100644 --- a/tilt-resources/examples/deployment.yaml +++ b/tilt-resources/examples/deployment.yaml @@ -101,7 +101,7 @@ spec: secret: secretName: cloud-credentials --- -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: